{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":19989,"databundleVersionId":1160143,"sourceType":"competition"},{"sourceId":8185631,"sourceType":"datasetVersion","datasetId":4840143}],"dockerImageVersionId":29963,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!git clone https://github.com/facebookresearch/detr.git   #cloning github repo of detr to import its unique loss","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-04-27T13:25:06.495904Z","iopub.execute_input":"2024-04-27T13:25:06.496372Z","iopub.status.idle":"2024-04-27T13:25:07.499383Z","shell.execute_reply.started":"2024-04-27T13:25:06.496323Z","shell.execute_reply":"2024-04-27T13:25:07.498232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm.autonotebook import tqdm\n\n\n#Torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\n#sklearn\nfrom sklearn.model_selection import StratifiedKFold\n\n#CV\nimport cv2\n\n################# DETR FUCNTIONS FOR LOSS######################## \nimport sys\nsys.path.append('./detr/')\n\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion\n#################################################################\n\n#Albumenatations\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#Glob\nfrom glob import glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-27T17:25:55.234163Z","iopub.execute_input":"2024-04-27T17:25:55.234537Z","iopub.status.idle":"2024-04-27T17:25:56.873014Z","shell.execute_reply.started":"2024-04-27T17:25:55.234500Z","shell.execute_reply":"2024-04-27T17:25:56.872113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils\n\n* AverageMeter - class for averaging loss,metric,etc over epochs","metadata":{}},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:25:56.874939Z","iopub.execute_input":"2024-04-27T17:25:56.875297Z","iopub.status.idle":"2024-04-27T17:25:56.882567Z","shell.execute_reply.started":"2024-04-27T17:25:56.875259Z","shell.execute_reply":"2024-04-27T17:25:56.881657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration\n\nBasic configuration for this model","metadata":{}},{"cell_type":"code","source":"n_folds = 5\nseed = 42\nnull_class_coef = 0.5\nnum_classes = 1\nnum_queries = 100\nBATCH_SIZE = 8\nLR = 5e-5\nlr_dict = {'backbone':0.1,'transformer':1,'embed':1,'final': 5}\nEPOCHS = 2\nmax_norm = 0\nmodel_name = 'detr_resnet50'","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:25:57.132404Z","iopub.execute_input":"2024-04-27T17:25:57.132792Z","iopub.status.idle":"2024-04-27T17:25:57.138772Z","shell.execute_reply.started":"2024-04-27T17:25:57.132748Z","shell.execute_reply":"2024-04-27T17:25:57.138051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed Everything\n\nSeeding everything for reproducible results","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:25:58.073983Z","iopub.execute_input":"2024-04-27T17:25:58.074368Z","iopub.status.idle":"2024-04-27T17:25:58.081243Z","shell.execute_reply.started":"2024-04-27T17:25:58.074318Z","shell.execute_reply":"2024-04-27T17:25:58.080131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:25:58.499782Z","iopub.execute_input":"2024-04-27T17:25:58.500167Z","iopub.status.idle":"2024-04-27T17:25:58.507037Z","shell.execute_reply.started":"2024-04-27T17:25:58.500135Z","shell.execute_reply":"2024-04-27T17:25:58.506252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations\n\n* As suggested by aleksendra in her kernel ,augentations will play a major role and hence took her up advice and use awesome augmentations , cut-mix and other will be included in future versions","metadata":{}},{"cell_type":"code","source":"transform = A.Compose([\n    A.Resize(height=512, width=512),\n    ToTensorV2()\n])","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:25:59.352059Z","iopub.execute_input":"2024-04-27T17:25:59.352433Z","iopub.status.idle":"2024-04-27T17:25:59.357359Z","shell.execute_reply.started":"2024-04-27T17:25:59.352399Z","shell.execute_reply":"2024-04-27T17:25:59.356523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataset\n\n* I hope you have the video by now , DETR accepts data in coco format which is (x,y,w,h)(for those who do not know there are two formats coco and pascal(smin,ymin,xmax,ymax) which are widely used) . So now we need to prepare data in that format","metadata":{}},{"cell_type":"code","source":"class CarlaDataset(Dataset):\n    def __init__(self, path, frame, transforms=None):\n        self.im_path = path + 'front_cam'\n        self.bb_path = path + 'bb_front'\n        self.frame = frame\n        self.transforms = transforms\n\n    def __len__(self) -> int:\n        return len(self.frame)\n\n    def __getitem__(self,index):\n        image_id = self.frame[index]\n#         print(image_id)\n        \n        image = cv2.imread(f'{self.im_path}/{self.frame[index]}.png', cv2.IMREAD_COLOR)\n#         print(f'{self.im_path}/{self.frame[index]}.png')\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        bb = np.load(f'{self.bb_path}/{self.frame[index]}.npy')\n        if len(bb)>0:\n            bb = bb[:, [0,2,1,3]]\n\n        labels = np.zeros(len(bb), dtype=np.int32)\n        \n#         _,h,w = image.shape\n        h,w,_ = image.shape\n#         print(image.shape)\n        bb_norm = np.array(A.augmentations.bbox_utils.normalize_bboxes(bb,rows=h,cols=w))\n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': bb_norm,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image  = sample['image']\n            bb_norm  = np.array(sample['bboxes'])\n            labels = sample['labels']\n\n#         bb_norm = np.array(bb)\n        \n        ## detr uses center_x,center_y,width,height !!\n        if len(bb_norm) > 0:\n            width = bb_norm[:,2]-bb_norm[:,0]\n            height = bb_norm[:,3]-bb_norm[:,1]\n            center_x = bb_norm[:,0]+width/2\n            center_y = bb_norm[:,1]+height/2\n            bb_norm = np.array([center_x, center_y, width, height]).T\n#             print(bb_norm)\n        else:\n            bb_norm = np.zeros((0,4))\n\n        target = {}\n        target['boxes'] = torch.as_tensor(bb_norm,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        \n        return image, target, image_id","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:00.168253Z","iopub.execute_input":"2024-04-27T17:26:00.168656Z","iopub.status.idle":"2024-04-27T17:26:00.189144Z","shell.execute_reply.started":"2024-04-27T17:26:00.168621Z","shell.execute_reply":"2024-04-27T17:26:00.187950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nPATH = '../input/carla-sensor-fusion-v1/sensor-fusion-v2/sensor-fusion-v1/'\n\nframe_list = sorted(list(map(lambda x: x.split('.')[0], os.listdir(PATH+'bev_cam'))))\n\ntrain_frame, valid_frame = train_test_split(frame_list, test_size=0.2, shuffle=True)\n\nprint(len(train_frame))\nprint(len(valid_frame))\n\ntrain_ds = CarlaDataset(PATH, train_frame, transform)\nvalid_ds = CarlaDataset(PATH, valid_frame, transform)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:00.555921Z","iopub.execute_input":"2024-04-27T17:26:00.556255Z","iopub.status.idle":"2024-04-27T17:26:01.415321Z","shell.execute_reply.started":"2024-04-27T17:26:00.556224Z","shell.execute_reply":"2024-04-27T17:26:01.414428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntrain_data_loader = DataLoader(\n    train_ds,\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    num_workers=2,\n    collate_fn=collate_fn\n)\n\nvalid_data_loader = DataLoader(\n    valid_ds,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=2,\n    collate_fn=collate_fn\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:01.417557Z","iopub.execute_input":"2024-04-27T17:26:01.417934Z","iopub.status.idle":"2024-04-27T17:26:01.423999Z","shell.execute_reply.started":"2024-04-27T17:26:01.417895Z","shell.execute_reply":"2024-04-27T17:26:01.423091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_iter = iter(valid_data_loader)\nbatch  = next(valid_iter)\n\nimages,targets,image_id = batch\ntorch.cat([v['boxes'] for v in targets])","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:01.425422Z","iopub.execute_input":"2024-04-27T17:26:01.425772Z","iopub.status.idle":"2024-04-27T17:26:02.014497Z","shell.execute_reply.started":"2024-04-27T17:26:01.425736Z","shell.execute_reply":"2024-04-27T17:26:02.013451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_example(data_loader, nrows, ncols, offset=0):\n    fig, axes = plt.subplots(nrows, ncols, figsize=(20, 10))\n\n    sample_data = next(iter(data_loader))\n\n    for i, ax in enumerate(axes.flatten()):\n        frame = sample_data[2][i]\n        img = sample_data[0][i].cpu().numpy().transpose(1,2,0)\n        h,w,_ = img.shape\n        bb_boxes = sample_data[1][i]['boxes']\n        bb_boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(bb_boxes,h,w)]\n        for box in bb_boxes:\n            x1,y1,x2,y2 = [box[0]-box[2]//2, box[1]-box[3]//2, box[0]+box[2]//2, box[1]+box[3]//2]\n            cv2.rectangle(img,\n                      (x1 - offset, y1 - offset),\n                      (x2 + offset, y2 + offset),\n                      (220, 0, 0), 2)\n        ax.axis('off')\n        ax.imshow(img)\n        ax.set_title(f'Frame: {frame}')","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:02.016817Z","iopub.execute_input":"2024-04-27T17:26:02.017155Z","iopub.status.idle":"2024-04-27T17:26:02.032463Z","shell.execute_reply.started":"2024-04-27T17:26:02.017118Z","shell.execute_reply":"2024-04-27T17:26:02.031348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_example(train_data_loader, 2, 3, offset=5)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:02.280215Z","iopub.execute_input":"2024-04-27T17:26:02.280612Z","iopub.status.idle":"2024-04-27T17:26:04.284247Z","shell.execute_reply.started":"2024-04-27T17:26:02.280577Z","shell.execute_reply":"2024-04-27T17:26:04.282994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\n* Initial DETR model is trained on coco dataset , which has 91 classes + 1 background class , hence we need to modify it to take our own number of classes\n* Also DETR model takes in 100 queries ie ,it outputs total of 100 bboxes for every image , we can very well change that too","metadata":{}},{"cell_type":"code","source":"class DETRModel(nn.Module):\n    def __init__(self,num_classes=1):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        \n        self.model = torch.hub.load('facebookresearch/detr', model_name, pretrained=True)\n        \n        self.out = nn.Linear(in_features=self.model.class_embed.out_features,out_features=num_classes+1)\n        \n    def forward(self,images):\n        d = self.model(images)\n        d['pred_logits'] = self.out(d['pred_logits'])\n        return d\n    \n    def parameter_groups(self):\n        return { \n            'backbone': [p for n,p in self.model.named_parameters()\n                              if ('backbone' in n) and p.requires_grad],\n            'transformer': [p for n,p in self.model.named_parameters() \n                                 if (('transformer' in n) or ('input_proj' in n)) and p.requires_grad],\n            'embed': [p for n,p in self.model.named_parameters()\n                                 if (('class_embed' in n) or ('bbox_embed' in n) or ('query_embed' in n)) \n                           and p.requires_grad],\n            'final': self.out.parameters()\n            }","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:06.144565Z","iopub.execute_input":"2024-04-27T17:26:06.144914Z","iopub.status.idle":"2024-04-27T17:26:06.157339Z","shell.execute_reply.started":"2024-04-27T17:26:06.144882Z","shell.execute_reply":"2024-04-27T17:26:06.156513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DETRModel()\nmodel.parameter_groups().keys()\n#type(model)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:06.832388Z","iopub.execute_input":"2024-04-27T17:26:06.832722Z","iopub.status.idle":"2024-04-27T17:26:10.398105Z","shell.execute_reply.started":"2024-04-27T17:26:06.832693Z","shell.execute_reply":"2024-04-27T17:26:10.397401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matcher and Bipartite Matching Loss\n\nNow we make use of the unique loss that the model uses and for that we need to define the matcher. DETR calcuates three individual losses :\n* Classification Loss for labels(its weight can be set by loss_ce)\n* Bbox Loss (its weight can be set by loss_bbox)\n* Loss for Background class","metadata":{}},{"cell_type":"code","source":"'''\ncode taken from github repo detr , 'code present in engine.py'\n'''\n\nmatcher = HungarianMatcher()\n\nweight_dict = weight_dict = {'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1}\n\nlosses = ['labels', 'boxes', 'cardinality']","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:12.644531Z","iopub.execute_input":"2024-04-27T17:26:12.644918Z","iopub.status.idle":"2024-04-27T17:26:12.651396Z","shell.execute_reply.started":"2024-04-27T17:26:12.644884Z","shell.execute_reply":"2024-04-27T17:26:12.650289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import util.box_ops  as box_ops\n\ndef challenge_metric(outputs,targets):\n    logits = outputs['pred_logits']\n    boxes  = outputs['pred_boxes']\n    return sum(avg_precision(logit[:,0]-logit[:,1],box,target['boxes'])\n            for logit,box,target in zip(logits,boxes,targets))/len(logits)\n\n    return {target['image_id']:avg_precision(logit[:,0]-logit[:,1],box,target['boxes'])\n            for logit,box,target in zip(logits,boxes,targets)}\n\n\n@torch.no_grad()\ndef avg_precision(logit,pboxes,tboxes,reduce=True):\n    idx = logit.gt(0)\n    if sum(idx)==0 and len(tboxes)==0: \n        return 1 if reduce else [1]*6\n    if sum(idx)>0 and len(tboxes)==0: \n        return 0 if reduce else [0]*6\n    \n    pboxes = pboxes[idx]\n    logit = logit[idx]\n    \n    idx = logit.argsort(descending=True)\n    pboxes=box_ops.box_cxcywh_to_xyxy(pboxes.detach()[idx])\n    tboxes=box_ops.box_cxcywh_to_xyxy(tboxes)\n    \n    iou = box_ops.box_iou(pboxes,tboxes)[0].cpu().numpy()\n    prec = [precision(iou,th) for th in [0.5,0.55,0.6,0.65,0.7,0.75]]\n    if reduce:\n        return sum(prec)/6\n    return prec\n    \n\ndef precision(iou,th):\n    #if iou.shape==(0,0): return 1\n\n    #if min(*iou.shape)==0: return 0\n    tp = 0\n    iou = iou.copy()\n    num_pred,num_gt = iou.shape\n    for i in range(num_pred):\n        _iou = iou[i]\n        n_hits = (_iou>th).sum()\n        if n_hits>0:\n            tp += 1\n            j = np.argmax(_iou)\n            iou[:,j] = 0\n    return tp/(num_pred+num_gt-tp)\n    \ndef gen_box(n,scale=1):\n    par = torch.randn((n,4)).mul(scale).sigmoid() \n    max_hw = 2*torch.min(par[:,:2],1-par[:,:2])\n    par[:,2:] = par[:,2:].min(max_hw)\n    return par\n\npboxes = gen_box(50)\nlogit = torch.randn(50)\ntboxes = gen_box(3) \navg_precision(logit,pboxes,tboxes)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:12.932278Z","iopub.execute_input":"2024-04-27T17:26:12.932633Z","iopub.status.idle":"2024-04-27T17:26:12.975810Z","shell.execute_reply.started":"2024-04-27T17:26:12.932599Z","shell.execute_reply":"2024-04-27T17:26:12.975071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function\n\nTraining of DETR is unique and different from FasteRRcnn  and EfficientDET , as we train the criterion as well , the training function can be viewed here : https://github.com/facebookresearch/detr/blob/master/engine.py","metadata":{}},{"cell_type":"code","source":"def train_fn(data_loader,model,criterion,optimizer,device,scheduler,epoch):\n    model.train()\n    criterion.train()\n    \n    tk0 = tqdm(data_loader, total=len(data_loader),leave=False)\n    log = None\n    \n    for step, (images, targets, image_ids) in enumerate(tk0):\n        \n        batch_size = len(images)\n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        \n\n        output = model(images)\n        \n        loss_dict = criterion(output, targets)\n        \n        if log is None:\n            log = {k:AverageMeter() for k in loss_dict}\n            log['total_loss'] = AverageMeter()\n            log['avg_prec'] = AverageMeter()\n            \n        weight_dict = criterion.weight_dict\n        \n        total_loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        \n        optimizer.zero_grad()\n\n        total_loss.backward()\n        \n        if max_norm > 0:\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)\n        \n        optimizer.step()\n        \n        if scheduler is not None:\n            scheduler.step()\n        \n        log['total_loss'].update(total_loss.item(),batch_size)\n        \n        for k,v in loss_dict.items():\n            log[k].update(v.item(),batch_size)\n            \n        log['avg_prec'].update(challenge_metric(output,targets),batch_size)\n            \n        tk0.set_postfix({k:v.avg for k,v in log.items()}) \n        \n    return log","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:13.980467Z","iopub.execute_input":"2024-04-27T17:26:13.980790Z","iopub.status.idle":"2024-04-27T17:26:13.996609Z","shell.execute_reply.started":"2024-04-27T17:26:13.980762Z","shell.execute_reply":"2024-04-27T17:26:13.995500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eval Function","metadata":{}},{"cell_type":"code","source":"def eval_fn(data_loader, model,criterion, device):\n    model.eval()\n    criterion.eval()\n    log = None\n    \n    with torch.no_grad():\n        \n        tk0 = tqdm(data_loader, total=len(data_loader),leave=False)\n        for step, (images, targets, image_ids) in enumerate(tk0):\n            \n            batch_size = len(images)\n            \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            output = model(images)\n        \n            loss_dict = criterion(output, targets)\n            weight_dict = criterion.weight_dict\n        \n            if log is None:\n                log = {k:AverageMeter() for k in loss_dict}\n                log['total_loss'] = AverageMeter()\n                log['avg_prec'] = AverageMeter()\n            \n            for k,v in loss_dict.items():\n                log[k].update(v.item(),batch_size)\n        \n            total_loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n            log['total_loss'].update(total_loss.item(),batch_size)\n            log['avg_prec'].update(challenge_metric(output,targets),batch_size)\n            \n            tk0.set_postfix({k:v.avg for k,v in log.items()}) \n    \n    return log #['total_loss']","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:14.806004Z","iopub.execute_input":"2024-04-27T17:26:14.806363Z","iopub.status.idle":"2024-04-27T17:26:14.820987Z","shell.execute_reply.started":"2024-04-27T17:26:14.806299Z","shell.execute_reply":"2024-04-27T17:26:14.820181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engine","metadata":{}},{"cell_type":"code","source":"import json\n\nclass Logger:\n    def __init__(self,filename,format='csv'):\n        self.filename = filename + '.' + format\n        self._log = []\n        self.format = format\n    def save(self,log,epoch=None):\n        log['epoch'] = epoch+1\n        self._log.append(log)\n        if self.format == 'json':\n            with open(self.filename,'w') as f:\n                json.dump(self._log,f)\n        else:\n            pd.DataFrame(self._log).to_csv(self.filename,index=False)\n            \n            \ndef run(train_data_loader, valid_data_loader, epochs=EPOCHS):\n    \n    logger = Logger(f'log_0')\n    device = torch.device('cuda')\n    model = DETRModel(num_classes=num_classes)\n    model = model.to(device)\n    criterion = SetCriterion(num_classes, \n                             matcher, weight_dict, \n                             eos_coef = null_class_coef, \n                             losses=losses)\n    \n    criterion = criterion.to(device)\n    \n\n    optimizer = torch.optim.AdamW([{\n        'params': v,\n        'lr': lr_dict.get(k,1)*LR\n    } for k,v in model.parameter_groups().items()], weight_decay=1e-4)\n    \n    best_precision = 0\n    header_printed = False\n    for epoch in range(epochs):\n        train_log = train_fn(train_data_loader, model,criterion, optimizer,device,scheduler=None,epoch=epoch)\n        valid_log = eval_fn(valid_data_loader, model,criterion, device)\n    \n        log = {k:v.avg for k,v in train_log.items()}\n        log.update({'V/'+k:v.avg for k,v in valid_log.items()})\n        logger.save(log,epoch)\n        keys = sorted(log.keys())\n        \n        if not header_printed:\n            print(' '.join(map(lambda k: f'{k[:8]:8}',keys)))\n            header_printed = True\n        print(' '.join(map(lambda k: f'{log[k]:8.3f}'[:8],keys)))\n        \n        if log['V/avg_prec'] > best_precision:\n            best_precision = log['V/avg_prec']\n            print('Best model found at epoch {}'.format(epoch+1))\n            torch.save(model.state_dict(), f'detr_best_0.pth')\n        ","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:15.912594Z","iopub.execute_input":"2024-04-27T17:26:15.912920Z","iopub.status.idle":"2024-04-27T17:26:15.933658Z","shell.execute_reply.started":"2024-04-27T17:26:15.912892Z","shell.execute_reply":"2024-04-27T17:26:15.932619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:26:17.045321Z","iopub.execute_input":"2024-04-27T17:26:17.045860Z","iopub.status.idle":"2024-04-27T17:26:17.230128Z","shell.execute_reply.started":"2024-04-27T17:26:17.045800Z","shell.execute_reply":"2024-04-27T17:26:17.229008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run(train_data_loader, valid_data_loader,epochs=50)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T13:25:20.783681Z","iopub.execute_input":"2024-04-27T13:25:20.783954Z","iopub.status.idle":"2024-04-27T15:52:09.831860Z","shell.execute_reply.started":"2024-04-27T13:25:20.783928Z","shell.execute_reply":"2024-04-27T15:52:09.830930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import requests\n\ndef send_msg(text):\n   token = \"6803534305:AAGtC8WOAYV77INSMM9K4XH5fcNwB2YQwTE\"\n   chat_id = \"1252643816\"\n   url_req = \"https://api.telegram.org/bot\" + token + \"/sendMessage\" + \"?chat_id=\" + chat_id + \"&text=\" + text \n   results = requests.get(url_req)\n   print(results.json())\n\nsend_msg(\"Hello there!\")\n\nfrom IPython.display import FileLinks\n\nFileLinks('.')","metadata":{"execution":{"iopub.status.busy":"2024-04-27T17:23:09.013657Z","iopub.execute_input":"2024-04-27T17:23:09.014014Z","iopub.status.idle":"2024-04-27T17:23:09.857284Z","shell.execute_reply.started":"2024-04-27T17:23:09.013979Z","shell.execute_reply":"2024-04-27T17:23:09.856546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample\n\n* I know we might be naive to visualize the model ouput just after one epoch but lets do that and see what are the results like","metadata":{}},{"cell_type":"code","source":"GROUND_TRUTH = PATH + 'bb_front_img'\n\ndef show_prediction(model, data_loader, nrows, ncols, offset=0):\n# model = DETRModel(num_classes=num_classes)\n# model.eval()\n# model.cuda()\n# data_loader = valid_data_loader\n# nrows = 2\n# ncols = 3\n# offset = 5\n\n    fig, axes = plt.subplots(nrows, ncols, figsize=(20, 10))\n\n    sample_data = next(iter(data_loader))\n\n    for i, ax in enumerate(axes.flatten()):\n#         if i % 2 == 0:\n        frame = sample_data[2][i]\n        img = sample_data[0][i]\n        _,w,h = img.shape\n        with torch.no_grad():\n            output = model(img.unsqueeze(0).cuda())\n\n        bb_boxes = output['pred_boxes'][0].detach().cpu().numpy()\n        bb_boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(bb_boxes,h,w)]\n        prob_boxes = output['pred_logits'][0].softmax(1).detach().cpu().numpy()[:, 0]\n\n        img_show = img.permute(1,2,0).cpu().numpy()\n        for box, p in zip(bb_boxes, prob_boxes):\n            x1, y1, x2, y2 = box[0]-box[2]//2, box[1]-box[3]//2, box[0]+box[2]//2, box[1]+box[3]//2\n            if p > 0.5:\n                color = (220, 0, 0) if p > 0.5 else (0, 220, 0)\n                cv2.rectangle(img_show,\n                             (x1 - offset, y1 - offset),\n                             (x2 + offset, y2 + offset),\n                             color, 2)\n        ax.axis('off')\n        ax.imshow(img_show)\n        ax.set_title(f'Frame: {frame}')\n#         else:\n#             img_ground_truth = cv2.imread(f'{GROUND_TRUTH}/{frame}.png', cv2.IMREAD_COLOR)\n#             img_ground_truth = cv2.cvtColor(img_ground_truth, cv2.COLOR_BGR2RGB).astype(np.float32)\n#             ax.axis('off')\n#             ax.imshow(img_ground_truth)\n#             ax.set_title(f'Frame: {frame} (Ground Truth)')\n","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:03:39.068402Z","iopub.execute_input":"2024-04-27T18:03:39.068745Z","iopub.status.idle":"2024-04-27T18:03:39.085799Z","shell.execute_reply.started":"2024-04-27T18:03:39.068716Z","shell.execute_reply":"2024-04-27T18:03:39.084914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DETRModel(num_classes=num_classes)\nmodel.load_state_dict(torch.load(\"./detr_best_0.pth\"))\nmodel.to(torch.device('cuda'))\n\nshow_prediction(model, valid_data_loader, 2, 4, offset=5)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:03:39.788143Z","iopub.execute_input":"2024-04-27T18:03:39.788488Z","iopub.status.idle":"2024-04-27T18:03:42.722171Z","shell.execute_reply.started":"2024-04-27T18:03:39.788459Z","shell.execute_reply":"2024-04-27T18:03:42.721274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_iter = iter(valid_data_loader)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:03:51.941062Z","iopub.execute_input":"2024-04-27T18:03:51.941459Z","iopub.status.idle":"2024-04-27T18:03:52.009233Z","shell.execute_reply.started":"2024-04-27T18:03:51.941419Z","shell.execute_reply":"2024-04-27T18:03:52.007206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images,targets,image_id = next(valid_iter)\ndev_images = [img.to(torch.device('cuda')) for img in images]\nmodel.eval()\nwith torch.no_grad():\n    outputs = model(dev_images)\noutputs = {k: v.cpu() for k, v in outputs.items()}\n# show_predictions(images,outputs,targets)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:04:13.704917Z","iopub.execute_input":"2024-04-27T18:04:13.705259Z","iopub.status.idle":"2024-04-27T18:04:14.954437Z","shell.execute_reply.started":"2024-04-27T18:04:13.705231Z","shell.execute_reply":"2024-04-27T18:04:14.953256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocess(output):\n    logits  = output['pred_logits'][0]\n    idx     = (logits[:,0]-logits[:,1]).gt(0)\n    return {'pred_logits': logits[idx,0]-logits[idx,1],'pred_boxes':output['pred_boxes'][0][idx]}","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:04:18.646898Z","iopub.execute_input":"2024-04-27T18:04:18.647252Z","iopub.status.idle":"2024-04-27T18:04:18.653851Z","shell.execute_reply.started":"2024-04-27T18:04:18.647220Z","shell.execute_reply":"2024-04-27T18:04:18.652608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 2\nlogit = outputs['pred_logits'][0]\nprec = avg_precision(logit[:,0]-logit[:,1],outputs['pred_boxes'][0],targets[0]['boxes'],reduce=False)\nprec,sum(prec)/len(prec),(logit[:,0]-logit[:,1]).gt(0)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:04:19.070479Z","iopub.execute_input":"2024-04-27T18:04:19.070819Z","iopub.status.idle":"2024-04-27T18:04:19.083098Z","shell.execute_reply.started":"2024-04-27T18:04:19.070790Z","shell.execute_reply":"2024-04-27T18:04:19.082204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs['pred_logits'][0].softmax(1).detach().cpu().numpy()[:,0]\n#len(targets[0]['labels']),len(targets[0]['boxes'])","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:04:19.806495Z","iopub.execute_input":"2024-04-27T18:04:19.806840Z","iopub.status.idle":"2024-04-27T18:04:19.817445Z","shell.execute_reply.started":"2024-04-27T18:04:19.806811Z","shell.execute_reply":"2024-04-27T18:04:19.816466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs['pred_logits'][0].argmax(-1)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:04:20.502265Z","iopub.execute_input":"2024-04-27T18:04:20.502648Z","iopub.status.idle":"2024-04-27T18:04:20.511373Z","shell.execute_reply.started":"2024-04-27T18:04:20.502614Z","shell.execute_reply":"2024-04-27T18:04:20.510567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = '../input/carla-sensor-fusion-v1/sensor-fusion-v2/sensor-fusion-v1/'\nTEST_IMAGE_FRAME = ['004608', '004609', '004610', '004611', '004612']\n\ntest_ds = CarlaDataset(TEST_PATH, TEST_IMAGE_FRAME, transform)\n\ntest_data_loader = DataLoader(\n    test_ds,\n    batch_size=4,\n    shuffle=False,\n    num_workers=2,\n    collate_fn=collate_fn\n)\n\ntest_iter = iter(test_data_loader)\n\nshow_prediction(model, test_data_loader, 1, 4, offset=5)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T18:07:32.428021Z","iopub.execute_input":"2024-04-27T18:07:32.428437Z","iopub.status.idle":"2024-04-27T18:07:33.378040Z","shell.execute_reply.started":"2024-04-27T18:07:32.428397Z","shell.execute_reply":"2024-04-27T18:07:33.377046Z"},"trusted":true},"execution_count":null,"outputs":[]}]}