{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports \n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch, torchvision, os, math, re, wandb,pdb\nfrom ast import literal_eval as lv\nimport pytorch_lightning as pl\nimport matplotlib.pyplot as plt\nfrom torchvision.utils import draw_bounding_boxes\nfrom torchvision.transforms.functional import to_pil_image\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\n!git clone https://github.com/facebookresearch/detr.git -q\nimport sys\nsys.path.append('./detr/')\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-22T02:20:23.608074Z","iopub.execute_input":"2022-05-22T02:20:23.608449Z","iopub.status.idle":"2022-05-22T02:20:39.470536Z","shell.execute_reply.started":"2022-05-22T02:20:23.608343Z","shell.execute_reply":"2022-05-22T02:20:39.469195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* [Initial df generation](https://www.kaggle.com/code/pestipeti/pytorch-starter-fasterrcnn-train)\n\n* [best repo for small object detection resources](https://github.com/kuanhungchen/awesome-tiny-object-detection#tiny-object-detection)\n\n* [detr base](https://www.kaggle.com/code/tanulsingh077/end-to-end-object-detection-with-transformers-detr/notebook)","metadata":{}},{"cell_type":"markdown","source":"# Helpers","metadata":{}},{"cell_type":"code","source":"def read_img(path):\n    img=torchvision.io.read_image(path)\n    return img\n\ndef display_(path_df,base_dir,disp=False):\n    df=pd.read_csv(path_df)\n    df['path']=base_dir+'/'+df.image_id+'.jpg'\n    df['x_min']=df.bbox.apply(lambda x: lv(x)[0])\n    df['y_min']=df.bbox.apply(lambda x: lv(x)[1])\n    df['x_max']=df.bbox.apply(lambda x: lv(x)[0]+lv(x)[2])\n    df['y_max']=df.bbox.apply(lambda x: lv(x)[1]+lv(x)[3])\n    df['area']=df.bbox.apply(lambda x: lv(x)[2]*lv(x)[3])\n    \n    if disp:\n\n        display(df)\n        print(df.info())\n        for col in df.columns:\n            print(f'unique values in {col} = {df[col].nunique()}')\n\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:20:39.472716Z","iopub.execute_input":"2022-05-22T02:20:39.473474Z","iopub.status.idle":"2022-05-22T02:20:39.484051Z","shell.execute_reply.started":"2022-05-22T02:20:39.473436Z","shell.execute_reply":"2022-05-22T02:20:39.483135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\ndef get_train_transform(format_='coco'):\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format=format_,min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:20:39.485769Z","iopub.execute_input":"2022-05-22T02:20:39.486185Z","iopub.status.idle":"2022-05-22T02:20:41.221468Z","shell.execute_reply.started":"2022-05-22T02:20:39.486141Z","shell.execute_reply":"2022-05-22T02:20:41.220532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Base pipeline","metadata":{}},{"cell_type":"code","source":"class basic_pipeline(Dataset):\n    \n    def __init__(self ,\n                 df ,\n                 apply_transform=True,\n                 apply_normalise=True,\n                 format_='coco',\n                 transforms=None\n                ):\n        \n        super().__init__()\n        self.df=df\n        self.image_id=df['image_id'].unique()\n        self.apply_transform=apply_transform\n        self.apply_normalise=apply_normalise\n        self.format_=format_\n        self.transforms=transforms\n        \n    def __len__(self):\n        return len(self.image_id)\n    \n    def read_img(self,path):\n        img=torchvision.io.read_image(path)\n        return img\n    \n    def transform_(self,image,bbox,labels):\n        sample = {\n                'image': image.numpy(),\n                'bboxes': bbox,\n                'labels': labels\n                 }\n        pdb.set_trace()\n        sample = self.transforms(**sample)\n        return  sample\n    \n    def normalise_(self,sample,image):\n        _,h,w = image.shape\n        bbox = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        return bbox\n        \n    def get_image_bboxes(self,img_unique_df,format_='coco'):\n        bbox=[]\n        for row in img_unique_df.iterrows():\n            row=row[1]\n            if format_=='coco':\n                bbox.append([row.x_min,row.y_min,row.x_max-row.x_min,row.y_max-row.y_min])\n            else:\n                bbox.append([row.x_min,row.y_min,row.x_max,row.y_max])\n        image=img=self.read_img(row.path)\n        \n        return image,bbox\n    \n    def __getitem__(self,idx):\n        img_id=self.image_id[idx]\n        img_unique_df=self.df[self.df.image_id==img_id]\n        target={}\n        area=img_unique_df.area\n        image,bbox=self.get_image_bboxes(img_unique_df,self.format_)\n        labels = np.zeros(len(bbox), dtype=np.int32)\n        \n        if self.apply_transform:\n            sample=self.transform_(image,bbox,labels)\n            image,bbox,labels=sample['image'],sample['bboxes'],sample['labels']\n            \n        if self.apply_normalise:\n            bbox=self.normalise_(sample,image)\n            \n        target['bbox']=torch.tensor(bbox)\n        target['area']=area\n        target['label']=torch.tensor(labels)\n        return image , target\n\ndef display_ds(ds,r=4,c=4,size=40):\n    _,axs=plt.subplots(r,c,figsize=(size,size))\n    axs=axs.flatten()\n\n    for n,ax in enumerate(axs):\n        img,target=ds.__getitem__(n)\n        boxes=target['bbox']\n        img=draw_bounding_boxes(img,boxes)\n        ax.imshow(to_pil_image(img))\n        ax.axis('off')\n        \n    plt.tight_layout() \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:20:41.224193Z","iopub.execute_input":"2022-05-22T02:20:41.224555Z","iopub.status.idle":"2022-05-22T02:20:41.251371Z","shell.execute_reply.started":"2022-05-22T02:20:41.224505Z","shell.execute_reply":"2022-05-22T02:20:41.250288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_path='../input/global-wheat-detection/train.csv'\nsubmission_df_path='../input/global-wheat-detection/sample_submission.csv'\ntrain_base_dir='../input/global-wheat-detection/train'\nsubmission__base_dir='../input/global-wheat-detection/test'\ntrain_df=display_(train_df_path,train_base_dir)\nds=basic_pipeline(train_df,transforms=get_valid_transforms())\ni,t=ds.__getitem__(2)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:35:20.360214Z","iopub.execute_input":"2022-05-22T02:35:20.360550Z","iopub.status.idle":"2022-05-22T02:37:42.029415Z","shell.execute_reply.started":"2022-05-22T02:35:20.360513Z","shell.execute_reply":"2022-05-22T02:37:42.027972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i,t=ds.__getitem__(2)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:26:49.232545Z","iopub.execute_input":"2022-05-22T02:26:49.232771Z","iopub.status.idle":"2022-05-22T02:28:31.555051Z","shell.execute_reply.started":"2022-05-22T02:26:49.232740Z","shell.execute_reply":"2022-05-22T02:28:31.553613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class DETRModel(pl.LightningModule):\n    def __init__(self,num_classes,num_queries):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=True)\n        self.in_features = self.model.class_embed.in_features\n        self.model.class_embed = torch.nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:20:41.253079Z","iopub.execute_input":"2022-05-22T02:20:41.253472Z","iopub.status.idle":"2022-05-22T02:20:41.274258Z","shell.execute_reply.started":"2022-05-22T02:20:41.253416Z","shell.execute_reply":"2022-05-22T02:20:41.273322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clssifier Logics\n","metadata":{}},{"cell_type":"code","source":"class classifier(pl.LightningModule):\n    def __init__(\n        self,\n        ds,\n        df,\n        model,\n        LR=2e-5,\n        null_class_coef = 0.5,\n        num_classes = 2,\n        num_queries = 100,\n        matcher = HungarianMatcher(),\n        weight_dict  = {'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1},\n        losses = ['labels', 'boxes', 'cardinality'],\n        *args\n    ):\n        super().__init__()\n        self.ds=ds\n        self.df=df\n        self.train_df,self.val_df=train_test_split(self.df)\n        self.model=model\n        self.LR=LR\n        self.losses=losses\n        self.matcher=matcher\n        self.weight_dict=weight_dict\n        self.num_classes=num_classes\n        self.num_queries=num_queries\n        self.null_class_coef=null_class_coef\n        self.criterion=SetCriterion(self.num_classes-1, self.matcher, self.weight_dict, eos_coef = self.null_class_coef, losses=self.losses),\n        \n    def train_dataloader(self):\n        train_ds=self.ds(self.train_df,transforms=get_train_transform())\n        train_loader=DataLoader(train_ds,batch_size=32)\n        return train_loader\n      \n    def val_dataloader(self):\n        val_ds=self.ds(self.val_df,transforms=get_valid_transforms())\n        val_loader=DataLoader(val_ds,batch_size=32)\n        return val_loader\n    \n    def training_step(self,batch,batch_idx):\n        images,targets=batch\n        outputs=self.model(images)\n        loss_dict = self.criterion(outputs, targets)\n        weight_dict = self.criterion.weight_dict\n        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        return losses\n    \n    def validation_step(self,batch,batch_idx):\n        images,targets=batch\n        outputs=self.model(images)\n        loss_dict = self.criterion(outputs, targets)\n        weight_dict = self.criterion.weight_dict\n        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        return losses\n    \n    def configure_optimizers(self):\n        return torch.optim.Adam(self.model.parameters(),lr=self.LR)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:20:41.275340Z","iopub.execute_input":"2022-05-22T02:20:41.275560Z","iopub.status.idle":"2022-05-22T02:20:41.293946Z","shell.execute_reply.started":"2022-05-22T02:20:41.275528Z","shell.execute_reply":"2022-05-22T02:20:41.292687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Play","metadata":{}},{"cell_type":"code","source":"def main():\n    train_df_path='../input/global-wheat-detection/train.csv'\n    submission_df_path='../input/global-wheat-detection/sample_submission.csv'\n    train_base_dir='../input/global-wheat-detection/train'\n    submission__base_dir='../input/global-wheat-detection/test'\n    train_df=display_(train_df_path,train_base_dir)\n    ds=basic_pipeline(train_df)\n    \n    Classifier=classifier(\n        basic_pipeline,\n        train_df,\n        DETRModel(num_classes=2,num_queries=100),\n    )\n    \n    Trainer=pl.Trainer(accelerator='cpu')\n    Trainer.fit(Classifier)\n    \nmain()","metadata":{"execution":{"iopub.status.busy":"2022-05-22T02:20:41.295977Z","iopub.execute_input":"2022-05-22T02:20:41.296392Z","iopub.status.idle":"2022-05-22T02:22:28.648779Z","shell.execute_reply.started":"2022-05-22T02:20:41.296341Z","shell.execute_reply":"2022-05-22T02:22:28.646753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}