{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports \n","metadata":{}},{"cell_type":"markdown","source":"N2 : Making things work\n\n[best repo for small object detection resources](https://github.com/kuanhungchen/awesome-tiny-object-detection#tiny-object-detection)\n\n[detr base](https://www.kaggle.com/code/tanulsingh077/end-to-end-object-detection-with-transformers-detr/notebook)\n\n[Git Repo to implement any object detection research paper on any data](https://github.com/karkisa/super-enigma)\n\n[Weights and Biases Monitoring](https://wandb.ai/karkisa/Super_Enigma?workspace=user-karkisa) \n\n","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/facebookresearch/detr.git  -q\nimport os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random , pdb\n\n#Torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nimport pytorch_lightning as pl\n\n#sklearn\nfrom sklearn.model_selection import StratifiedKFold\n\n#CV\nimport cv2\n\n################# DETR FUCNTIONS FOR LOSS######################## \nimport sys\nsys.path.append('./detr/')\n\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion\n#################################################################\n\n#Albumenatations\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#Glob\nfrom glob import glob\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom pytorch_lightning.loggers import WandbLogger\nimport wandb\n\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nkey = user_secrets.get_secret(\"kaggle_key\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-20T22:33:00.652112Z","iopub.execute_input":"2022-08-20T22:33:00.652795Z","iopub.status.idle":"2022-08-20T22:33:15.793072Z","shell.execute_reply.started":"2022-08-20T22:33:00.652682Z","shell.execute_reply":"2022-08-20T22:33:15.791762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper","metadata":{}},{"cell_type":"code","source":"def display_(path,n_folds=5,seed=42):\n    marking = pd.read_csv(path)\n    bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n    for i, column in enumerate(['x', 'y', 'w', 'h']):\n        marking[column] = bboxs[:,i]\n    marking.drop(columns=['bbox'], inplace=True)\n    \n    skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)\n\n    df_folds = marking[['image_id']].copy()\n    df_folds.loc[:, 'bbox_count'] = 1\n    df_folds = df_folds.groupby('image_id').count()\n    df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\n    df_folds.loc[:, 'stratify_group'] = np.char.add(\n        df_folds['source'].values.astype(str),\n        df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n    )\n    df_folds.loc[:, 'fold'] = 0\n\n    for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n        df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n    \n    marking['x_max']=marking['x']+marking['w']\n    marking['y_max']=marking['y']+marking['h']\n    \n    return df_folds,marking","metadata":{"execution":{"iopub.status.busy":"2022-08-20T22:33:15.798615Z","iopub.execute_input":"2022-08-20T22:33:15.801147Z","iopub.status.idle":"2022-08-20T22:33:15.815931Z","shell.execute_reply.started":"2022-08-20T22:33:15.801103Z","shell.execute_reply":"2022-08-20T22:33:15.815200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pipeline","metadata":{}},{"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([A.OneOf([A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, val_shift_limit=0.2, p=0.9),\n                               \n                      A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.9)],p=0.9),\n                      \n                      A.ToGray(p=0.01),\n                      \n                      A.HorizontalFlip(p=0.5),\n                      \n                      A.VerticalFlip(p=0.5),\n                      \n                      A.Resize(height=512, width=512, p=1),\n                      \n                      A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n                      \n                      ToTensorV2(p=1.0)],\n                      \n                      p=1.0,\n                     \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )","metadata":{"execution":{"iopub.status.busy":"2022-08-20T22:33:15.822108Z","iopub.execute_input":"2022-08-20T22:33:15.825033Z","iopub.status.idle":"2022-08-20T22:33:15.838108Z","shell.execute_reply.started":"2022-08-20T22:33:15.824990Z","shell.execute_reply":"2022-08-20T22:33:15.836989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WheatDataset(Dataset):\n    def __init__(self,image_ids,dataframe,transforms=None,DIR_TRAIN='../input/global-wheat-detection/train'):\n        self.image_ids = image_ids\n        self.df = dataframe\n        self.transforms = transforms\n        self.DIR_TRAIN=DIR_TRAIN\n        \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n    \n    def get_img(self,image_id):\n        path=f'{self.DIR_TRAIN}/{image_id}.jpg'\n        image = cv2.imread(path, cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        return image\n    \n    def get_boxes(self,records,format_='coco'):\n         # DETR takes in data in coco format \n        boxes = records[['x', 'y', 'w', 'h']].values\n        return boxes\n    \n    def __getitem__(self,index):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n        image = self.get_img(image_id)\n        \n        # DETR takes in data in coco format \n        boxes = self.get_boxes(records)\n        \n        #Area of bb\n        area = boxes[:,2]*boxes[:,3]\n        area = torch.as_tensor(area, dtype=torch.float32)\n        \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n        labels =  np.zeros(len(boxes), dtype=np.int32)\n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            boxes = sample['bboxes']\n            labels = sample['labels']\n            \n        #Normalizing BBOXES\n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        \n        return image, target#, image_id","metadata":{"execution":{"iopub.status.busy":"2022-08-20T22:33:15.844629Z","iopub.execute_input":"2022-08-20T22:33:15.847286Z","iopub.status.idle":"2022-08-20T22:33:15.865168Z","shell.execute_reply.started":"2022-08-20T22:33:15.847145Z","shell.execute_reply":"2022-08-20T22:33:15.864086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class DETRModel(nn.Module):\n    def __init__(self,num_classes,num_queries):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        \n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=True)\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T22:33:15.869618Z","iopub.execute_input":"2022-08-20T22:33:15.871893Z","iopub.status.idle":"2022-08-20T22:33:15.881573Z","shell.execute_reply.started":"2022-08-20T22:33:15.871857Z","shell.execute_reply":"2022-08-20T22:33:15.880450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logic\n","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T22:33:15.886369Z","iopub.execute_input":"2022-08-20T22:33:15.888864Z","iopub.status.idle":"2022-08-20T22:33:15.895107Z","shell.execute_reply.started":"2022-08-20T22:33:15.888825Z","shell.execute_reply":"2022-08-20T22:33:15.894169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class classifier(pl.LightningModule):\n    def __init__(\n        self,\n        ds,\n        bs,\n        df,\n        df_folds,\n        model,\n        c,\n        run_,\n        fold=1,\n        LR=2e-5,\n    ):\n        super().__init__()\n        self.ds=ds\n        self.bs=bs\n        self.df=df\n        self.train_img_ids,self.val_img_ids=df_folds[df_folds['fold'] != fold].index.values, df_folds[df_folds['fold'] == fold].index.values\n        self.model=model\n        self.LR=LR\n        self.criterion = c\n        self.run_=run_\n        \n    def train_dataloader(self):\n        train_ds=self.ds(self.train_img_ids,self.df,transforms=get_train_transforms())\n        train_loader=DataLoader(train_ds,batch_size=self.bs,shuffle=False,\n                                num_workers=4,\n                                collate_fn=collate_fn)\n        return train_loader\n      \n    def val_dataloader(self):\n        val_ds=self.ds(self.val_img_ids,self.df,transforms=get_valid_transforms())\n        val_loader=DataLoader(val_ds,batch_size=self.bs,shuffle=False,\n                            num_workers=4,\n                            collate_fn=collate_fn)\n        return val_loader\n    \n    def log_boxes_valid(self,images,outputs):\n        _,h,w=images[0].shape\n        oboxes = outputs['pred_boxes'].detach().cpu().numpy()\n        oboxes = [np.array(box) for box in A.augmentations.bbox_utils.denormalize_bboxes(oboxes,h,w)]\n        wandb_imgs=[]\n        for img,boxes in zip(images,oboxes):\n            wandb_imgs.append(self.wandb_bbox(img,boxes))\n        self.run_.log({\"preds\": wandb_imgs})\n \n    def wandb_bbox(self,image, bboxes):\n        all_boxes = []\n        for bbox in bboxes:\n            box_data = {\"position\": {\n                            \"minX\": bbox[0].astype('float'),\n                            \"minY\": bbox[1].astype('float'),\n                            \"maxX\": bbox[0].astype('float')+bbox[2].astype('float'),\n                            \"maxY\": bbox[1].astype('float')+bbox[3].astype('float')\n                        },\n                         \"class_id\" : int(0),\n                         \"box_caption\": \"Wheat\",\n                         \"domain\" : \"pixel\"}\n            all_boxes.append(box_data)\n\n        return wandb.Image(image, boxes={\n            \"prediction\": {\n                \"box_data\": all_boxes,\n              \"class_labels\": {0:\"Wheat\"}\n            }\n        })\n            \n    def training_step(self,batch,batch_idx):\n        images,targets=batch\n        images = list(images)\n        self.criterion.train()\n        outputs=self.model(images)\n        loss_dict = self.criterion(outputs, targets)\n        weight_dict = self.criterion.weight_dict\n        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        self.run_.log({\"train\": {\"loss\":losses}})\n        return losses\n    \n    def validation_step(self,batch,batch_idx):\n        images,targets=batch\n        images=list(images)\n        self.criterion.eval()\n        outputs=self.model(images)\n        self.log_boxes_valid(images,outputs)\n        loss_dict = self.criterion(outputs, targets)\n        weight_dict = self.criterion.weight_dict\n        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        self.run_.log({\"val\": {\"loss\":losses}})\n        return losses\n    \n    def configure_optimizers(self):\n        return torch.optim.Adam(self.model.parameters(),lr=self.LR)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T22:33:15.899805Z","iopub.execute_input":"2022-08-20T22:33:15.902315Z","iopub.status.idle":"2022-08-20T22:33:15.929536Z","shell.execute_reply.started":"2022-08-20T22:33:15.902277Z","shell.execute_reply":"2022-08-20T22:33:15.928495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lets play","metadata":{}},{"cell_type":"code","source":"def main():\n    train_df_path='../input/global-wheat-detection/train.csv'\n    fold_df,markings=display_(train_df_path)\n    bs=32\n    \n    model=DETRModel(num_classes=2,num_queries=100)\n    c=SetCriterion(1, matcher=HungarianMatcher(), weight_dict={'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1}, eos_coef = 0.5, losses=['labels', 'boxes', 'cardinality']).to('cuda')\n    wandb.login(key=key)\n    run_ = wandb.init(\n                        project='e3e3',\n                        group='ee',\n                        name='exp1_3'\n                    )\n    \n    Classifier=classifier(\n        WheatDataset,\n        bs,\n        markings,\n        fold_df,\n        model,\n        c,\n        run_\n    )\n    \n    Trainer=pl.Trainer(accelerator='gpu',\n                       max_epochs=2,\n                      )\n    Trainer.fit(Classifier)\n    \nmain()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T22:33:15.934546Z","iopub.execute_input":"2022-08-20T22:33:15.937039Z"},"trusted":true},"execution_count":null,"outputs":[]}]}