{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!git clone https://github.com/facebookresearch/detr.git   #cloning github repo of detr to import its unique loss","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm.autonotebook import tqdm\n\n\n#Torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\n#sklearn\nfrom sklearn.model_selection import StratifiedKFold\n\n#CV\nimport cv2\n\n################# DETR FUCNTIONS FOR LOSS######################## \nimport sys\nsys.path.append('./detr/')\n\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion\n#################################################################\n\n#Albumenatations\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#Glob\nfrom glob import glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils\n\n* AverageMeter - class for averaging loss,metric,etc over epochs","metadata":{}},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration\n\nBasic configuration for this model","metadata":{}},{"cell_type":"code","source":"n_folds = 5\nseed = 42\nnum_classes = 2\nnum_queries = 100\nnull_class_coef = 0.5\nBATCH_SIZE = 8\nLR = 2e-5\nEPOCHS = 12","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed Everything\n\nSeeding everything for reproducible results","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing the Data\n\n* For preparation of data I use code from Alex's awesome kernel [here](https://www.kaggle.com/shonenkov/training-efficientdet)\n* The data can be split into any number of folds as you want , split is stratified based on number of boxes and source","metadata":{}},{"cell_type":"code","source":"import ast\nmarking = pd.read_csv('../input/dates-dataset/training_crop_useful_images.csv')\nmarking['image_id'] = marking['image_id'].apply(lambda x : x.split(\".\")[0])\nmarking['source'] = ['self']*len(marking)\nmarking = marking.loc[marking[\"image_id\"]!=\"288\",:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ast.literal_eval(marking['bbox'][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"marking['bbox'] = marking['bbox'].apply(lambda x : ast.literal_eval(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"marking['bbox'] = marking['bbox'].apply(lambda x : [x[0],x[1],x[2]-x[0],x[3]-x[1]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"marking","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bboxs = np.stack(marking['bbox'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:,i]\nmarking.drop(columns=['bbox'], inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating Folds\nskf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)\n\ndf_folds = marking[['image_id']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count()\ndf_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\ndf_folds.loc[:, 'fold'] = 0\n\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations\n","metadata":{}},{"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([A.OneOf([A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, val_shift_limit=0.2, p=0.9),\n                               \n                      A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.9)],p=0.9),\n                      \n                      A.ToGray(p=0.01),\n                      \n                      A.HorizontalFlip(p=0.5),\n                      \n                      A.VerticalFlip(p=0.5),\n                      \n                      A.Resize(height=512, width=512, p=1),\n                      \n                      A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n                      \n                      ToTensorV2(p=1.0)],\n                      \n                      p=1.0,\n                     \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataset\n\n*DETR accepts data in coco format which is (x,y,w,h)(for those who do not know there are two formats coco and pascal(smin,ymin,xmax,ymax) which are widely used) . So now we need to prepare data in that format","metadata":{}},{"cell_type":"code","source":"DIR_TRAIN = '../input/dates-dataset/crop_useful_images/crop_useful_images'\n\nclass DatesDataset(Dataset):\n    def __init__(self,image_ids,dataframe,transforms=None):\n        self.image_ids = image_ids\n        self.df = dataframe\n        self.transforms = transforms\n        \n        \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n    \n    def __getitem__(self,index):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n        \n        image = cv2.imread(f'{DIR_TRAIN}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        # DETR takes in data in coco format \n        boxes = records[['x', 'y', 'w', 'h']].values\n        \n        #Area of bb\n        area = boxes[:,2]*boxes[:,3]\n        area = torch.as_tensor(area, dtype=torch.float32)\n        \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n        labels =  np.zeros(len(boxes), dtype=np.int32)\n\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            boxes = sample['bboxes']\n            labels = sample['labels']\n            \n            \n        #Normalizing BBOXES\n            \n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        \n        return image, target, image_id","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\n* Initial DETR model is trained on coco dataset , which has 91 classes + 1 background class , hence we need to modify it to take our own number of classes\n* Also DETR model takes in 100 queries ie ,it outputs total of 100 bboxes for every image , we can very well change that too","metadata":{}},{"cell_type":"code","source":"class DETRModel(nn.Module):\n    def __init__(self,num_classes,num_queries):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        \n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=True)\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matcher and Bipartite Matching Loss\n\nNow we make use of the unique loss that the model uses and for that we need to define the matcher. DETR calcuates three individual losses :\n* Classification Loss for labels(its weight can be set by loss_ce)\n* Bbox Loss (its weight can be set by loss_bbox)\n* Loss for Background class","metadata":{}},{"cell_type":"code","source":"'''\ncode taken from github repo detr , 'code present in engine.py'\n'''\n\nmatcher = HungarianMatcher()\n\nweight_dict = weight_dict = {'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1}\n\nlosses = ['labels', 'boxes', 'cardinality']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function\n\nTraining of DETR is unique and different from FasteRRcnn  and EfficientDET , as we train the criterion as well , the training function can be viewed here : https://github.com/facebookresearch/detr/blob/master/engine.py","metadata":{}},{"cell_type":"code","source":"def train_fn(data_loader,model,criterion,optimizer,device,scheduler,epoch):\n    model.train()\n    criterion.train()\n    \n    summary_loss = AverageMeter()\n    \n    tk0 = tqdm(data_loader, total=len(data_loader))\n    \n    for step, (images, targets, image_ids) in enumerate(tk0):\n        \n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        \n\n        output = model(images)\n        \n        loss_dict = criterion(output, targets)\n        weight_dict = criterion.weight_dict\n        \n        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        \n        optimizer.zero_grad()\n\n        losses.backward()\n        optimizer.step()\n        if scheduler is not None:\n            scheduler.step()\n        \n        summary_loss.update(losses.item(),BATCH_SIZE)\n        tk0.set_postfix(loss=summary_loss.avg)\n        \n    return summary_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eval Function","metadata":{}},{"cell_type":"code","source":"def eval_fn(data_loader, model,criterion, device):\n    model.eval()\n    criterion.eval()\n    summary_loss = AverageMeter()\n    \n    with torch.no_grad():\n        \n        tk0 = tqdm(data_loader, total=len(data_loader))\n        for step, (images, targets, image_ids) in enumerate(tk0):\n            \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            output = model(images)\n        \n            loss_dict = criterion(output, targets)\n            weight_dict = criterion.weight_dict\n        \n            losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n            \n            summary_loss.update(losses.item(),BATCH_SIZE)\n            tk0.set_postfix(loss=summary_loss.avg)\n    \n    return summary_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engine","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(fold):\n    \n    df_train = df_folds[df_folds['fold'] != fold]\n    df_valid = df_folds[df_folds['fold'] == fold]\n    \n    train_dataset = DatesDataset(\n    image_ids=df_train.index.values,\n    dataframe=marking,\n    transforms=get_train_transforms()\n    )\n\n    valid_dataset = DatesDataset(\n    image_ids=df_valid.index.values,\n    dataframe=marking,\n    transforms=get_valid_transforms()\n    )\n    \n    train_data_loader = DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n    )\n\n    valid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n    )\n    \n    device = torch.device('cuda')\n    model = DETRModel(num_classes=num_classes,num_queries=num_queries)\n    model = model.to(device)\n    criterion = SetCriterion(num_classes-1, matcher, weight_dict, eos_coef = null_class_coef, losses=losses)\n    criterion = criterion.to(device)\n    \n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n    \n    best_loss = 10**5\n    for epoch in range(EPOCHS):\n        train_loss = train_fn(train_data_loader, model,criterion, optimizer,device,scheduler=None,epoch=epoch)\n        valid_loss = eval_fn(valid_data_loader, model,criterion, device)\n        \n        print('|EPOCH {}| TRAIN_LOSS {}| VALID_LOSS {}|'.format(epoch+1,train_loss.avg,valid_loss.avg))\n        \n        if valid_loss.avg < best_loss:\n            best_loss = valid_loss.avg\n            print('Best model found for Fold {} in Epoch {}........Saving Model'.format(fold,epoch+1))\n            torch.save(model.state_dict(), f'detr_best_{fold}.pth')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run(fold=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample\n","metadata":{}},{"cell_type":"code","source":"def view_sample(df_valid,model,device):\n    '''\n    Code taken from Peter's Kernel \n    https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-train\n    '''\n    valid_dataset = DatesDataset(image_ids=df_valid.index.values,\n                                 dataframe=marking,\n                                 transforms=get_valid_transforms()\n                                )\n     \n    valid_data_loader = DataLoader(\n                                    valid_dataset,\n                                    batch_size=BATCH_SIZE,\n                                    shuffle=False,\n                                   num_workers=4,\n                                   collate_fn=collate_fn)\n    \n    images, targets, image_ids = next(iter(valid_data_loader))\n    _,h,w = images[0].shape # for de normalizing images\n    \n    images = list(img.to(device) for img in images)\n    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n    \n    boxes = targets[0]['boxes'].cpu().numpy()\n    boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(boxes,h,w)]\n    sample = images[0].permute(1,2,0).cpu().numpy()\n    \n    model.eval()\n    model.to(device)\n    cpu_device = torch.device(\"cpu\")\n    \n    with torch.no_grad():\n        outputs = model(images)\n        \n    outputs = [{k: v.to(cpu_device) for k, v in outputs.items()}]\n    \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n#     for box in boxes:\n#         cv2.rectangle(sample,\n#                   (box[0], box[1]),\n#                   (box[2]+box[0], box[3]+box[1]),\n#                   (220, 0, 0), 1)\n        \n\n    oboxes = outputs[0]['pred_boxes'][0].detach().cpu().numpy()\n    oboxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(oboxes,h,w)]\n    prob   = outputs[0]['pred_logits'][0].softmax(1).detach().cpu().numpy()[:,0]\n    plist=list()\n    for box,p in zip(oboxes,prob):\n        plist.append(p)\n        if p >0.09:\n            color = (220,0,0) #if p>0.5 else (0,0,0)\n            cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2]+box[0], box[3]+box[1]),\n                  color, 1)\n    print(max(plist))\n    ax.set_axis_off()\n    ax.imshow(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DETRModel(num_classes=num_classes,num_queries=num_queries)\nmodel.load_state_dict(torch.load(\"./detr_best_0.pth\"))\nview_sample(df_folds[df_folds['fold'] == 0],model=model,device=torch.device('cuda'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}