{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this kernal we will try to improve the perfomance of model that we trained using fastrcnn. The training data is small, therefore more data usually help.So we will psudo label test data. Then we retrain the model by combining psudo label data with our train data.\n\n* You can refer my [baseline training notebook](http://www.kaggle.com/arunmohan003/fasterrcnn-using-pytorch-baseline/)\n* We will submit our predictions in [inference notebook](https://www.kaggle.com/arunmohan003/inferance-kernel-fasterrcnn).\n\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch\nimport os\nimport time\nimport shutil\nimport torch.nn as nn\nfrom skimage import io\nimport torchvision\nimport cv2\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nfrom albumentations.pytorch import ToTensor\nfrom torchvision import utils\nfrom albumentations import (HorizontalFlip, Flip,ShiftScaleRotate, VerticalFlip, Normalize,\n                            Compose, GaussNoise)\n\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csv_path = '/kaggle/input/global-wheat-detection/train.csv'\ntest_dir = '/kaggle/input/global-wheat-detection/test'\ntrain_dir = '/kaggle/input/global-wheat-detection/train'\n\n# Load the trained weights\nweights = '/kaggle/input/fasterrcnn-using-pytorch-baseline/bestmodel.pt'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Psudo Labelling "},{"metadata":{},"cell_type":"markdown","source":"psudo labelling reference [notebook](https://www.kaggle.com/gc1023/fork-of-fasterrcnn-pseudo-labeling)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(phase):\n            list_transforms = []\n                \n            list_transforms.extend(\n                    [\n            ToTensor(),\n                    ])\n            list_trfms = Compose(list_transforms)\n            return list_trfms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Wheatset(Dataset):\n    def __init__(self,image_dir,phase):\n        super().__init__()\n   \n        self.image_dir = image_dir\n        self.images = os.listdir(image_dir)\n        self.transforms = get_transforms(phase)\n        \n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self,idx):\n        image = self.images[idx]\n        image_arr = io.imread(os.path.join(self.image_dir,image))\n        image_id = str(image.split('.')[0])\n        \n        if self.transforms:\n            sample = {\n                'image': image_arr,\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n               \n        return image, image_id\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\n\ntest_dataset = Wheatset(test_dir,phase='validation')\n\ntest_data_loader = DataLoader(\n    test_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load a model; pre-trained on COCO\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=False)\n    \nnum_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n\ncheckpoint = torch.load(weights,map_location=device)\nmodel.load_state_dict(checkpoint['state_dict'])\nmodel.eval()\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detection_threshold = 0.5\nresults = []\nfor images, image_ids in test_data_loader:\n\n    images = list(image.to(device) for image in images)\n    outputs = model(images)\n\n    \n    for i, image in enumerate(images):\n\n        boxes = outputs[i]['boxes'].data.cpu().numpy()\n        scores = outputs[i]['scores'].data.cpu().numpy()\n        boxes = boxes[scores >= detection_threshold].astype(np.int32)\n        scores = scores[scores >= detection_threshold]\n        image_id = image_ids[i]\n               \n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n        \n        for box in boxes:\n            result = {\n                'image_id': 'psudo_' + image_id,\n                'width': 1024,\n                'height': 1024,\n                'source': 'psudo',\n                'bbox': [box[0],box[1],box[2],box[3]]}\n        \n            results.append(result)\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\npseudo = pd.DataFrame(results, columns=['image_id', 'width', 'height','source','bbox'])\nprint(f'shape before preprocessing: {pseudo.shape}')\npseudo.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_bbox(df,phase):\n    ids = []\n    values = []\n    imd = np.unique(df['image_id'])\n#     imd = [i.split('_')[1] for i in imd]\n    if phase=='train':\n        df['bbox'] = df['bbox'].apply(lambda x: eval(x))\n    for image_id in os.listdir(train_dir):\n        image_id = image_id.split('.')[0]\n        if image_id not in imd :\n            ids.append(image_id)\n            values.append(str([-1,-1,-1,-1]))\n    new_df = {'image_id':ids, 'bbox':values}\n    new_df = pd.DataFrame(new_df)\n    df = df[['image_id','bbox']]\n    df.append(new_df)\n    df = df.sample(frac=1).reset_index(drop=True)\n    df['x'] = df['bbox'].apply(lambda x: x[0])\n    df['y'] = df['bbox'].apply(lambda x: x[1])\n    df['w'] = df['bbox'].apply(lambda x: x[2])\n    df['h'] = df['bbox'].apply(lambda x: x[3])\n\n    df.drop(columns=['bbox'],inplace=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Processing psudo labels we created"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = process_bbox(pseudo,phase='validation')\nprint(f'shape of test frame is {df_test.shape}')\ndf_test.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Retraining model using psudo labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(csv_path)\ndf_train = process_bbox(df_train,phase='train')\nprint(f'shape of train frame is {df_train.shape}')\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# concatinating both frames\nframes = [df_train, df_test]\nfinal_df = pd.concat(frames).reset_index(drop=True)\nprint(f'shape of final train frame is {final_df.shape}')\nfinal_df.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# we will shuffle dataframe\nfinal_df = final_df.sample(frac=1).reset_index(drop=True)\nfinal_df.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_ids = final_df['image_id'].unique()\ntrain_ids = image_ids[0:int(0.8*len(image_ids))]\nval_ids = image_ids[int(0.8*len(image_ids)):]\nprint(f'Total images {len(image_ids)}')\nprint(f'No of train images {len(train_ids)}')\nprint(f'No of validation images {len(val_ids)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_df = final_df[final_df['image_id'].isin(train_ids)]\nval_df = final_df[final_df['image_id'].isin(val_ids)]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(phase):\n            list_transforms = []\n            if phase == 'train':\n                list_transforms.extend([\n                       Flip(p=0.5)\n                         ])\n            list_transforms.extend(\n                    [\n            ToTensor(),\n                    ])\n            list_trfms = Compose(list_transforms,\n                                 bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n            return list_trfms\n\n\n\n\n\n\n\nclass Wheatset(Dataset):\n    def __init__(self,data_frame,train_dir,test_dir,phase='train'):\n        super().__init__()\n        self.df = data_frame\n        self.train_dir = train_dir\n        self.test_dir = test_dir\n        self.images = data_frame['image_id'].unique()\n        self.transforms = get_transforms(phase)\n        \n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self,idx):\n        image_id = self.images[idx]\n        \n        if 'psudo' in image_id:\n            image = image_id.split('_')[1] + '.jpg'\n            image_arr = cv2.imread(os.path.join(self.test_dir,image), cv2.IMREAD_COLOR)\n            image_arr = cv2.cvtColor(image_arr, cv2.COLOR_BGR2RGB).astype(np.float32)\n        else:\n            image = image_id + '.jpg'\n            image_arr = cv2.imread(os.path.join(self.train_dir,image), cv2.IMREAD_COLOR)\n            image_arr = cv2.cvtColor(image_arr, cv2.COLOR_BGR2RGB).astype(np.float32)\n        \n        image_arr /= 255.0\n#         image_id = str(image.split('.')[0])\n        point = self.df[self.df['image_id'] == image_id]\n        boxes = point[['x', 'y', 'w', 'h']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        \n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        area = torch.as_tensor(area, dtype=torch.float32)\n        \n        # there is only one class\n        labels = torch.ones((point.shape[0],), dtype=torch.int64)\n        \n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((point.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor(idx)\n        target['area'] = area\n        target['iscrowd'] = iscrowd\n        \n        if self.transforms:\n            sample = {\n                'image': image_arr,\n                'bboxes': target['boxes'],\n                'labels': target['labels']\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            \n        target['boxes'] = torch.stack(tuple(map(torch.tensor, \n                                                zip(*sample['bboxes'])))).permute(1, 0)\n        \n        return image, target, image_id\n            \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = Wheatset(train_df,train_dir,test_dir,phase='train')\nval_data = Wheatset(val_df,train_dir,test_dir,phase='validation')\n\nprint(f'Length of train data {len(train_data)}')\nprint(f'Length of validation data {len(val_data)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# batching\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ntrain_data_loader = DataLoader(\n    train_data,\n    batch_size=8,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)\n\n\nvalid_data_loader = DataLoader(\n    val_data,\n    batch_size=8,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_convert(image):\n    image = image.clone().cpu().numpy()\n    image = image.transpose((1,2,0))\n\n    image = (image * 255).astype(np.uint8)\n    return image\n\ndef plot_img(data,idx):\n    out = data.__getitem__(idx)\n    image = image_convert(out[0])\n    image = np.ascontiguousarray(image)\n    bb = out[1]['boxes'].numpy()\n    for i in bb:\n        cv2.rectangle(image, (i[0],i[1]), (i[2],i[3]), (0,255,0), thickness=2)\n    plt.figure(figsize=(10,10))\n    plt.imshow(image)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_img(train_data,129)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n\nimages, targets, ids = next(iter(train_data_loader))\nimages = list(image.to(device) for image in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\n# optimizer = torch.optim.Adam(params, lr=0.001)\noptimizer = torch.optim.SGD(params, lr=0.01, momentum=0.9, weight_decay=0.00001)\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_ckp(state, is_best, checkpoint_path, best_model_path):\n    \"\"\"\n    state: checkpoint we want to save\n    is_best: is this the best checkpoint; min validation loss\n    checkpoint_path: path to save checkpoint\n    best_model_path: path to save best model\n    \"\"\"\n    # save checkpoint data to the path given, checkpoint_path\n    torch.save(state, checkpoint_path)\n    # if it is a best model, min validation loss\n    if is_best:\n        # copy that checkpoint file to best path given, best_model_path\n        shutil.copyfile(checkpoint_path, best_model_path)\n        \ndef load_ckp(checkpoint_fpath, model, optimizer):\n    \"\"\"\n    checkpoint_path: path to save checkpoint\n    model: model that we want to load checkpoint parameters into       \n    optimizer: optimizer we defined in previous training\n    \"\"\"\n    # load check point\n    checkpoint = torch.load(checkpoint_fpath)\n    # initialize state_dict from checkpoint to model\n    model.load_state_dict(checkpoint['state_dict'])\n    # initialize optimizer from checkpoint to optimizer\n    optimizer.load_state_dict(checkpoint['optimizer'])\n    # initialize valid_loss_min from checkpoint to valid_loss_min\n    valid_loss_min = checkpoint['valid_loss_min']\n    # return model, optimizer, epoch value, min validation loss \n    return model, optimizer, checkpoint['epoch'], valid_loss_min.item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 3\ntrain_loss_min = 0.9\ntotal_train_loss = []\n\n\ncheckpoint_path = '/kaggle/working/chkpoint_'\nbest_model_path = '/kaggle/working/psudomodel.pt'\n\nfor epoch in range(num_epochs):\n    print(f'Epoch :{epoch + 1}')\n    start_time = time.time()\n    train_loss = []\n    model.train()\n    for images, targets, image_ids in train_data_loader:\n        \n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n        loss_dict = model(images, targets)\n\n        losses = sum(loss for loss in loss_dict.values())\n        train_loss.append(losses.item())        \n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n        \n#     if lr_scheduler is not None:\n#         lr_scheduler.step()\n    #train_loss/len(train_data_loader.dataset)\n    epoch_train_loss = np.mean(train_loss)\n    total_train_loss.append(epoch_train_loss)\n    print(f'Epoch train loss is {epoch_train_loss}')\n    \n\n    \n    # create checkpoint variable and add important data\n    checkpoint = {\n            'epoch': epoch + 1,\n            'train_loss_min': epoch_train_loss,\n            'state_dict': model.state_dict(),\n            'optimizer': optimizer.state_dict(),\n        }\n    \n    # save checkpoint\n    save_ckp(checkpoint, False, checkpoint_path, best_model_path)\n    ## TODO: save the model if validation loss has decreased\n    if epoch_train_loss <= train_loss_min:\n            print('Train loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(train_loss_min,epoch_train_loss))\n            # save checkpoint as best model\n            save_ckp(checkpoint, True, checkpoint_path, best_model_path)\n            train_loss_min = epoch_train_loss\n    \n    time_elapsed = time.time() - start_time\n    print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title('train loss')\nplt.plot(total_train_loss)\nplt.xlabel('epochs')\nplt.ylabel('loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### If you like my kernel please do upvote"},{"metadata":{},"cell_type":"markdown","source":"**reference : [psudo labeling](https://www.kaggle.com/gc1023/fork-of-fasterrcnn-pseudo-labeling)**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}