{"cells":[{"metadata":{},"cell_type":"markdown","source":"Check the EDA notebook <a href = \"https://www.kaggle.com/daenys2000/global-wheat-detection-eda\">here</a>.<br>\nCode has been adapted for the wheat dataset using this <a href = \"https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html\">tutorial</a>.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport time\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport albumentations as alb\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport torch\nfrom torch.utils.data import DataLoader,Dataset\nfrom torch.utils.data import SubsetRandomSampler\nimport torchvision \nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.rpn import AnchorGenerator\nfrom torchvision.models.detection import FasterRCNN\nimport cv2\nfrom tqdm.notebook import tqdm\nimport torch.nn as nn\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare the dataset","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_path = '/kaggle/input/global-wheat-detection/train.csv'\ntrain_img_path = '/kaggle/input/global-wheat-detection/train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(train_path)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#append .jpg to image ids for easier handling\ntrain['image_id'] = train['image_id'].apply(lambda x: str(x) + '.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#separating x,y,w,h into separate columns for convenience\nbboxes = np.stack(train['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep = ',')))\nfor i, col in enumerate(['x_min', 'y_min', 'w', 'h']):\n    train[col] = bboxes[:,i]\n\ntrain.drop(columns = ['bbox'], inplace = True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['bbox_area'] = train['w']*train['h']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#setting thresholds for maximum and minimum areas of boxes\nmax_area = 100000\nmin_area = 40","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#remove boxes that are too big or small\ntrain_clean = train[(train['bbox_area'] < max_area) & (train['bbox_area'] > min_area)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#splitting into train and valid ids\ntrain_split = 0.8\n\nimage_ids = train_clean['image_id'].unique()\ntrain_ids = image_ids[0:int(train_split*len(image_ids))]\nval_ids = image_ids[int(train_split*len(image_ids)):]\n\nprint('Length of training ids', len(train_ids))\nprint('Length of validation ids', len(val_ids))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_clean[train_clean['image_id'].isin(train_ids)]\nvalid_df = train_clean[train_clean['image_id'].isin(val_ids)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WheatDataset(Dataset):\n    def __init__(self, df, image_dir,transform = None):\n        super().__init__()\n        self.df = df\n        self.img_ids = df['image_id'].unique()\n        self.image_dir = image_dir\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.img_ids)\n    \n    def __getitem__(self, idx: int):\n        image_id = self.img_ids[idx]\n        pts = self.df[self.df['image_id'] == image_id]\n        \n        image = cv2.imread(os.path.join(self.image_dir, image_id), cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image = image/255.0\n        \n        boxes = pts[['x_min', 'y_min', 'w', 'h']].values\n        \n        #convert boxes to x1,y1,x2,y2 format because that is what resnet50 faster cnn in pytorch expects\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        \n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0]) #width times height\n        area = torch.as_tensor(area, dtype = torch.float32)\n        \n        # there is only one class\n        labels = torch.ones((pts.shape[0],), dtype=torch.int64)\n        \n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((pts.shape[0],), dtype=torch.int32)\n        \n        target = {}\n        target[\"boxes\"] = boxes\n        target[\"labels\"] = labels\n        target[\"image_id\"] = torch.tensor(idx)\n        target[\"area\"] = area\n        target[\"iscrowd\"] = iscrowd\n\n        if self.transform:\n            sample = {\n                'image': image,\n                'bboxes': target['boxes'],\n                'labels': target['labels']\n            }\n            sample = self.transform(**sample)\n            image = sample['image']\n            \n            if len(sample['bboxes']) > 0:\n                target['boxes'] = torch.as_tensor(sample['bboxes'], dtype=torch.float32)\n            else:\n                target['boxes'] = torch.linspace(0,3, steps = 4, dtype = torch.float32)\n                target['boxes'] = target['boxes'].reshape(-1,4)\n            \n            #target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n            #target['boxes'] = torch.as_tensor(sample['bboxes'], dtype=torch.float32)\n            #target['boxes'] = torch.tensor(sample['bboxes'], dtype = torch.float32)\n            \n        return image, target, image_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#defining the transformations\ndef get_training_transforms():\n    return alb.Compose([\n    alb.VerticalFlip(p = 0.5),\n    alb.HorizontalFlip(p = 0.5),\n    #alb.RandomBrightness(p = 0.5),\n    #alb.RandomContrast(p = 0.5),\n    ToTensorV2(p = 1.0)\n], p=1.0, bbox_params=alb.BboxParams(format='pascal_voc', label_fields=['labels']))\n\ndef get_validation_transforms():\n    return alb.Compose([ToTensorV2(p = 1.0)], p = 1.0, bbox_params = alb.BboxParams(format='pascal_voc', label_fields=['labels']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create the Model!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#num_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\n#in_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\n#model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load a pre-trained model for classification and return\n# only the features\n\ndensenet_net = torchvision.models.densenet169(pretrained=True)\n# FasterRCNN needs to know the number of\n# output channels in a backbone. For mobilenet_v2, it's 1280\n# so we need to add it here\nmodules = list(densenet_net.children())[:-1]\nbackbone = nn.Sequential(*modules)\nbackbone.out_channels = 1664\n\n# let's make the RPN generate 5 x 3 anchors per spatial\n# location, with 5 different sizes and 3 different aspect\n# ratios. We have a Tuple[Tuple[int]] because each feature\n# map could potentially have different sizes and\n# aspect ratios\nanchor_generator = AnchorGenerator(sizes=((32, 64, 128, 256, 512),),\n                                   aspect_ratios=((0.5, 1.0, 2.0),))\n\n# let's define what are the feature maps that we will\n# use to perform the region of interest cropping, as well as\n# the size of the crop after rescaling.\n# if your backbone returns a Tensor, featmap_names is expected to\n# be [0]. More generally, the backbone should return an\n# OrderedDict[Tensor], and in featmap_names you can choose which\n# feature maps to use.\nroi_pooler = torchvision.ops.MultiScaleRoIAlign(featmap_names=[\"0\"],\n                                                output_size=7,\n                                                sampling_ratio=2)\n\n# put the pieces together inside a FasterRCNN model\nmodel = FasterRCNN(backbone,\n                   num_classes=2,\n                   rpn_anchor_generator=anchor_generator,\n                   box_roi_pool=roi_pooler)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#setting up dataloaders \ntraining_dataset = WheatDataset(train_df, train_img_path, get_training_transforms())\nvalidation_dataset = WheatDataset(valid_df, train_img_path, get_validation_transforms())\n\ntrain_dataloader = DataLoader(\n        training_dataset, batch_size=2, shuffle= True, num_workers=4,\n        collate_fn= collate_fn)\n\nvalid_dataloader = DataLoader(\n        validation_dataset, batch_size=2, shuffle=False, num_workers=4,\n        collate_fn=collate_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train on the GPU or on the CPU, if a GPU is not available\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ntorch.cuda.empty_cache()\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Before iterating over the dataset, it’s good to see what the model expects during training and inference time on sample data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"images, targets, image_ids = next(iter(train_dataloader))\nimages = list(image.to(device) for image in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes = targets[0]['boxes'].cpu().numpy().astype(np.int32)\nimg = images[0].permute(1,2,0).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(20, 10))\n\nfor box in boxes:\n    rect = patches.Rectangle((box[0],box[1]),box[2] - box[0],box[3] - box[1],linewidth=2,edgecolor='r',facecolor='none')\n    ax.add_patch(rect)\n    \nax.set_axis_off()\nax.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model.to(device)\n# construct an optimizer\nparams = [p for p in model.parameters() if p.requires_grad]\n\noptimizer = torch.optim.SGD(params, lr= 0.01, momentum=0.9, dampening=0, weight_decay=0, nesterov=False)\n\n# and a learning rate scheduler\nlr_scheduler =  torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=1, verbose=True, threshold=0.0001, threshold_mode='abs', cooldown=0, min_lr=1e-8, eps=1e-08)\n# let's train it for 10 epochs\nnum_epochs = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total_train_loss = []\ntotal_test_loss = []\n#itr = 1\n\nfor epoch in range(num_epochs):\n    model.train()\n\n    print('Epoch: ', epoch + 1)\n    train_loss = []\n    \n    for images, targets, image_ids in tqdm(train_dataloader):\n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        \n        loss_dict = model(images, targets)  \n        losses = sum(loss for loss in loss_dict.values())\n        loss_value = losses.item()\n        train_loss.append(loss_value)\n        \n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n        \n        #if itr%50 == 0:\n            #print('Iteration: ' + str(itr) + '\\n' + 'Loss: '+ str(loss_value))\n            \n        #itr += 1\n        \n    epoch_loss = np.mean(train_loss)\n    print('Epoch Loss is: ' , epoch_loss)\n    total_train_loss.append(epoch_loss)\n    \n    with torch.no_grad():\n        test_losses = []\n        for images, targets, image_ids in tqdm(valid_dataloader):\n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n            \n            loss_dict = model(images, targets)\n\n            losses = sum(loss for loss in loss_dict.values())\n            test_loss = losses.item()\n            test_losses.append(test_loss)\n            \n    test_losses_epoch = np.mean(test_losses)\n    print('Test Loss: ' ,test_losses_epoch)\n    total_test_loss.append(test_losses_epoch)\n    \n    if lr_scheduler is not None:\n        lr_scheduler.step(test_losses_epoch)\n        \ntorch.save(model.state_dict(), 'fasterrcnn.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(np.arange(num_epochs),total_train_loss ,label = 'train')\nplt.plot(np.arange(num_epochs),total_test_loss, label = 'test')\nplt.title('Loss over epochs')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, targets, image_ids in next(iter(valid_dataloader))\nimages = list(image.to(device) for image in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]\nimages\n\nprediction = model(images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize = (20,10))\nboxes = targets[0]['boxes'].cpu().numpy().astype(np.int32)\nimg = images[0].permute(1,2,0).cpu().numpy()\n\nfor box in boxes:\n    rect = patches.Rectangle((box[0],box[1]),box[2] - box[0],box[3] - box[1],linewidth=2,edgecolor='r',facecolor='none')\n    ax[0].add_patch(rect)\nax[0].set_title('Actual')\nax[0].set_axis_off()\nax[0].imshow(img)\n\nthresh = 0.5\nbox_preds = prediction[0]['boxes'].cpu().detach().numpy()\nscore_preds = prediction[0]['scores'].cpu().detach().numpy()\nbox_preds = box_preds[score_preds >= thresh].astype(np.int32)\nfor box in box_preds:\n    rect = patches.Rectangle((box[0],box[1]),box[2] - box[0],box[3] - box[1],linewidth=2,edgecolor='r',facecolor='none')\n    ax[1].add_patch(rect)\nax[1].set_title('Predicted')\nax[1].set_axis_off()\nax[1].imshow(img)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}