{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this competition, you will detect wheat heads from outdoor images of wheat plants, including wheat datasets from around the globe.you will focus on solution to estimate the number and size of wheat heads.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport re\nfrom tqdm.notebook import tqdm\nfrom PIL import Image,ImageDraw\nimport hashlib\nimport ast\nfrom ast import literal_eval\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport os\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport torch\nimport torchvision\n\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\n\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nimport seaborn as sns\nsns.set()\n\nDIR_INPUT = '/kaggle/input/global-wheat-detection'\nDIR_TRAIN_IMAGES = f'{DIR_INPUT}/train'\nDIR_TEST_IMAGES = f'{DIR_INPUT}/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train= pd.read_csv(f'{DIR_INPUT}/train.csv')\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub=pd.read_csv(f'{DIR_INPUT}/sample_submission.csv')\nsub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Image Meta**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['image_id'].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['height'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['width'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['image_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* We have 3373 unique images in the train dataset.\n* It seems all of the images have same size (1024 x 1024).\n* Minimum number of bbox per image:0\n* Maximum number of bbox per image:116\n* No duplicate images in(train)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**EDA**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's Check the Dimension of images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['width'].unique() == train['height'].unique() == [1024]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bbox_area(bbox):\n    bbox = literal_eval(bbox)\n    return bbox[2]*bbox[3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['bbox_area']=train['bbox'].apply(get_bbox_area)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['bbox_area'].value_counts().hist(bins=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_images = train['image_id'].unique()\nlen(unique_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_hash(im):\n    md5 = hashlib.md5()\n    md5.update(np.array(im).tostring())\n    \n    return md5.hexdigest()\n    \ndef get_image_meta(image_id, image_src, dataset='train'):\n    im = Image.open(image_src)\n    extrema = im.getextrema()\n\n    meta = {\n        'image_id': image_id,\n        'dataset': dataset,\n        'hash': calculate_hash(im),\n        'r_min': extrema[0][0],\n        'r_max': extrema[0][1],\n        'g_min': extrema[1][0],\n        'g_max': extrema[1][1],\n        'b_min': extrema[2][0],\n        'b_max': extrema[2][1],\n        'height': im.size[0],\n        'width': im.size[1],\n        'format': im.format,\n        'mode': im.mode\n    }\n    return meta\ndata = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor i, image_id in enumerate(tqdm(train['image_id'].unique(), total=train['image_id'].unique().shape[0])):\n    data.append(get_image_meta(image_id, DIR_TRAIN_IMAGES + '/{}.jpg'.format(image_id)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_df = pd.DataFrame(data)\nmeta_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Duplication**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"duplicates = meta_df.groupby(by='hash')[['image_id']].count().reset_index()\nduplicates = duplicates[duplicates['image_id'] > 1]\nduplicates.reset_index(drop=True, inplace=True)\n\nduplicates = duplicates.merge(meta_df[['image_id', 'hash']], on='hash')\n\nduplicates.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Bounding box data**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['x'] = -1\ntrain['y'] = -1\ntrain['w'] = -1\ntrain['h'] = -1\n\ndef expand_bbox(x):\n    r = np.array(re.findall(\"([0-9]+[.]?[0-9]*)\", x))\n    if len(r) == 0:\n        r = [-1, -1, -1, -1]\n    return r\n\ntrain[['x', 'y', 'w', 'h']] = np.stack(train['bbox'].apply(lambda x: expand_bbox(x)))\ntrain.drop(columns=['bbox'], inplace=True)\ntrain['x'] = train['x'].astype(np.float)\ntrain['y'] = train['y'].astype(np.float)\ntrain['w'] = train['w'].astype(np.float)\ntrain['h'] = train['h'].astype(np.float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby(by='image_id')['source'].count().agg(['min', 'max', 'mean'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"source = train['source'].value_counts()\nsource","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look at how many bounding boxes do we have for each image:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train['image_id'].value_counts(), bins=10)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure(data=[\n    go.Pie(labels=source.index, values=source.values)\n])\n\nfig.update_layout(title='Source distribution')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_images(image_ids):\n    \n    col = 5\n    row = min(len(image_ids) // col, 5)\n    \n    fig, ax = plt.subplots(row, col, figsize=(16, 8))\n    ax = ax.flatten()\n\n    for i, image_id in enumerate(image_ids):\n        image = cv2.imread(DIR_TRAIN_IMAGES + '/{}.jpg'.format(image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        ax[i].set_axis_off()\n        ax[i].imshow(image)\n        ax[i].set_title(image_id)\n        \ndef show_image_bb(image_data):\n    \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n    \n    image = cv2.imread(DIR_TRAIN_IMAGES + '/{}.jpg'.format(image_data.iloc[0]['image_id']))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    for i, row in image_data.iterrows():\n        \n        cv2.rectangle(image,\n                      (int(row['x']), int(row['y'])),\n                      (int(row['x']) + int(row['w']), int(row['y']) + int(row['h'])),\n                      (220, 0, 0), 3)\n\n    ax.set_axis_off()\n    ax.imshow(image)\n    ax.set_title(image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_images(train.sample(n=15)['image_id'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image_bb(train[train['image_id'] == '5e0747034'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image_bb(train[train['image_id'] == '5b13b8160'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image_bb(train[train['image_id'] == '1f2b1a759'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"What can we tell from visulaizations:\n\n* there are plenty of overlappind bounding boxes\n* all photos seem to be taken vertically\n* all plants are can be rotated differently, there is no single orientation this means that different flip and rotation should probably help\n* colors of wheet heads are quite different seems to depend a little bit on the source\n* wheet heads themselves are seen from very different angles of a view relevant to the observer.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DIR_RESULTS = '/kaggle/input/global-wheat-detection-public'\n# Your OOF predictions\nVALID_RESULTS = [\n    f\"{DIR_RESULTS}/validation_results_fold0_best.csv\",\n    f\"{DIR_RESULTS}/validation_results_fold1_best.csv\",\n    f\"{DIR_RESULTS}/validation_results_fold2_best.csv\",\n    f\"{DIR_RESULTS}/validation_results_fold3_best.csv\",\n    f\"{DIR_RESULTS}/validation_results_fold4_best.csv\",\n]\n\nWEIGHTS_FILE = f'{DIR_RESULTS}/fasterrcnn_resnet50_fpn_best.pth'\n\n# Below this area the size category of the box is 'small'\nAREA_SMALL = 56 * 56\n\n# Below this (and above small) is medium;\n# Above this is large.\nAREA_MEDIUM = 96 * 96\n\n# If the box is at most this far from either of the borders\n# we mark the box as 'is_border = True'\nBORDER_SIZE = 2\n\n# In these experiments I used 800px inputs.\n# For analysis, we have to scale back to 1024px\n# because the GT boxes are in that size.\nSCALE = 1024/512\n\n# Analizing at this threshold\nTHRESHOLD = 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_prediction_string(pred_str):\n    data = list(map(float, pred_str.split(\" \")))\n    data = np.array(data)\n    \n    return data.reshape(-1, 5)[:, 1:]\n\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0])\n\n    if dx < 0:\n        return 0.0\n\n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1])\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0]) * (gt[3] - gt[1]) +\n            (pr[2] - pr[0]) * (pr[3] - pr[1]) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n\ndef find_best_match(gts, pred, pred_idx, threshold=0.5, form='pascal_voc', ious=None) -> int:\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n\n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n\n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n\n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\ndef gen_images(data, filters, output_folder='./output', prefix='', limit=100):\n    \n    res = 'fp'\n    resdata = data.copy()\n\n    for filt in filters:\n        resdata = resdata[resdata[filt[0]] == filt[1]]\n        \n        prefix = f\"{prefix}_{filt[1]}\"\n        \n        if filt[0] == 'result':\n            res = filt[1]\n        \n        \n    if limit > 0:\n        resdata = resdata.sample(n=limit)\n        \n    image_ids = resdata['image_id'].unique()\n    res_images = []\n    \n    for image_id in image_ids:\n        img = cv2.imread(DIR_TRAIN_IMAGES + '/{}.jpg'.format(image_id))\n        \n        if res == 'fn':\n            boxes = resdata[resdata['image_id'] == image_id][['gt_x1', 'gt_y1', 'gt_x2', 'gt_y2']].values\n        elif res == 'fp':\n            boxes = resdata[resdata['image_id'] == image_id][['pred_x1', 'pred_y1', 'pred_x2', 'pred_y2']].values\n        \n        for box in boxes:\n            # tp\n            color = (0, 220, 0)\n\n            if res == 'fp':\n                # Showing GT boxes nearby\n                tpfilt = (\n                    (data['image_id'] == image_id) &\n                    (data['gt_x1'] < box[2] + 16) &\n                    (data['gt_x2'] > box[0] - 16) &\n                    \n                    (data['gt_y1'] < box[3] + 16) &\n                    (data['gt_y2'] > box[1] - 16)\n                )\n            \n                tps = data[tpfilt][['gt_x1', 'gt_y1', 'gt_x2', 'gt_y2']].values\n                for tpbox in tps:\n                    cv2.rectangle(img,\n                                  (int(tpbox[0]), int(tpbox[1])),\n                                  (int(tpbox[2]), int(tpbox[3])),\n                                  color, 3)\n            \n            if res == 'fn':\n                color = (40, 40, 198)\n            elif res == 'fp':\n                color = (198, 40, 40)\n\n            cv2.rectangle(img,\n                          (int(box[0]), int(box[1])),\n                          (int(box[2]), int(box[3])),\n                          color, 3)\n                \n            \n        res_images.append((img, f\"{output_folder}/{prefix}_{image_id}.jpg\"))\n        \n    return res_images\n    \ndef save_images(data, filters, output_folder='./output', prefix='', limit=100):\n    images = gen_images(data=data, filters=filters, limit=limit)\n    \n    for image, path in images:\n        cv2.imwrite(path, image)\n        \ndef show_images(data, filters, rows=3, cols=2):\n    \n    images = gen_images(data=data, filters=filters, output_folder='', limit=rows*cols)\n    \n    fig, ax = plt.subplots(rows, cols, figsize=(16,16))\n    ax = ax.flatten()\n    \n    for i, (image, path) in enumerate(images):\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        ax[i].set_axis_off()\n        ax[i].imshow(image)\n        ax[i].set_title(path)\n        \n        \ndef show_image_boxes(train, data):\n    data = data.to_dict('records')\n\n    fig, ax = plt.subplots(1, 2, figsize=(16, 10))\n    ax = ax.flatten()\n    \n    image = cv2.imread(DIR_TRAIN_IMAGES + '/{}.jpg'.format(data[0]['image_id']))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    src_img = image.copy()\n    \n    boxes = train[train['image_id'] == data[0]['image_id']][['x', 'y', 'x2', 'y2']].values\n    \n    for box in boxes:\n        cv2.rectangle(src_img,\n                      (int(box[0]), int(box[1])),\n                      (int(box[2]), int(box[3])),\n                      (0, 220, 0), 2)\n\n    ax[0].set_axis_off()\n    ax[0].imshow(src_img)\n    ax[0].set_title(\"Image + GT boxes\")\n        \n    # noisy targets\n    for box_data in data:\n        # fn\n        color = (40, 40, 198)\n        box = [0, 0, 0, 0]\n\n        if box_data['result'] == 'fn':\n            box[0], box[1], box[2], box[3] = box_data['gt_x1'],\\\n                                             box_data['gt_y1'],\\\n                                             box_data['gt_x2'],\\\n                                             box_data['gt_y2']\n\n\n        elif box_data['result'] == 'fp':\n            \n            box[0], box[1], box[2], box[3] = box_data['pred_x1'],\\\n                                             box_data['pred_y1'],\\\n                                             box_data['pred_x2'],\\\n                                             box_data['pred_y2']\n\n            color = (198, 40, 40)\n\n        cv2.rectangle(image,\n                      (int(box[0]), int(box[1])),\n                      (int(box[2]), int(box[3])),\n                      color, 2)\n\n    ax[1].set_axis_off()\n    ax[1].imshow(image)\n    ax[1].set_title(\"Blue: FP (predicted, no GT) | Red: FN (GT, no prediction)\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ntrain['x2'] = train['x'] + train['w']\ntrain['y2'] = train['y'] + train['h']\n\n# Calculate the area of the boxes.\ntrain['area'] = train['w'] * train['h']\n\n# Is the box at the edge of the image\ntrain['is_border'] = False\n\nborder_filt = ((train['x'] < BORDER_SIZE) | (train['y'] < BORDER_SIZE) |\n             (train['x2'] > 1024 - BORDER_SIZE) | (train['y2'] > 1024 - BORDER_SIZE))\ntrain.loc[border_filt, 'is_border'] = True\n\ntrain['size'] = 'large'\ntrain.loc[train['area'] < AREA_MEDIUM, 'size'] = 'medium'\ntrain.loc[train['area'] < AREA_SMALL, 'size'] = 'small'\n\n# These are the ground-truth boxes\ntrain['is_gt'] = True\n\ntrain['brightness'] = 0.0\ntrain['contrast'] = 0.0\ntrain['overlap_iou'] = 0.0\n\ntrain.sort_values(by='image_id', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# - Brightness\n# - Contrast\n# - Hightest overlap with other GT box\n\nlast_src_id = None\nsrc = None\n\nfor i, row in tqdm(train.iterrows(), total=train.shape[0]):\n    \n    if last_src_id != row['image_id']:\n        src = cv2.imread(DIR_TRAIN_IMAGES+ '/{}.jpg'.format(row['image_id']))\n        last_src_id = row['image_id']\n\n    \n    y1 = int(row['y'])\n    y2 = int(row['y2'])\n    x1 = int(row['x'])\n    x2 = int(row['x2'])\n\n    image = src[y1:y2, x1:x2].copy()\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n    \n    train.loc[i, 'brightness'] = image[:, :, 2].mean()\n    \n    image = cv2.cvtColor(image, cv2.COLOR_HSV2RGB)\n    image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n    \n    train.loc[i, 'contrast'] = image.std()\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Validation data**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"I trained 5-folds; these validation results are the out-of-fold results.By combining these, we can analyze the entire training set.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Format of the validation dataframes\npd.read_csv(VALID_RESULTS[0], usecols=['image_id', 'PredictionString']).head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid= []\n\n# helper.\nimage = train.groupby(by=['image_id', 'source'])[['is_gt']].nunique().reset_index()[['image_id', 'source']]\n\nfor src in VALID_RESULTS:\n    valid_df = pd.read_csv(src, usecols=['image_id', 'PredictionString'])\n    valid_df = valid_df.merge(image[['image_id', 'source']], how='left', on='image_id')\n    valid_df.reset_index(drop=True, inplace=True)\n\n    res = []\n\n    for i, row in valid_df.iterrows():\n        boxes = decode_prediction_string(row['PredictionString'])\n        for box in boxes:\n            valid.append({'image_id': row['image_id'],'width': 1024,'height': 1024,'bbox': '',\n'source': row['source'],'x': box[0] * SCALE,'y': box[1] * SCALE,'x2': (box[0] + box[2]) * SCALE,'y2': (box[1] + box[3]) * SCALE,'w': box[2] * SCALE,'h': box[3] * SCALE,'area': (box[2] * box[3]) * SCALE,\n'size': 'large','is_border': False,'is_gt': False,'brightness': 0.0,'contrast': 0.0})\n\n\n# Convert the list to a pd.DataFrame\nvalid = pd.DataFrame(valid)\n\nborder_filt = ((valid['x'] < BORDER_SIZE) | (valid['y'] < BORDER_SIZE) |\n             (valid['x2'] > 1024 - BORDER_SIZE) | (valid['y2'] > 1024 - BORDER_SIZE))\nvalid.loc[border_filt, 'is_border'] = True\n\nvalid.loc[valid['area'] < AREA_MEDIUM, 'size'] = 'medium'\nvalid.loc[valid['area'] < AREA_SMALL, 'size'] = 'small'\n\nvalid.sort_values(by='image_id', inplace=True)\n# Calculate box infos\n# - Brightness","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Prepare the final dataframe**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc(gts, preds, threshold=0.5, form='pascal-voc'):\n    \n    def _get_data(image_id, res, gt, pr):\n        return {\n                'image_id': image_id,\n                'gt_x1': gt[1] if gt is not None else np.nan,\n                'gt_y1': gt[2] if gt is not None else np.nan,\n                'gt_x2': gt[3] if gt is not None else np.nan,\n                'gt_y2': gt[4] if gt is not None else np.nan,\n                'gt_w': gt[5] if gt is not None else np.nan,\n                'gt_h': gt[6] if gt is not None else np.nan,\n                'gt_area': gt[7] if gt is not None else np.nan,\n                'gt_is_border': gt[8] if gt is not None else False,\n                'gt_brightness': gt[12] if gt is not None else np.nan,\n                'gt_contrast': gt[13] if gt is not None else np.nan,\n                \n                'pred_x1': pr[1] if pr is not None else np.nan,\n                'pred_y1': pr[2] if pr is not None else np.nan,\n                'pred_x2': pr[3] if pr is not None else np.nan,\n                'pred_y2': pr[4] if pr is not None else np.nan,\n                'pred_w': pr[5] if pr is not None else np.nan,\n                'pred_h': pr[6] if pr is not None else np.nan,\n                'pred_area': pr[7] if pr is not None else np.nan,\n                'pred_is_border': pr[8] if pr is not None else False,\n                'pred_brightness': pr[12] if pr is not None else np.nan,\n                'pred_contrast': pr[13] if pr is not None else np.nan,\n                \n                'size': gt[10] if gt is not None else pr[10],\n                'source': gt[11] if gt is not None else pr[11],\n            \n                'result': res\n            }\n    \n    results = []\n    \n    # Number of predictions\n    n = len(preds)\n    \n    for pred_idx in range(n):\n        pr = preds[pred_idx]\n        \n        best_match_gt_idx = find_best_match(gts[:, 1:5], pr[1:5], pred_idx, threshold=threshold, form=form)\n        \n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            gt = gts[best_match_gt_idx]\n            results.append(_get_data(gt[0], 'tp', gt, pr))\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            results.append(_get_data(pr[0], 'fp', None, pr))\n\n    for gt in gts:\n        if gt[1] < 0:\n            continue\n            \n        results.append(_get_data(gt[0], 'fn', gt, None))\n    \n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = ['image_id', 'x', 'y', 'x2', 'y2', 'w', 'h', 'area', 'is_border',\n        'is_gt', 'size', 'source', 'brightness', 'contrast']\n\nvalid_img_ids = valid['image_id'].unique()\n\nresults = []\n\nfor img_id in tqdm(valid_img_ids, total=len(valid_img_ids)):\n    gt_boxes = train[train['image_id'] == img_id][cols].values\n    pred_boxes = valid[valid['image_id'] == img_id][cols].values\n    \n    results += calc(gt_boxes, pred_boxes, threshold=THRESHOLD, form='pascal-voc')\n    \nresults = pd.DataFrame(results)\n\nresults['is_border'] = False\nresults.loc[(results['gt_is_border'] == True) | (results['pred_is_border'] == True), 'is_border'] = True\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Results by Source**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_by_group(data, filt, group, idx, cols, title='', names=None, colors=None, order=None):\n    \n    if filt is not None:\n        data = data[filt] \n    \n    res = data.groupby(by=group).count()[['image_id']].reset_index().sort_index()\n    res = res.pivot(index=idx, columns=cols, values='image_id')\n    \n    fig = go.Figure()\n    \n    if order is None:\n        order = range(res.shape[0])\n\n    for row_idx in order:\n        fig.add_trace(go.Bar(\n            x=res.columns,\n            y=res.iloc[row_idx].values,\n            name=names[row_idx] if names is not None else res.index[row_idx],\n            marker_color=colors[row_idx] if colors is not None else None\n        ))\n        \n    fig.update_layout(\n        barmode='stack',\n        barnorm = 'percent',\n        title = {\n            'text': title\n        }\n    )\n    \n    return res, fig\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res, fig = show_by_group(data=results, filt=None, group=['source', 'result'],\n                         idx='result',\n                         cols='source',\n                         names=['False Negative', 'False Positive', 'True Positive'],\n                         colors=['#c62828', '#3f51b5', '#4caf50'],\n                         title='Results (TP|FP|FN) by sources'\n                        )\n\nres","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sources = results[results['result'] == 'fn']['source'].value_counts().sort_index().sort_index()\n\nfig = go.Figure([go.Pie(labels=sources.index, values=sources.values)])\nfig.update_layout(\n    title = {\n        'text': f'False negatives - at threshold {THRESHOLD}'\n    }\n)\nfig.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filters = [\n    ('result', 'fn'),\n    ('is_border', True),\n    ('source', 'rres_1')\n]\n\nshow_images(results.copy(), filters, rows=3, cols=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Brightness differences**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Histogram(x=results[results['result'] == 'fn']['gt_brightness'],\n                           histnorm='probability', name='False negatives', marker={'color': '#c62828'}))\nfig.add_trace(go.Histogram(x=results[results['result'] == 'tp']['gt_brightness'],\n                           histnorm='probability', name='True positives', marker={'color': '#4caf50'}))\n\nfig.update_layout(barmode='overlay', title={\n    'text': 'Brightness'\n})\nfig.update_traces(opacity=0.75)\n\nfig.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Contrast difference**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Histogram(x=results[results['result'] == 'fn']['gt_contrast'],\n                           histnorm='probability', name='False negatives', marker={'color': '#c62828'}))\nfig.add_trace(go.Histogram(x=results[results['result'] == 'tp']['gt_contrast'],\n                           histnorm='probability', name='True positives', marker={'color': '#4caf50'}))\n\nfig.update_layout(barmode='overlay', title={\n    'text': 'Contrast'\n})\nfig.update_traces(opacity=0.75)\n\nfig.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filters = [\n    ('result', 'fp'),\n    ('size', 'large'),\n    ('source', 'inrae_1')\n]\n\nshow_images(results.copy(), filters, rows=3, cols=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filters=[\n    ('result', 'fp'),\n    ('is_border', True),\n    ('source', 'inrae_1'),\n]\n\nshow_images(results.copy(), filters, rows=3, cols=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filters = [\n    ('result', 'fp'),\n    ('size', 'small'),\n    ('is_border', True)\n]\n\nshow_images(results.copy(), filters, rows=3, cols=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Results of \"normal\" boxes**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"res, fig = show_by_group(data=results, filt=results['is_border'] == False,\n                         group=['size', 'result'],\n                         idx='result',\n                         cols='size',\n                         names=['False Negative', 'False Positive', 'True Positive'],\n                         colors=['#c62828', '#3f51b5', '#4caf50'],\n                         title='Normal results by size'\n                        )\n\nres","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filters = [\n    ('result', 'fp'),\n    ('size', 'small'),\n    ('is_border', False)\n]\n\nshow_images(results.copy(), filters, rows=3, cols=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Noisy targets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image_ids = results['image_id'].unique()\n\nresults_noisy = []\n\nfor image_id in tqdm(image_ids, total=image_ids.shape[0]):\n    fps = results[(results['image_id'] == image_id) & (results['result'] == 'fp')]\n    fps.reset_index(drop=True, inplace=True)\n\n    fns = results[(results['image_id'] == image_id) & (results['result'] == 'fn')]\n    fns.reset_index(drop=True, inplace=True)\n\n    for fpi, fp in fps.iterrows():\n        \n        for fni, fn in fns.iterrows():\n            \n            if ((fp['pred_x1'] <= fn['gt_x1']) and\n                (fp['pred_y1'] <= fn['gt_y1']) and\n                (fp['pred_x2'] >= fn['gt_x2']) and   \n                (fp['pred_y2'] >= fn['gt_y2'])):\n                \n                # GT inside predicted\n                results_noisy.append(fp.to_dict())\n                results_noisy.append(fn.to_dict())\n            \n            elif ((fp['pred_x1'] >= fn['gt_x1']) and\n                  (fp['pred_y1'] >= fn['gt_y1']) and\n                  (fp['pred_x2'] <= fn['gt_x2']) and   \n                  (fp['pred_y2'] <= fn['gt_y2'])):\n                \n                # PREDICTED inside GT\n                results_noisy.append(fp.to_dict())\n                results_noisy.append(fn.to_dict())\n\n\nresults_noisy = pd.DataFrame(results_noisy)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_noisy.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"noisy_sources = pd.DataFrame(train['source'].value_counts().sort_index())\nnoisy_sources['noisy'] = (results_noisy['source'].value_counts() // 2).sort_index().values\nnoisy_sources['p'] = noisy_sources['noisy'] / noisy_sources['source'] * 100\n\nnoisy_sources.sort_values(by='p', ascending=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image_boxes(train, results_noisy[results_noisy['image_id'] == '4021d47d4'].copy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image_boxes(train, results_noisy[results_noisy['image_id'] == '7b72ea0fb'].copy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WheatTestDataset(Dataset):\n\n    def __init__(self, dataframe, image_dir, transforms=None):\n        super().__init__()\n\n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n\n        image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n\n        if self.transforms:\n            sample = {\n                'image': image,\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        return image, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Albumentations\ndef get_test_transform():\n    return A.Compose([\n        # A.Resize(512, 512),\n        ToTensorV2(p=1.0)\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load a model; pre-trained on COCO\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\nnum_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n# Load the trained weights\nmodel.load_state_dict(torch.load(WEIGHTS_FILE))\nmodel.eval()\n\nx = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntest_dataset = WheatTestDataset(sub, DIR_TEST_IMAGES, get_test_transform())\n\ntest_data_loader = DataLoader(\n    test_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detection_threshold = 0.5\nresults = []\n\nfor images, image_ids in test_data_loader:\n\n    images = list(image.to(device) for image in images)\n    outputs = model(images)\n\n    for i, image in enumerate(images):\n\n        boxes = outputs[i]['boxes'].data.cpu().numpy()\n        scores = outputs[i]['scores'].data.cpu().numpy()\n        \n        boxes = boxes[scores >= detection_threshold].astype(np.int32)\n        scores = scores[scores >= detection_threshold]\n        image_id = image_ids[i]\n        \n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n        \n        result = {\n            'image_id': image_id,\n            'PredictionString': format_prediction_string(boxes, scores)\n        }\n\n        \n        results.append(result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results[0:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\nsub.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = images[1].permute(1,2,0).cpu().numpy()\nboxes = outputs[1]['boxes'].data.cpu().numpy()\nscores = outputs[1]['scores'].data.cpu().numpy()\n\nboxes = boxes[scores >= detection_threshold].astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 2)\n    \nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Please upvote !!! After reading this notebook.**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}