{"cells":[{"metadata":{},"cell_type":"markdown","source":"Someting **wrong** with the official data annotations [Here](https://www.kaggle.com/c/global-wheat-detection/discussion/149032)\n\nLet's find them and fix the annotations before official solutions come out."},{"metadata":{},"cell_type":"markdown","source":"We define some functions for processing the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport re\nimport numpy as np\nfrom collections import namedtuple\nfrom typing import List, Union\nfrom matplotlib import pyplot as plt\nimport cv2\nimport json\n\n\nDIR_INPUT = '/kaggle/input/global-wheat-detection'\nDIR_TRAIN = f'{DIR_INPUT}/train'\nDIR_TEST = f'{DIR_INPUT}/test'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# define some functions, some of them were modified from other kernels\ndef expand_bbox(x):\n    r = np.array(re.findall(\"([0-9]+[.]?[0-9]*)\", x))\n    if len(r) == 0:\n        r = [-1, -1, -1, -1]\n    return r\n\n\ndef transform_data_struct(train_df):\n    # transform data structure\n    train_df['x'] = -1\n    train_df['y'] = -1\n    train_df['w'] = -1\n    train_df['h'] = -1\n\n    train_df[['x', 'y', 'w', 'h']] = np.stack(train_df['bbox'].apply(lambda x: expand_bbox(x)))\n    #train_df.drop(columns=['bbox'], inplace=True)\n    train_df['x'] = train_df['x'].astype(np.float)\n    train_df['y'] = train_df['y'].astype(np.float)\n    train_df['w'] = train_df['w'].astype(np.float)\n    train_df['h'] = train_df['h'].astype(np.float)\n    return train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"functions for calculate the overlap between pics"},{"metadata":{"trusted":true},"cell_type":"code","source":"def overlap(gt: List[Union[int, float]],\n                  pred: List[Union[int, float]],\n                  form: str = 'pascal_voc') -> float:\n    \"\"\"Calculates the overlap percentage.\n\n    Args:\n        gt: List[Union[int, float]] coordinates of the ground-truth box\n        pred: List[Union[int, float]] coordinates of the prdected box\n        form: str gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        IoU: float Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    Box = namedtuple('Box', 'xmin ymin xmax ymax')\n\n    if form == 'coco':\n        bgt = Box(gt[0], gt[1], gt[0] + gt[2], gt[1] + gt[3])\n        bpr = Box(pred[0], pred[1], pred[0] + pred[2], pred[1] + pred[3])\n    else:\n        bgt = Box(gt[0], gt[1], gt[2], gt[3])\n        bpr = Box(pred[0], pred[1], pred[2], pred[3])\n\n    overlap_area = 0.0\n\n    # Calculate overlap area\n    dx = min(bgt.xmax, bpr.xmax) - max(bgt.xmin, bpr.xmin)\n    dy = min(bgt.ymax, bpr.ymax) - max(bgt.ymin, bpr.ymin)\n\n    if (dx > 0) and (dy > 0):\n        overlap_area = dx * dy\n\n    return overlap_area","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"function for draw the error annos"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_image_bboxes(img_id: str = '', box: list = None):\n    box = [int(x) for x in box]\n    image = cv2.imread(f'{DIR_TRAIN}/{img_id}.jpg', cv2.IMREAD_COLOR)\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.set_axis_off()\n    ax.imshow(image / 255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(f'{DIR_INPUT}/train.csv')\ntrain_df = transform_data_struct(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We filter out the bounding box that contains(overlap more than 80%, a parameter) 2(the other parameter) other bounding boxes"},{"metadata":{"trusted":true},"cell_type":"code","source":"seg_list = []\ncnt_max_num = 2\ncnt_ratio = 0.8\ntotal_img_num = len(train_df['image_id'].unique())\ntrain_cnt = 0\nerror_img_id = []\n\nfor img_id in train_df['image_id'].unique():\n    train_cnt += 1\n    print('processing on '+img_id+'('+str(train_cnt)+'/'+str(total_img_num)+')')\n    img_df = train_df[train_df['image_id']==img_id]\n    for index_1, detect_df in img_df.iterrows():\n        cnt = 0\n        detect_bb = [detect_df['x'], detect_df['y'], detect_df['w'], detect_df['h']]\n        for _, test_df in img_df.iterrows():\n            test_bb = [test_df['x'], test_df['y'], test_df['w'], test_df['h']]\n            overlap_area = overlap(detect_bb, test_bb, 'coco')\n            if overlap_area > test_bb[2] * test_bb[3] * cnt_ratio:\n                cnt += 1\n            if cnt > cnt_max_num:\n                print(img_id)\n                print(detect_bb)\n                error_img_id.append({'img_id':img_id, 'index': index_1, 'detect_bb': detect_bb})\n                img_df = img_df.drop(labels=index_1, axis=0)\n                break\n    seg_list.append(img_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save\npd.concat(seg_list).to_csv('calibrate_train.csv')\nwith open('error_img_id.json', 'w') as f:\n    json.dump(error_img_id, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# vis\nfor im in error_img_id:\n    img_id = im['img_id']\n    box = im['detect_bb']\n    plot_image_bboxes(img_id, box)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"error_img_id\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(error_img_id)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}