{"cells":[{"metadata":{},"cell_type":"markdown","source":"As host said that they had adjusted the wrong label in the data set, but the training set did not adjust the label. So we need to adjust the label of the training set ourselves to avoid huge shake, hope it will be helpful to someone :))","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nimport ast","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/global-wheat-detection/train.csv')\ntrain[['x', 'y', 'w', 'h']] = pd.DataFrame(np.stack(train['bbox'].apply(lambda x: ast.literal_eval(x)))).astype(np.float32)\ntrain['x1'] = train['x'] + train['w']\ntrain['y1'] = train['y'] + train['h']\ntrain['area'] = train['w'] * train['h']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(95, 100, 1):\n    perc = np.percentile(train['area'], i)\n    print(f\"{i} percentile of area is {perc}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(0, 5, 1):\n    perc = np.percentile(train['area'], i)\n    print(f\"{i} percentile of area is {perc}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check the different the images with different sizes:\n\n    Big images size percentile:\n        95 percentile of area is 16102.799999999988\n        96 percentile of area is 17460.0\n        97 percentile of area is 19264.47999999998\n        98 percentile of area is 22046.0\n        99 percentile of area is 27456.23999999996\n\n    Small images size percentile:\n        0 percentile of area is 2.0\n        1 percentile of area is 1064.0\n        2 percentile of area is 1344.0\n        3 percentile of area is 1551.0\n        4 percentile of area is 1716.0","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let check the small boxes with area 80","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']<100]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(4)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['6284044ed','ad256655b', '233cb8750', '6a8522f06']\nbbox_id = [36287, 40034, 114998, 119089]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thses boxes should be remove which almost can't be see anything in it.What's about area 500","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']<200]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(4)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['78752f185','71b460a14', '3f8f6b1a1', 'd0ab06fc3']\nbbox_id = [115412, 4128, 145578, 66114]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']<300]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(4)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['0b2967a7a','1e9ff110c', 'c1577d6ff', '217c8fd61']\nbbox_id = [112057, 3465, 119930, 125886]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']<400]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(4)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['060543bbf','94ea18562', 'bbce58f71', '408013a9d']\nbbox_id = [88815, 84884, 122802, 112357]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In a word, we should remow w/h less than 10 and the area less than 300","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']<500]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(4)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['1f255e0c5','ea88fb8ec', 'b7c9166b6', '9a50eab86']\nbbox_id = [65825, 63607, 124682, 63881]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let check the big boxes at 1 percentile of area is 27456.23999999996","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']>27456.23999999996]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.head(4)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['9858d67dc','ffc870198', '536ef8d03', '3b552c95a']\nbbox_id = [33828, 4793, 54363, 120167]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Obviously this is still a suitable size，Let check the 0.5 percentile with the area 34200.0","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']>27456.23999999996]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(700)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['c4dc3c575','93d67b171', '93d67b171', 'd89f4ea06']\nbbox_id = [42775, 4360, 4352, 33604]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Obviously this is still a suitable size，Let check the 0.25 percentile with the area 41860.0","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']>27456.23999999996]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(350)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['be11c4e40','b8ddb6c73', '5a76259a0', '73ed5eb37']\nbbox_id = [2817, 128030, 116083, 90575]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> The box of fig4 seem to be too big, but it no a thing.Let's check the 0.125 percentile with the area 50912.0","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']>27456.23999999996]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(175)\nTrain_Box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 2\ngrid_height = 2\nimages_id = ['2c836cccb','d5943ea17', 'dcafcae79', '1e58125ec']\nbbox_id = [123482, 54427, 52733, 38737]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"okay with no problem.Check the top 15 big guys","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_Box = train[train['area']>27456.23999999996]\nTrain_Box = Train_Box.sort_values(axis=0, ascending=True, by=['area'])\nTrain_Box = Train_Box.tail(15)\nTrain_Box.head(15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid_width = 3\ngrid_height = 5\nimages_id = ['b8ddb6c73', 'f1a8585e0', '51f2e0a05', '69fc3d3ff', '9adbfe503', '41c0123cc','a1321ca95', 'ad6e9eea2', '9a30dd802', 'd7a02151d', '409a8490c', '2cc75e9f5', 'a1321ca95', 'd067ac2b1', '42e6efaaa']\nbbox_id = [128028, 53790, 53930, 1259, 54892, 173, 2169, 54702, 52868, 118211, 117344, 3687, 2159, 121633, 113947]\nfig, axs = plt.subplots(grid_height, grid_width,\n                        figsize=(15, 15))\n\nfor i, (img_id, box) in enumerate(zip(images_id, bbox_id)):\n    ax = axs[int(i / grid_width), i % grid_width]\n    image = cv2.imread(f'../input/global-wheat-detection/train/{img_id}.jpg', cv2.IMREAD_COLOR)\n    box = [int(Train_Box['x'][box]),int(Train_Box['y1'][box]),int(Train_Box['x1'][box]),int(Train_Box['y'][box])]\n    cv2.rectangle(image,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    ax.set_title(img_id)\n    ax.imshow(image.squeeze())\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is the one which we should remove","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Clean the bboxs to output new train ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/global-wheat-detection/train.csv')\ntrain[['x', 'y', 'w', 'h']] = pd.DataFrame(np.stack(train['bbox'].apply(lambda x: ast.literal_eval(x)))).astype(np.float32)\ntrain['area'] = train['w'] * train['h']\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_clean = train[train['area']>300]\ntrain_clean = train_clean[train['w']>10]\ntrain_clean = train_clean[train['h']>10]\ntrain_clean = train_clean.drop([173,2169,118211,52868,117344,3687,2159,121633,113947])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"remove {} boxes\".format(train.shape[0] - train_clean.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_clean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_clean.to_csv('train_clean.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}