{"cells":[{"metadata":{},"cell_type":"markdown","source":"reference: https://www.kaggle.com/ravi02516/end-to-end-effiecientdet-training-keras/notebook","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_df=pd.read_csv('../input/global-wheat-detection/train.csv')\ntrain_data_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id=[f'{i}.jpg' for i in train_data_df.image_id]\nxmins,ymins,xmaxs,ymaxs,area=[],[],[],[],[]\nfor bbox in train_data_df.bbox:\n    real_bbox=eval(bbox)\n    \n    xmin, ymin ,w ,h=real_bbox\n    \n    \n    \n    a=int(xmin+w)\n    b=int(ymin+h)\n    xmaxs.append(a)\n    ymaxs.append(b)\n\n    \n    c=int(xmin)\n    d=int(ymin)\n    xmins.append(c)\n    ymins.append(d)\n    \n    area.append(w*h)\n    \ndata=pd.DataFrame()\ndata['filename']=image_id\ndata['width']=train_data_df.width\ndata['width']=train_data_df.height\n\ndata['class']=['wheat']*len(image_id)\n\ndata['xmin']=xmins\ndata['ymin']=ymins\n\ndata['xmax']=xmaxs\ndata['ymax']=ymaxs\n\ndata['iscrowd']=[0]*len(image_id)\n\ndata['area']=area\ndata['source']=train_data_df.source\n\ndata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/raininbox/check-clean-big-small-bboxes\ndata=data.drop(data[(data[\"area\"]<300) | (data[\"xmax\"]-data[\"xmin\"]<10) | (data[\"xmax\"]-data[\"xmin\"]<10)].index)\ndata=data.drop([173,2169,118211,52868,117344,3687,2159,121633,113947])\ndata.reset_index(drop=True, inplace=True)\ndata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import namedtuple\n\nwidth = 1024\nheight = 1024\ncolumns = list(data)\ngrouped_data = []\n\ndef split(df, group):\n    gb = df.groupby(group)\n    return [namedtuple('data', ['filename', 'object'])(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]\n\n\ngroups = split(data, 'filename')\n\nfor group in groups:\n    xmins = []\n    xmaxs = []\n    ymins = []\n    ymaxs = []\n    classes_text = []\n    classes = []\n    iscrowd=[]\n    area=[]\n    for index, row in group.object.iterrows():\n        xmins.append(row['xmin'] / width)\n        xmaxs.append(row['xmax'] / width)\n        ymins.append(row['ymin'] / height)\n        ymaxs.append(row['ymax'] / height)\n        iscrowd.append(row[\"iscrowd\"])\n        area.append(row[\"area\"])\n        classes_text.append(row['class'].encode('utf8'))\n        classes.append(1)\n    grouped_data.append(dict(zip(columns + ['class_text'], [row['filename'], width, classes, xmins, ymins, xmaxs, ymaxs, iscrowd, area, row['source'], classes_text])))\n\ndataset = pd.DataFrame(grouped_data)\ndataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\ndataset.loc[:, 'fold'] = 0\nfor fold_number , (train_index, val_index) in enumerate(skf.split(dataset.index.values, y=dataset['source'].values)):\n    dataset.loc[val_index, 'fold'] = fold_number\ndataset = dataset.sort_values(['fold'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/alexandersoare/how-to-prepare-a-stratified-split/comments\nimport matplotlib.pyplot as plt\n\ntrain_df = dataset[dataset['fold'] != 0]\nval_df = dataset[dataset['fold'] == 0]\n\nfig = plt.figure(figsize=(20, 5))\ncounts = train_df['source'].value_counts()\nax1 = fig.add_subplot(1,2,1)\na = ax1.bar(counts.index, counts)\ncounts = val_df['source'].value_counts()\nax2 = fig.add_subplot(1,2,2)\na = ax2.bar(counts.index, counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom PIL import Image\nimport hashlib\nimport io\n\ndef int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef int64_list_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\n\ndef bytes_feature(value):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef bytes_list_feature(value):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\n\ndef float_list_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef create_tf_example(item, i):\n    with tf.io.gfile.GFile(os.path.join('../input/global-wheat-detection/train', '{}'.format(item.filename)), 'rb') as fid:\n        encoded_jpg = fid.read()\n    encoded_jpg_io = io.BytesIO(encoded_jpg)\n    image = Image.open(encoded_jpg_io)\n    key = hashlib.sha256(encoded_jpg).hexdigest()\n    filename = item.filename.encode('utf8')\n    width = 1024\n    height = 1024\n\n    tf_example = tf.train.Example(features=tf.train.Features(feature={\n        'image/height': int64_feature(height),\n        'image/width': int64_feature(width),\n        'image/filename': bytes_feature(filename),\n        'image/source_id':bytes_feature(str(i).encode('utf8')),\n        'image/key/sha256':bytes_feature(key.encode('utf8')),\n        'image/encoded':bytes_feature(encoded_jpg),\n        'image/format': bytes_feature('jpg'.encode('utf8')),\n        'image/object/bbox/xmin': float_list_feature(item.xmin),\n        'image/object/bbox/xmax': float_list_feature(item.xmax),\n        'image/object/bbox/ymin': float_list_feature(item.ymin),\n        'image/object/bbox/ymax': float_list_feature(item.ymax),\n        'image/object/class/text':bytes_list_feature(item.class_text),\n        'image/object/class/label':int64_list_feature(item['class']),\n        'image/object/is_crowd':int64_list_feature(item.iscrowd),\n        'image/object/area':float_list_feature(item.area)\n    }))\n    return tf_example\n\nfor fold in range(0,10):\n    val_df = dataset[dataset['fold'] == fold]\n    train_writer = tf.io.TFRecordWriter(f'{fold}.tfrecord')\n    for i, row in val_df.iterrows():    \n        tf_example = create_tf_example(row, i)       \n        train_writer.write(tf_example.SerializeToString())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}