{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n# #         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_df=pd.read_csv(\"../input/global-wheat-detection/train.csv\")\ntrain_data_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id=[f\"{i}.jpg\" for i in train_data_df.image_id]\nxmins,ymins,xmaxs,ymaxs=[],[],[],[]\nfor bbox in train_data_df.bbox:\n    real_bbox=eval(bbox)\n    \n    xmin, ymin ,w ,h=real_bbox\n    \n    \n    \n    a=int(xmin+w)\n    b=int(ymin+h)\n    xmaxs.append(a)\n    ymaxs.append(b)\n\n    \n    c=int(xmin)\n    d=int(ymin)\n    xmins.append(c)\n    ymins.append(d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data=pd.DataFrame()\ndata[\"filename\"]=image_id\ndata[\"width\"]=train_data_df.width\ndata[\"width\"]=train_data_df.height\n\ndata[\"class\"]=[\"wheat\"]*len(image_id)\n\ndata[\"xmin\"]=xmins\ndata[\"ymin\"]=ymins\n\ndata[\"xmax\"]=xmaxs\ndata[\"ymax\"]=ymaxs\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.to_csv(\"train_labels.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv(\"/kaggle/working/train_labels.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Create tfrecord for training**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def int64_feature(value):\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef int64_list_feature(value):\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\n\ndef bytes_feature(value):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef bytes_list_feature(value):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\n\ndef float_list_feature(value):\n  return tf.train.Feature(float_list=tf.train.FloatList(value=value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import division\nfrom __future__ import print_function\nfrom __future__ import absolute_import\n\nimport os\nimport io\nimport pandas as pd\nimport tensorflow as tf\n\nfrom PIL import Image\nfrom collections import namedtuple, OrderedDict\n\n\n# TO-DO replace this with label map\ndef class_text_to_int(row_label):\n    if row_label == 'wheat':\n        return 1\n    else:\n        None\n\n\ndef split(df, group):\n    data = namedtuple('data', ['filename', 'object'])\n    gb = df.groupby(group)\n    return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]\n\n\ndef create_tf_example(group, path):\n    with tf.io.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid:\n        encoded_jpg = fid.read()\n    encoded_jpg_io = io.BytesIO(encoded_jpg)\n    image = Image.open(encoded_jpg_io)\n    \n    width, height = image.size\n\n    filename = group.filename.encode('utf8')\n    image_format = b'jpg'\n    xmins = []\n    xmaxs = []\n    ymins = []\n    ymaxs = []\n    classes_text = []\n    classes = []\n\n    for index, row in group.object.iterrows():\n        xmins.append(row['xmin'] / width)\n        xmaxs.append(row['xmax'] / width)\n        ymins.append(row['ymin'] / height)\n        ymaxs.append(row['ymax'] / height)\n        classes_text.append(row['class'].encode('utf8'))\n        classes.append(class_text_to_int(row['class']))\n\n    tf_example = tf.train.Example(features=tf.train.Features(feature={\n        'image/height': int64_feature(height),\n        'image/width': int64_feature(width),\n        'image/filename': bytes_feature(filename),\n        'image/source_id':bytes_feature(filename),\n        'image/encoded':bytes_feature(encoded_jpg),\n        'image/format': bytes_feature(image_format),\n        'image/object/bbox/xmin': float_list_feature(xmins),\n        'image/object/bbox/xmax': float_list_feature(xmaxs),\n        'image/object/bbox/ymin': float_list_feature(ymins),\n        'image/object/bbox/ymax': float_list_feature(ymaxs),\n        'image/object/class/text':bytes_list_feature(classes_text),\n        'image/object/class/label':int64_list_feature(classes),\n    }))\n    return tf_example\n\n\ndef main(csv_input, output_path, image_dir):\n    writer = tf.io.TFRecordWriter(output_path)\n    path = os.path.join(image_dir)\n    examples = pd.read_csv(csv_input)\n    grouped = split(examples, 'filename')\n    for group in grouped:\n        tf_example = create_tf_example(group, path)\n        writer.write(tf_example.SerializeToString())\n\n    writer.close()\n    output_path = os.path.join(os.getcwd(), output_path)\n    print('Successfully created the TFRecords: {}'.format(output_path))\n\n\nif __name__ == '__main__':\n    csv_input=\"train_labels.csv\"\n    output_path=\"train_label.record\"\n    image_dir=\"/kaggle/input/global-wheat-detection/train\"\n    main(csv_input, output_path, image_dir)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Now you can train efficientDet d0-d7**\n\nHere is the link of different efficientDet models pretrained weights on coco dataset\n\n\n\nEfficientDet-D0 \t[https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d0.tar.gz](http://)\n\nEfficientDet-D1 \t[https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d1.tar.gz](http://)\n\nEfficientDet-D2 \t[https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d2.tar.gz](http://)\n\nEfficientDet-D3* \t[https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d3_softnms.tar.gz](http://)\n\nEfficientDet-D4 \t[https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d4.tar.gz](http://)\n\n\nEfficientDet-D5 \t[https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d5.tar.gz](http://)\n\nEfficientDet-D6 \t[https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d6.tar.gz](http://)\n\nEfficientDet-D7*    [https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco/efficientdet-d7.tar.gz](http://)\n\n\n\nTo know more about efficientDet checkout the google automl repo on github \n\n[https://github.com/google/automl/tree/master/efficientdet](http://)","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cd \"/kaggle/input/kerasversionefficientdet\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir(\"/kaggle/working/model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pycocotools","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After creating tf record files you have to put your .record file in to efficientdet repo else it will throw an error no match pattern which i don't know why if someone knows why this error occurs do let me know in comment","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# uncomment for training\n# !python main.py --mode=train --training_file_pattern=train.record --model_name=efficientdet-d3 --model_dir=/kaggle/working/model --model_name=efficientdet-d3 --ckpt=/kaggle/input/effiecientdetd3-10k-epoch-checkpoints --train_batch_size=4 --num_epochs=3000 --num_examples_per_epoch=16","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"currently kaggle does not support tensorboard and i having trouble with showing losses on terminal because of tpu estimator.I have made the dataset which i created and repo publically so you can download repo with tfrecord and train it on google colab which supports tensorboard or locally if you have better gpu","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"for Easy inferencing checkout my kernel\n[https://www.kaggle.com/ravi02516/tensorflow-efficientdet-d3-with-default-parameters](http://)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}