{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import os\nimport skimage.io\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pandas as pd\nimport numpy as np\nimport shutil\nimport cv2\nimport math","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef xy_visualization(x, y = None):\n    plt.imshow(x)\n    plt.axis('off')\n    plt.show()\n    if y is not None:\n        print(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/global-wheat-detection/train.csv') \ndf['bbox'] = df['bbox'].apply(lambda x: x[1:-1].split(\",\")) \ndf['x'] = df['bbox'].apply(lambda x: x[0]).astype('float32').astype('int32') \ndf['y'] = df['bbox'].apply(lambda x: x[1]).astype('float32').astype('int32') \ndf['w'] = df['bbox'].apply(lambda x: x[2]).astype('float32').astype('int32') \ndf['h'] = df['bbox'].apply(lambda x: x[3]).astype('float32').astype('int32') \ndf = df[['image_id','x', 'y', 'w', 'h']]\n\ncache = {}\nfor indx, row in tqdm(df.iterrows()):\n    image_id, x, y, w, h = row['image_id'], row['x'], row['y'], row['w'], row['h']\n    x1, y1, x2, y2 = x, y, x + w, y + h\n    if image_id not in cache:\n        cache[image_id] = {\n            \"x\" : '../input/global-wheat-detection/train/' + image_id + '.jpg',\n            \"y\": np.zeros((120, 5), np.float32),\n            'y_cnt': 0\n        }\n        cache[image_id][\"y\"][:, 4] = -1 # Set all classes_idx to -1 \n    cache[image_id][\"y\"][cache[image_id][\"y_cnt\"]] = [x1, y1, x2, y2, 0]\n    cache[image_id][\"y_cnt\"] += 1\n    \ncache = [i for i in cache.values()]\n# cv2.cvtColor(cv2.imread('../input/global-wheat-detection/train/' + image_id + '.jpg'), cv2.COLOR_BGR2RGB),","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"n_max = 0\nimage_id = ''\nfor item in cache:\n    if(n_max < item['y_cnt']):\n        n_max = item['y_cnt']\n        image_id = item['x']\nprint(n_max, image_id)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"for item in cache:\n    xy_visualization(cv2.cvtColor(cv2.imread(item[\"x\"]), cv2.COLOR_BGR2RGB), item[\"y\"][:item[\"y_cnt\"]])","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"print((len(cache)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport os\nimport random\nfrom tqdm import tqdm\n\nclass encode_and_write:\n    def __init__(self):\n        self.feature_dict = {\n            'ndarray' : self._ndarray_feature, \n            'bytes' : self._bytes_feature, \n            'float' : self._float_feature,\n            'double' : self._float_feature, \n            'bool' : self._int64_feature,\n            'enum' : self._int64_feature, \n            'int' : self._int64_feature,\n            'uint8' : self._int64_feature,\n            'int32' : self._int64_feature,\n            'uint' : self._int64_feature\n        }\n    def _ndarray_feature(self, value):\n        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value.tobytes()]))\n    \n    def _bytes_feature(self, value):\n        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n    def _float_feature(self, value):\n        return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n    def _int64_feature(self, value):\n        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n    \n    def _encode_example(self, example):\n        \"\"\"Creates a tf.Example message ready to be written to a file.\"\"\"\n        feature = {}\n        for vname in example:\n            vtype = type(example[vname]).__name__\n            feature[vname] = self.feature_dict[vtype](example[vname])\n        # Create a Features message using tf.train.Example.\n        example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n        return example_proto.SerializeToString()\n\n    def run(self, filename, datasets, split_thresold=1000):\n        datasets_itor = iter(datasets)\n        \n        try:\n            for file_idx in range(100000000):\n                pre_read = self._encode_example(datasets_itor.__next__())\n                with tf.io.TFRecordWriter('%s_%d.tfrecord'%(filename, file_idx)) as writer:\n                    for k in tqdm(range(split_thresold)):\n                        if pre_read is not None:\n                            writer.write(pre_read)\n                            pre_read = None\n                        else:\n                            writer.write(self._encode_example(datasets_itor.__next__()))\n        except StopIteration:\n            pass\n                    \n  \nclass datasets_stream:\n    def __init__(self, group_item_cnt, group_idx):\n        self.group_item_cnt = group_item_cnt\n        self.group_idx = group_idx\n        \n    def __iter__(self):\n        np.random.seed(0)\n        self.imgs_idxs = np.random.permutation([i for i in range(len(cache))])[self.group_item_cnt * self.group_idx: self.group_item_cnt * (self.group_idx + 1)]\n        self.imgs_idx = 0\n        return self\n\n    def __next__(self):\n        while True:\n            if self.imgs_idx < len(self.imgs_idxs):\n                idx = self.imgs_idxs[self.imgs_idx]\n                self.imgs_idx += 1\n                x, y = cv2.cvtColor(cv2.imread(cache[idx][\"x\"]), cv2.COLOR_BGR2RGB), cache[idx][\"y\"]\n                return {\"x\": x, \"y\": y }\n            else:\n                raise StopIteration\n\n            \nencode_and_write().run('./train', datasets_stream(675, 0), split_thresold=64)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}