{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"## reference: https://www.tensorflow.org/tutorials/load_data/tfrecord\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-15T08:09:07.048991Z","iopub.execute_input":"2022-04-15T08:09:07.049307Z","iopub.status.idle":"2022-04-15T08:09:07.054547Z","shell.execute_reply.started":"2022-04-15T08:09:07.049274Z","shell.execute_reply":"2022-04-15T08:09:07.053598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\ndf = pd.read_csv('/kaggle/input/dog-breed-identification/labels.csv')\ndf['image_path'] = '/kaggle/input/dog-breed-identification/train/' + df['id'] + '.jpg'\ndf['label'] = df.groupby(['breed'], sort=True).ngroup()\n\n\nkf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2021)\nfor fold, (train_index, val_index) in enumerate(kf.split(df.values, df[\"breed\"])):\n    df.loc[val_index, \"fold\"] = int(fold)\ndf[\"fold\"] = df[\"fold\"].astype(int)\ndf['shard'] = np.random.randint(0, 5, (len(df)))\ndf","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:09:07.434229Z","iopub.execute_input":"2022-04-15T08:09:07.434648Z","iopub.status.idle":"2022-04-15T08:09:07.501074Z","shell.execute_reply.started":"2022-04-15T08:09:07.434617Z","shell.execute_reply":"2022-04-15T08:09:07.500096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Write","metadata":{}},{"cell_type":"code","source":"# The following functions can be used to convert a value to a type compatible\n# with tf.train.Example.\n\ndef _bytes_feature(value):\n    '''Returns a bytes_list from a string / byte.'''\n    if isinstance(value, type(tf.constant(0))):\n    #if isinstance(value, tf.python.framework.ops.EagerTensor):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    '''Returns a float_list from a float / double.'''\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    '''Returns an int64_list from a bool / enum / int / uint.'''\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:09:09.494393Z","iopub.execute_input":"2022-04-15T08:09:09.494655Z","iopub.status.idle":"2022-04-15T08:09:09.501230Z","shell.execute_reply.started":"2022-04-15T08:09:09.494630Z","shell.execute_reply":"2022-04-15T08:09:09.500258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(image,image_id,label,breed):\n    feature = {\n        'image': _bytes_feature(image),\n        'image_id': _bytes_feature(image_id),\n        'label': _int64_feature(label),\n        'breed': _bytes_feature(breed),\n      }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:09:10.077725Z","iopub.execute_input":"2022-04-15T08:09:10.078442Z","iopub.status.idle":"2022-04-15T08:09:10.084121Z","shell.execute_reply.started":"2022-04-15T08:09:10.078405Z","shell.execute_reply":"2022-04-15T08:09:10.083159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_tfrecord(df: pd.DataFrame, filename: str):\n    options = tf.io.TFRecordOptions(\"GZIP\")\n    with tf.io.TFRecordWriter(filename, options=options) as writer:\n        for i in tqdm(range(len(df))):\n            image = tf.io.read_file(df.iloc[i]['image_path'])\n            image_id = str.encode(df.iloc[i]['id'])\n            label = df.iloc[i]['label']\n            breed = str.encode(df.iloc[i]['breed'])\n            #wave = np.load(npy_path).tobytes()\n            \n            tf_example = serialize_example(image,image_id,label,breed)\n            writer.write(tf_example)","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:09:10.911482Z","iopub.execute_input":"2022-04-15T08:09:10.911759Z","iopub.status.idle":"2022-04-15T08:09:10.918069Z","shell.execute_reply.started":"2022-04-15T08:09:10.911723Z","shell.execute_reply":"2022-04-15T08:09:10.917210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n#with ProcessPoolExecutor(max_workers=8) as executor:\n#    futures = [executor.submit(write_tfrecord, df[df['fold']==i], f'dogbreed_train_fold{i}.tfrec') for i in tqdm(range(5))]\nimport joblib\n_ = joblib.Parallel(n_jobs=8)(\n        joblib.delayed(write_tfrecord)(df[(df['fold']==i) & (df['shard']==j)], f'dogbreed_train_fold{i}_{j}.tfrec') for i in range(5) for j in range(5)\n    )\n\n#for i in range(5):\n#    write_tfrecord(df[df['fold']==i], f'dogbreed_train_fold{i}.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:09:17.701519Z","iopub.execute_input":"2022-04-15T08:09:17.702147Z","iopub.status.idle":"2022-04-15T08:09:44.132937Z","shell.execute_reply.started":"2022-04-15T08:09:17.702108Z","shell.execute_reply":"2022-04-15T08:09:44.131977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read ","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    return image\n\ndef read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'breed': tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    image = decode_image(example['image'])\n    label = example['label']\n    return image, label\n\ndef preprocess(image, label):\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.image.resize(image,[256, 256])\n    return image, label","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:16:19.297711Z","iopub.execute_input":"2022-04-15T08:16:19.298438Z","iopub.status.idle":"2022-04-15T08:16:19.310353Z","shell.execute_reply.started":"2022-04-15T08:16:19.298381Z","shell.execute_reply":"2022-04-15T08:16:19.309354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport matplotlib.pyplot as plt\n\nfiles = glob.glob('dogbreed_train_fold*.tfrec')\nAUTO = tf.data.experimental.AUTOTUNE\ndataset = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\ndataset = dataset.map(read_labeled_tfrecord)\ndataset = dataset.map(preprocess)\n\nfor d in dataset.take(5):\n    image = d[0].numpy()\n    plt.imshow(image)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:16:19.719568Z","iopub.execute_input":"2022-04-15T08:16:19.720238Z","iopub.status.idle":"2022-04-15T08:16:21.251742Z","shell.execute_reply.started":"2022-04-15T08:16:19.720184Z","shell.execute_reply":"2022-04-15T08:16:21.250876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}