{"cells":[{"metadata":{},"cell_type":"markdown","source":"#### Code based on this Chris Deotte kernel [How To Create TFRecords](https://www.kaggle.com/cdeotte/how-to-create-tfrecords)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Dependencies","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np, pandas as pd, os\nimport matplotlib.pyplot as plt, cv2\nimport tensorflow as tf, re, math","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Auxiliar functions","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def decode_image(image, HEIGHT, WIDTH, CHANNELS):\n    image = tf.image.decode_jpeg(image, channels=CHANNELS)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])\n    return image\n\ndef read_tfrecord(example, HEIGHT, WIDTH, CHANNELS):\n    tfrecord_format = {\n        'image':      tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'], HEIGHT, WIDTH, CHANNELS)\n    return image\n\ndef load_dataset(filenames, HEIGHT, WIDTH, CHANNELS=3):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(lambda example: read_tfrecord(example, HEIGHT, WIDTH, CHANNELS), num_parallel_calls=AUTO)      \n    return dataset\n\ndef display_samples(ds, row, col):\n    ds_iter = iter(ds)\n    plt.figure(figsize=(15, int(15*row/col)))\n    for j in range(row*col):\n        example_sample = next(ds_iter)\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(example_sample[0] * 0.5 + 0.5)\n    plt.show()\n\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n# Create TF Records\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef serialize_example(image):\n  feature = {\n      'image': _bytes_feature(image),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"PATH = '/kaggle/input/monet-paintings-jpg-berkeley/'\nIMGS = os.listdir(PATH)\nSIZE = (len(IMGS) // 5) + 1 # split images into 5 files\nIMAGE_SIZE = [256, 256]\n\nprint(f'Image samples: {len(IMGS)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generate TF records","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"CT = len(IMGS)//SIZE + int(len(IMGS)%SIZE!=0)\nfor j in range(CT):\n    print(); print('Writing TFRecord %i of %i...'%(j,CT))\n    CT2 = min(SIZE,len(IMGS)-j*SIZE)\n    with tf.io.TFRecordWriter('monet%.2i-%i.tfrec'%(j,CT2)) as writer:\n        for k in range(CT2):\n            img = cv2.imread(PATH+IMGS[SIZE*j+k])\n            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n            name = IMGS[SIZE*j+k].split('.')[0]\n            example = serialize_example(img)\n            writer.write(example)\n            if k%100==0: print(k,', ',end='')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validate created TF records","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nFILENAMES = tf.io.gfile.glob('monet*.tfrec')\nprint(f'TFRecords files: {FILENAMES}')\nprint(f'Created image samples: {count_data_items(FILENAMES)}')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_samples(load_dataset(FILENAMES, *IMAGE_SIZE).batch(1), 6, 6)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}