{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport tensorflow as tf\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-05T16:20:57.634459Z","iopub.execute_input":"2022-05-05T16:20:57.635274Z","iopub.status.idle":"2022-05-05T16:21:02.439053Z","shell.execute_reply.started":"2022-05-05T16:20:57.635131Z","shell.execute_reply":"2022-05-05T16:21:02.43817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [256, 256]\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    target = decode_image(example['target'])\n    image_name = decode_image(example['image_name'])\n#     print(image_name)\n#     print(\"----------------\")\n#     print(image)\n#     print(\"----------------\")\n#     print(target)\n#     print(\"----------------\")\n    print(target)\n    return image\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return dataset\nMONET_FILENAMES =  'gs://kds-27633b93744d213c4ff0cd297e0db1f03205ab2f9857a8f031ede014/photo_tfrec/photo00-352.tfrec'\nMONET_FILENAMES =  '../input/double-city-256/test.tfrec'\n\n\nmonet_ds = load_dataset(MONET_FILENAMES, labeled=True).batch(1)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:02.440833Z","iopub.execute_input":"2022-05-05T16:21:02.441101Z","iopub.status.idle":"2022-05-05T16:21:02.84258Z","shell.execute_reply.started":"2022-05-05T16:21:02.441066Z","shell.execute_reply":"2022-05-05T16:21:02.841648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(next(iter(monet_ds))[0] * 0.5 + 0.5)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:31.811587Z","iopub.execute_input":"2022-05-05T16:21:31.812303Z","iopub.status.idle":"2022-05-05T16:21:32.134421Z","shell.execute_reply.started":"2022-05-05T16:21:31.81224Z","shell.execute_reply":"2022-05-05T16:21:32.133812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:02.977197Z","iopub.execute_input":"2022-05-05T16:21:02.977414Z","iopub.status.idle":"2022-05-05T16:21:03.011657Z","shell.execute_reply.started":"2022-05-05T16:21:02.977388Z","shell.execute_reply":"2022-05-05T16:21:03.010825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = '../input/double-city/test.tfrec'\ntest_ds = load_dataset(test_path, labeled=True).batch(1)\nnext(iter(test_ds))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:03.012901Z","iopub.execute_input":"2022-05-05T16:21:03.013186Z","iopub.status.idle":"2022-05-05T16:21:03.414856Z","shell.execute_reply.started":"2022-05-05T16:21:03.013128Z","shell.execute_reply":"2022-05-05T16:21:03.413677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (next(iter(photo_ds)))\n# next(iter(photo_ds))[0]\n# example_photo = next(next(all_photo))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:03.415972Z","iopub.status.idle":"2022-05-05T16:21:03.416652Z","shell.execute_reply.started":"2022-05-05T16:21:03.416363Z","shell.execute_reply":"2022-05-05T16:21:03.416391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n    \nprint(tf.__version__)\nMONET_FILENAMES = '../input/double-city-256/train.tfrec'\nPHOTO_FILENAMES = '../input/double-city-256/test.tfrec'\nIMAGE_SIZE = [256, 256]\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    target = decode_image(example['target'])\n    image_name = decode_image(example['image_name'])\n    print(image_name)\n    print(\"----------------\")\n    print(image)\n    print(\"----------------\")\n    print(target)\n    print(\"----------------\")\n    return image\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return dataset\n\nmonet_ds = load_dataset(MONET_FILENAMES, labeled=True).batch(1)\nphoto_ds = load_dataset(PHOTO_FILENAMES, labeled=True).batch(1)\n# ParallelMapDataset 多线程读取一条数据\ntest = load_dataset(MONET_FILENAMES, labeled=True)\ntest # <ParallelMapDataset shapes: (256, 256, 3), types: tf.float32>\n# batch will take the first batch_size entries and make a batch out of them.\n# batch = 1 也就是为下面每次递归读取一张图片\nmonet_ds # <BatchDataset shapes: (None, 256, 256, 3), types: tf.float32>","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:03.418426Z","iopub.status.idle":"2022-05-05T16:21:03.419304Z","shell.execute_reply.started":"2022-05-05T16:21:03.419006Z","shell.execute_reply":"2022-05-05T16:21:03.419043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ParallelMapDataset 多线程读取一条数据\ntest = load_dataset(MONET_FILENAMES, labeled=True)\ntest # <ParallelMapDataset shapes: (256, 256, 3), types: tf.float32>\n# batch will take the first batch_size entries and make a batch out of them.\n# batch = 1 也就是为下面每次递归读取一张图片\nmonet_ds # <BatchDataset shapes: (None, 256, 256, 3), types: tf.float32>","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:03.420541Z","iopub.status.idle":"2022-05-05T16:21:03.421524Z","shell.execute_reply.started":"2022-05-05T16:21:03.421258Z","shell.execute_reply":"2022-05-05T16:21:03.421285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_monet = next(iter(monet_ds))\nexample_photo = next(iter(photo_ds))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T16:21:03.422599Z","iopub.status.idle":"2022-05-05T16:21:03.423556Z","shell.execute_reply.started":"2022-05-05T16:21:03.423307Z","shell.execute_reply":"2022-05-05T16:21:03.423332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}