{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport math\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nfrom google.cloud import storage\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PATHS TO IMAGES\nPATH = '../input/dog-breed-identification/train/'\nPATH2 = '../input/dog-breed-identification/test/'\nIMGS = os.listdir(PATH); IMGS2 = os.listdir(PATH2)\nprint('There are %i train images and %i test images'%(len(IMGS),len(IMGS2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOAD META DATA\ndf = pd.read_csv('../input/dog-breed-identification/sample_submission.csv')\ndf.rename({'id':'image_name'},axis=1,inplace=True)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = df.image_name.values\ny = df.image_name.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nAUTO = tf.data.experimental.AUTOTUNE\nfrom PIL import Image\nimport os\nimport IPython.display as display","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Transforming the Test Dataset into TFRecords for TPU usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytestring_feature(list_of_bytestrings):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))\n\ndef _int_feature(list_of_ints): # int64\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))\n\ndef _float_feature(list_of_floats): # float32\n    return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame()\ntest_df['image_path'] = X\ntest_df['image_name'] = y\ntest_df.head()\n\n\ntest_image_paths = test_df['image_path']\ntest_labels = test_df[['image_name']]\n\nos.makedirs('./tfrecords/test/')\n\ntfrecord_test_dir = './tfrecords/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SHARDS = 128\nnb_images = len(test_df)\nshard_size = math.ceil(1.0 * nb_images / SHARDS)\nprint(\"Pattern matches {} images which will be rewritten as {} .tfrec files containing {} images each.\".format(nb_images, SHARDS, shard_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_parse_function(filename, label):\n    img_raw = tf.io.read_file('../input/dog-breed-identification/test/' + filename + '.jpg')\n    return img_raw, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = tf.data.Dataset.from_tensor_slices((test_image_paths, test_labels))\ndataset = files.map(test_parse_function)\ndataset = dataset.batch(shard_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_tfrecord(tfrec_filewriter, img_bytes, label):\n    \n    feature = {\n        \"image\": _bytestring_feature([img_bytes]), # one image in the list\n        \"image_name\": _bytestring_feature([label[0]]),\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Writing TFRecords\")\nfor shard, (image, label) in enumerate(dataset):\n  # batch size used as shard size here\n  shard_size = image.numpy().shape[0]\n  # good practice to have the number of records in the filename\n  filename = tfrecord_test_dir + \"{:03d}-{}.tfrec\".format(shard, shard_size)\n  \n  with tf.io.TFRecordWriter(filename) as out_file:\n    for i in range(shard_size):\n        example = to_tfrecord(out_file,\n                              image.numpy()[i],\n                              label.numpy()[i])\n        out_file.write(example.SerializeToString())\n    \n    print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test the reading of TFRecords created"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [224,224]\nBATCH_SIZE = 128\n\ndef read_tfrecord(example):\n    features = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),  # tf.string = bytestring (not text string)\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),   # shape [] means scalar\n    }\n    \n    feature = tf.io.parse_single_example(example, features)\n    print(feature)\n    image = tf.image.decode_jpeg(feature['image'], channels=3)\n    image = tf.image.convert_image_dtype(image, tf.float32)\n    image = tf.image.resize(image, [*IMAGE_SIZE])\n    label = feature['image_name']\n    return image, label\n\n    \n# read from TFRecords. For optimal performance, read from multiple\n# TFRecord files at once and set the option experimental_deterministic = False\n# to allow order-altering optimizations.\n\noption_no_order = tf.data.Options()\noption_no_order.experimental_deterministic = False\n\ntest_path = tf.io.gfile.glob(tfrecord_test_dir+ \"*.tfrec\")\n\ntest_dataset = tf.data.TFRecordDataset(test_path, num_parallel_reads=AUTO)\ntest_dataset = test_dataset.with_options(option_no_order)\ntest_dataset = test_dataset.map(read_tfrecord, num_parallel_calls=AUTO)\ntest_dataset = test_dataset.batch(BATCH_SIZE)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for image, label in test_dataset.take(1):\n    print(image.numpy().shape)\n    print(label)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Google Cloud credentials"},{"metadata":{"trusted":true},"cell_type":"code","source":"from google.cloud import storage\n\n# For uploading to GCS buckets:\nSTORAGE_CLIENT = storage.Client.from_service_account_json('../input/cz4041/My Project 78884-3c1398ad9056.json')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_bucket(dataset_name):\n    \"\"\"Creates a new bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = STORAGE_CLIENT.create_bucket(dataset_name)\n    print('Bucket {} created'.format(bucket.name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bucket_name = 'cz4041_test'         \ntry:\n    create_bucket(bucket_name)   \nexcept:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def list_blobs(bucket_name):\n    \"\"\"Lists all the blobs in the bucket. https://cloud.google.com/storage/docs/\"\"\"\n    blobs = STORAGE_CLIENT.list_blobs(bucket_name)\n    for blob in blobs:\n        print(blob.name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def upload_blob(bucket_name, source_file_name, destination_blob_name):\n    \"\"\"Uploads a file to the bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = STORAGE_CLIENT.get_bucket(bucket_name)\n    blob = bucket.blob(destination_blob_name)\n    blob.upload_from_filename(source_file_name)\n    print('File {} uploaded to {}.'.format(\n        source_file_name,\n        destination_blob_name))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = os.listdir('./tfrecords/test')\nprint(test_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for file in test_files:\n    local_data = './tfrecords/test/'+file\n    file_name = file\n    upload_blob(bucket_name, local_data, file_name)\n\nprint('\\nData inside of the GCS Bucket ',bucket_name,':\\n')\nlist_blobs(bucket_name)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}