{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport math\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nfrom google.cloud import storage\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PATHS TO IMAGES\nPATH = '../input/dog-breed-identification/train/'\nPATH2 = '../input/dog-breed-identification/test/'\nIMGS = os.listdir(PATH); IMGS2 = os.listdir(PATH2)\nprint('There are %i train images and %i test images'%(len(IMGS),len(IMGS2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOAD META DATA\ndf = pd.read_csv('../input/dog-breed-identification/labels.csv')\ndf.rename({'id':'image_name'},axis=1,inplace=True)\ndf.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = pd.DataFrame(df['breed'].value_counts())\nx.astype('int64').dtypes\nx.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = x.plot.bar(figsize=(20,8),y='breed', rot=90)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Split the Kaggle Training data into Training and validation datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(df.image_name.values, df.breed.values, test_size=0.10, random_state=42, stratify=df[['breed']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64\nSTEPS_PER_EPOCH = len(X_train) // BATCH_SIZE\nVAL_STEPS_PER_EPOCH = len(X_test) // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VAL_STEPS_PER_EPOCH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.DataFrame()\ntrain_df['image_name'] = X_train\ntrain_df['breed'] = y_train\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = pd.DataFrame(train_df['breed'].value_counts())\nx.astype('int64').dtypes\nax = x.plot.bar(figsize=(17,6),y='breed', rot=90)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame()\ntest_df['image_name'] = X_test\ntest_df['breed'] = y_test\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = pd.DataFrame(test_df['breed'].value_counts())\nx.astype('int64').dtypes\nax = x.plot.bar(figsize=(17,6),y='breed', rot=90)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nAUTO = tf.data.experimental.AUTOTUNE\nfrom PIL import Image\nimport os\nimport IPython.display as display","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytestring_feature(list_of_bytestrings):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))\n\ndef _int_feature(list_of_ints): # int64\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))\n\ndef _float_feature(list_of_floats): # float32\n    return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Encode the Dog Breed Classes into integer classes for classification"},{"metadata":{"trusted":true},"cell_type":"code","source":"label_encoder = LabelEncoder().fit(df.breed.astype(str))\ntrain_df.breed = label_encoder.transform(train_df.breed.astype(str))\nkeys = label_encoder.classes_\nvalues = label_encoder.transform(label_encoder.classes_)\ndictionary = dict(zip(keys, values))\nlabel_encoder = LabelEncoder().fit(df.breed.astype(str))\ntest_df.breed = label_encoder.transform(test_df.breed.astype(str))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.breed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The dictionary of the 120 Dog Breeds and their respective numerical classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"dictionary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import csv\n\ncsv_columns = dictionary.keys() \ndict_data = [dictionary]\n\ncsv_file = \"./classes_mapping.csv\"\ntry:\n    with open(csv_file, 'w') as csvfile:\n        writer = csv.DictWriter(csvfile, fieldnames=csv_columns)\n        writer.writeheader()\n        for data in dict_data:\n            writer.writerow(data)\nexcept IOError:\n    print(\"I/O error\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_paths = train_df['image_name']\ntrain_labels = train_df[['breed']]\n\nval_image_paths = test_df['image_name']\nval_labels = test_df[['breed']]\n\nos.makedirs('./tfrecords/train/')\nos.makedirs('./tfrecords/val/')\n\ntfrecord_train_dir = './tfrecords/train/'\ntfrecord_val_dir = './tfrecords/val/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Data"},{"metadata":{},"cell_type":"markdown","source":"Transforming the Training Dataset into TFRecords for TPU usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"SHARDS = 144\nnb_images = len(train_df)\nshard_size = math.ceil(1.0 * nb_images / SHARDS)\nprint(\"Pattern matches {} images which will be rewritten as {} .tfrec files containing {} images each.\".format(nb_images, SHARDS, shard_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_parse_function(filename, label):\n    print(label)\n    img_raw = tf.io.read_file('../input/dog-breed-identification/train/' + filename + '.jpg')\n    return img_raw, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = tf.data.Dataset.from_tensor_slices((train_image_paths, train_labels))\ndataset = files.map(train_parse_function)\ndataset = dataset.batch(shard_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_tfrecord(tfrec_filewriter, img_bytes, label):\n    one_hot_class = [np.eye(120)[label[0]]]\n    \n    feature = {\n        \"image\": _bytestring_feature([img_bytes]), # one image in the list\n        \"breed\": _int_feature([label[0]]),\n        \"breed_oh\": _float_feature(one_hot_class[0].tolist())\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Writing TFRecords\")\nfor shard, (image, label) in enumerate(dataset):\n  # batch size used as shard size here\n  shard_size = image.numpy().shape[0]\n  # good practice to have the number of records in the filename\n  filename = tfrecord_train_dir + \"{:02d}-{}.tfrec\".format(shard, shard_size)\n  \n  with tf.io.TFRecordWriter(filename) as out_file:\n    for i in range(shard_size):\n        example = to_tfrecord(out_file,\n                              image.numpy()[i],\n                              label.numpy()[i])\n        out_file.write(example.SerializeToString())\n    \n    print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Data"},{"metadata":{},"cell_type":"markdown","source":"Transforming the Validation Dataset into TFRecords for TPU usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"SHARDS = 16\n\nnb_images = len(test_df)\nshard_size = math.ceil(1.0 * nb_images / SHARDS)\nprint(\"Pattern matches {} images which will be rewritten as {} .tfrec files containing {} images each.\".format(nb_images, SHARDS, shard_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = tf.data.Dataset.from_tensor_slices((val_image_paths, val_labels))\ndataset = files.map(train_parse_function)\ndataset = dataset.batch(shard_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Writing TFRecords\")\nfor shard, (image, label) in enumerate(dataset):\n  # batch size used as shard size here\n  shard_size = image.numpy().shape[0]\n  # good practice to have the number of records in the filename\n  filename = tfrecord_val_dir + \"{:02d}-{}.tfrec\".format(shard, shard_size)\n  \n  with tf.io.TFRecordWriter(filename) as out_file:\n    for i in range(shard_size):\n        example = to_tfrecord(out_file,\n                              image.numpy()[i],\n                              label.numpy()[i])\n        out_file.write(example.SerializeToString())\n    \n    print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test TFRecords Reading"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [224,224]\nBATCH_SIZE = 128\n\ndef read_tfrecord(example):\n    features = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),  # tf.string = bytestring (not text string)\n        \"breed\": tf.io.FixedLenFeature([], tf.int64),   # shape [] means scalar\n        \"breed_oh\": tf.io.VarLenFeature(tf.float32) # a certain number of floats\n    }\n    \n    feature = tf.io.parse_single_example(example, features)\n    print(feature)\n    image = tf.image.decode_jpeg(feature['image'], channels=3)\n    image = tf.image.convert_image_dtype(image, tf.float32)\n    image = tf.image.resize(image, [*IMAGE_SIZE])\n    label = feature['breed']\n    one_hot_class = tf.sparse.to_dense(feature['breed_oh'])\n    one_hot_class = tf.reshape(one_hot_class, [120])\n    return image, label, one_hot_class\n\n    \n# read from TFRecords. For optimal performance, read from multiple\n# TFRecord files at once and set the option experimental_deterministic = False\n# to allow order-altering optimizations.\n\noption_no_order = tf.data.Options()\noption_no_order.experimental_deterministic = False\n\ntrain_path = tf.io.gfile.glob(tfrecord_train_dir+ \"*.tfrec\")\nval_path = tf.io.gfile.glob(tfrecord_val_dir + \"*.tfrec\")\n\ntraining_dataset = tf.data.TFRecordDataset(train_path, num_parallel_reads=AUTO)\ntraining_dataset = training_dataset.with_options(option_no_order)\ntraining_dataset = training_dataset.map(read_tfrecord, num_parallel_calls=AUTO)\ntraining_dataset = training_dataset.batch(BATCH_SIZE)\n\nval_dataset = tf.data.TFRecordDataset(val_path, num_parallel_reads=AUTO)\nval_dataset = val_dataset.with_options(option_no_order)\nval_dataset = val_dataset.map(read_tfrecord, num_parallel_calls=AUTO)\nval_dataset = val_dataset.batch(BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for image, label,one_hot_class in training_dataset.take(1):\n    print(image.numpy().shape)\n    print(label)\n    print(one_hot_class.numpy().shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Upload to GS Bucket"},{"metadata":{},"cell_type":"markdown","source":"Google Cloud credentials"},{"metadata":{"trusted":true},"cell_type":"code","source":"from google.cloud import storage\n\n# For uploading to GCS buckets:\nSTORAGE_CLIENT = storage.Client.from_service_account_json('../input/cz4041/My Project 78884-3c1398ad9056.json')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_bucket(dataset_name):\n    \"\"\"Creates a new bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = STORAGE_CLIENT.create_bucket(dataset_name)\n    print('Bucket {} created'.format(bucket.name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bucket_name = 'cz4041_train_10'         \ntry:\n    create_bucket(bucket_name)   \nexcept:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def list_blobs(bucket_name):\n    \"\"\"Lists all the blobs in the bucket. https://cloud.google.com/storage/docs/\"\"\"\n    blobs = STORAGE_CLIENT.list_blobs(bucket_name)\n    for blob in blobs:\n        print(blob.name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def upload_blob(bucket_name, source_file_name, destination_blob_name):\n    \"\"\"Uploads a file to the bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = STORAGE_CLIENT.get_bucket(bucket_name)\n    blob = bucket.blob(destination_blob_name)\n    blob.upload_from_filename(source_file_name)\n    print('File {} uploaded to {}.'.format(\n        source_file_name,\n        destination_blob_name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = os.listdir('./tfrecords/train')\nprint(train_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for file in train_files:\n    local_data = './tfrecords/train/'+file\n    file_name = file\n    upload_blob(bucket_name, local_data, file_name)\n\nprint('\\nData inside of the GCS Bucket ',bucket_name,':\\n')\nlist_blobs(bucket_name)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = os.listdir('./tfrecords/val')\nprint(test_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bucket_name = 'cz4041_val_10'         \ntry:\n    create_bucket(bucket_name)   \nexcept:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for file in test_files:\n    local_data = './tfrecords/val/'+file\n    file_name = file\n    upload_blob(bucket_name, local_data, file_name)\n\nprint('\\nData inside of the GCS Bucket ',bucket_name,':\\n')\nlist_blobs(bucket_name)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def download_to_kaggle(bucket_name,destination_directory,file_name):\n    \"\"\"Takes the data from your GCS Bucket and puts it into the working directory of your Kaggle notebook\"\"\"\n    os.makedirs(destination_directory, exist_ok = True)\n    full_file_path = os.path.join(destination_directory, file_name)\n    blobs = STORAGE_CLIENT.list_blobs(bucket_name)\n    for blob in blobs:\n        blob.download_to_filename(full_file_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"destination_directory = './from/'       \nfor file_name in test_files:\n    download_to_kaggle(bucket_name,destination_directory,file_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}