{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport glob\n\nimport argparse\nimport matplotlib.pyplot as plt\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-19T09:46:54.633561Z","iopub.execute_input":"2022-04-19T09:46:54.633916Z","iopub.status.idle":"2022-04-19T09:46:54.638341Z","shell.execute_reply.started":"2022-04-19T09:46:54.633883Z","shell.execute_reply":"2022-04-19T09:46:54.637543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:46:54.646788Z","iopub.execute_input":"2022-04-19T09:46:54.647130Z","iopub.status.idle":"2022-04-19T09:47:02.268298Z","shell.execute_reply.started":"2022-04-19T09:46:54.647086Z","shell.execute_reply":"2022-04-19T09:47:02.267502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    return image\n\ndef read_tfrecord(example):\n    tfrec_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'breed': tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['image'] = tf.image.decode_jpeg(example['image'], channels=3)\n    return example\n\n\ndef prepocess(example):\n    image = example['image']\n    image = tf.image.resize(image, size=[256, 256])\n    image = tf.cast(image, tf.float32)# / 255.0\n    return image, example['label']\n\ndata_augmentation = tf.keras.Sequential(\n[\n    tf.keras.layers.experimental.preprocessing.Resizing(300, 300),\n    tf.keras.layers.experimental.preprocessing.RandomZoom(0.5, 0.5),\n    tf.keras.layers.experimental.preprocessing.RandomCrop(256, 256),\n    tf.keras.layers.experimental.preprocessing.RandomRotation(0.5),\n    tf.keras.layers.experimental.preprocessing.RandomFlip(),\n    tf.keras.layers.experimental.preprocessing.RandomContrast(0.3),\n    tf.keras.layers.experimental.preprocessing.RandomTranslation(0.3, 0.3)\n]\n)\n\ndef prepocess_with_aug(example):\n    image = example['image']\n    \n    #image = tf.image.resize(image, size=[480, 480])\n    #image = tf.image.random_crop(image, [468, 468, 3]) # 2\n    #image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_brightness(image, max_delta=0.2)\n    #image = tf.image.random_contrast(image, lower=0.8, upper=1.2)\n    #image = tf.image.random_saturation(image, lower=0.75, upper=1.2)\n\n    image = tf.image.resize(image, size=[256, 256])\n    #image = tf.cast(image, tf.float32)# / 255.0\n    \n    image = tf.image.random_brightness(image, max_delta=0.2)\n    image = tf.image.random_saturation(image, lower=0.75, upper=1.2)\n    #image = data_augmentation(image)\n    image = tf.cast(image, tf.float32)\n    return image, example['label']","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:58:17.841303Z","iopub.execute_input":"2022-04-19T09:58:17.841587Z","iopub.status.idle":"2022-04-19T09:58:17.892734Z","shell.execute_reply.started":"2022-04-19T09:58:17.841559Z","shell.execute_reply":"2022-04-19T09:58:17.892095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n#GCS_DS_PATH = KaggleDatasets().get_gcs_path('dog-breed-identification')\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('dogbreedtfrec')\nGCS_DS_PATH","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:58:18.161435Z","iopub.execute_input":"2022-04-19T09:58:18.162331Z","iopub.status.idle":"2022-04-19T09:58:18.591597Z","shell.execute_reply.started":"2022-04-19T09:58:18.162293Z","shell.execute_reply":"2022-04-19T09:58:18.590888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\ntrain_tfrec = []\nvalid_tfrec = []\nfor i in [0,1,2,3]:\n    train_tfrec += tf.io.gfile.glob(os.path.join(GCS_DS_PATH, f'dogbreed_train_fold{i}_*.tfrec'))\nvalid_tfrec += tf.io.gfile.glob(os.path.join(GCS_DS_PATH, f'dogbreed_train_fold4_*.tfrec'))\n\ntrain_dataset = (tf.data.TFRecordDataset(train_tfrec, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n.map(read_tfrecord)\n.map(prepocess_with_aug)\n.cache()\n#.repeat()\n.shuffle(2048)\n.batch(BATCH_SIZE)\n.prefetch(AUTO))\n\nvalid_dataset = (tf.data.TFRecordDataset(valid_tfrec, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n.map(read_tfrecord)\n.map(prepocess)\n.batch(BATCH_SIZE)\n.cache()\n.prefetch(AUTO))","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:58:18.592943Z","iopub.execute_input":"2022-04-19T09:58:18.593506Z","iopub.status.idle":"2022-04-19T09:58:19.099591Z","shell.execute_reply.started":"2022-04-19T09:58:18.593467Z","shell.execute_reply":"2022-04-19T09:58:19.098750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model_base = tf.keras.applications.efficientnet.EfficientNetB5(\n        include_top=False,\n        weights='imagenet',\n        input_tensor=None,\n        input_shape=(256, 256, 3),\n        pooling='avg',\n        #classes=120,\n        #classifier_activation='softmax'\n    )\n    model = tf.keras.Sequential([\n        model_base,\n        tf.keras.layers.Dropout(0.2),\n        tf.keras.layers.Dense(120, activation=\"softmax\")\n    ])\n    model.compile(optimizer='adam',\n                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),\n                  metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n    model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:58:19.100990Z","iopub.execute_input":"2022-04-19T09:58:19.101637Z","iopub.status.idle":"2022-04-19T09:58:42.854622Z","shell.execute_reply.started":"2022-04-19T09:58:19.101604Z","shell.execute_reply":"2022-04-19T09:58:42.853804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nLR = 0.0002 # 0.0005\nEPOCHS = 12\nWARMUP = 4\n\ndef get_cosine_schedule_with_warmup(lr, num_warmup_steps, num_training_steps, num_cycles=0.5):\n    \"\"\"\n    Modified the get_cosine_schedule_with_warmup from huggingface for tensorflow\n    (https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup)\n\n    Create a schedule with a learning rate that decreases following the\n    values of the cosine function between 0 and `pi * cycles` after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return float(epoch) / float(max(1, num_warmup_steps)) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR, num_warmup_steps=WARMUP, num_training_steps=EPOCHS)\n\nhistory = model.fit(\n  train_dataset,\n  validation_data=valid_dataset,\n  epochs=EPOCHS,\n  callbacks=[lr_schedule]\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:58:42.856418Z","iopub.execute_input":"2022-04-19T09:58:42.856910Z","iopub.status.idle":"2022-04-19T10:03:41.798558Z","shell.execute_reply.started":"2022-04-19T09:58:42.856866Z","shell.execute_reply":"2022-04-19T10:03:41.797670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get training and test loss histories\ntraining_loss = history.history['loss']\ntest_loss = history.history['val_loss']\n\n# Create count of the number of epochs\nepoch_count = range(1, len(training_loss) + 1)\n\n# Visualize loss history\nplt.plot(epoch_count, training_loss, 'r--')\nplt.plot(epoch_count, test_loss, 'b-')\nplt.legend(['Training Loss', 'Validation Loss'])\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:03:41.799850Z","iopub.execute_input":"2022-04-19T10:03:41.800093Z","iopub.status.idle":"2022-04-19T10:03:42.039875Z","shell.execute_reply.started":"2022-04-19T10:03:41.800065Z","shell.execute_reply":"2022-04-19T10:03:42.039074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}