{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Resource","metadata":{"id":"OCfY2c5_PM90"}},{"cell_type":"markdown","source":"* original code: [Two-objective discriminator](https://www.kaggle.com/code/unfriendlyai/two-objective-discriminator#CycleGAN-with-DiffAugment-and-two-objective-discriminator)\n* cycle gan from scratch: [cycleGAN_tutorial](https://www.kaggle.com/code/songseungwon/cyclegan-tutorial-from-scratch-monet-to-photo)\n* related works:\n1. [Dual Discriminator Generative Adversarial Nets](https://arxiv.org/pdf/1709.03831.pdf)\n1. [Multi-Generator Generative Adversarial Nets](https://arxiv.org/pdf/1708.02556.pdf)\n1. [Original CycleGAN paper](https://arxiv.org/pdf/1703.10593.pdf)\n1. [Differentiable Augmentation for Data-Efficient GAN Training](https://arxiv.org/pdf/2006.10738.pdf)","metadata":{"id":"T5TyhwGqPP6i"}},{"cell_type":"markdown","source":"# Import library and dataset","metadata":{"id":"eWO87iMnK96S"}},{"cell_type":"code","source":"!pip install tensorflow_addons","metadata":{"id":"PFPcHKfb6B0y","outputId":"6e457ad6-fdc4-4da2-d1b7-fef245dd6301","execution":{"iopub.status.busy":"2022-09-22T17:57:09.610343Z","iopub.execute_input":"2022-09-22T17:57:09.610797Z","iopub.status.idle":"2022-09-22T17:57:23.190679Z","shell.execute_reply.started":"2022-09-22T17:57:09.610703Z","shell.execute_reply":"2022-09-22T17:57:23.188896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport re\nfrom datetime import datetime\nimport pandas as pd\ntry:\n    from kaggle_datasets import KaggleDatasets\nexcept:\n    pass","metadata":{"id":"9mgRQAfw6Da7","execution":{"iopub.status.busy":"2022-09-22T17:57:23.194700Z","iopub.execute_input":"2022-09-22T17:57:23.195258Z","iopub.status.idle":"2022-09-22T17:57:30.900744Z","shell.execute_reply.started":"2022-09-22T17:57:23.195199Z","shell.execute_reply":"2022-09-22T17:57:30.898708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%xmode Verbose","metadata":{"id":"7mDaxnmDTA23","outputId":"ddd50e9d-be22-4cf9-d6b8-71c66c717e40","execution":{"iopub.status.busy":"2022-09-22T17:57:30.902709Z","iopub.execute_input":"2022-09-22T17:57:30.903705Z","iopub.status.idle":"2022-09-22T17:57:30.910906Z","shell.execute_reply.started":"2022-09-22T17:57:30.903661Z","shell.execute_reply":"2022-09-22T17:57:30.909287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set up TPU","metadata":{"id":"o-m2SePbLD_a"}},{"cell_type":"code","source":"try:\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n  print('Device:', tpu.master())\n  tf.config.experimental_connect_to_cluster(tpu)\n  tf.tpu.experimental.initialize_tpu_system(tpu)\n  strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n  strategy = tf.distribute.get_strategy()\nprint('Number of replicas: ', strategy.num_replicas_in_sync)\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nprint(tf.__version__)","metadata":{"id":"1aVrqaNX6E6i","outputId":"35411746-b1d1-40dd-d6cd-88e4e4e58ef8","execution":{"iopub.status.busy":"2022-09-22T17:57:30.916283Z","iopub.execute_input":"2022-09-22T17:57:30.916940Z","iopub.status.idle":"2022-09-22T17:57:30.951416Z","shell.execute_reply.started":"2022-09-22T17:57:30.916899Z","shell.execute_reply":"2022-09-22T17:57:30.950050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set up dataloader","metadata":{"id":"dGmo2T2JLVhd"}},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path(\"gan-getting-started\")\nMONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))\nPHOTO_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))","metadata":{"id":"jM8IroxS6oUb","execution":{"iopub.status.busy":"2022-09-22T17:57:30.955520Z","iopub.execute_input":"2022-09-22T17:57:30.956401Z","iopub.status.idle":"2022-09-22T17:57:34.753796Z","shell.execute_reply.started":"2022-09-22T17:57:30.956347Z","shell.execute_reply":"2022-09-22T17:57:34.752627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [256, 256]\n\ndef decode_image(image):\n  image = tf.image.decode_jpeg(image, channels=3)\n  image = (tf.cast(image, tf.float32) / 127.5) - 1\n  image = tf.reshape(image, [*IMAGE_SIZE, 3])\n  return image\n\ndef read_tfrecord(example):\n  tfrecord_format = {\n      'image': tf.io.FixedLenFeature([], tf.string)\n  }\n  example = tf.io.parse_single_example(example, tfrecord_format)\n  image = decode_image(example['image'])\n  return image","metadata":{"id":"kq56Hyb_7jUt","execution":{"iopub.status.busy":"2022-09-22T17:57:34.755264Z","iopub.execute_input":"2022-09-22T17:57:34.755836Z","iopub.status.idle":"2022-09-22T17:57:34.764174Z","shell.execute_reply.started":"2022-09-22T17:57:34.755801Z","shell.execute_reply":"2022-09-22T17:57:34.762206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames):\n  dataset = tf.data.TFRecordDataset(filenames)\n  dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n  return dataset\n\nmonet_ds = load_dataset(MONET_FILENAMES).batch(1)\nphoto_ds = load_dataset(PHOTO_FILENAMES).batch(1)\n\nfast_photo_ds = load_dataset(PHOTO_FILENAMES).batch(32*strategy.num_replicas_in_sync).prefetch(32)\nfid_photo_ds = load_dataset(PHOTO_FILENAMES).take(1024).batch(32*strategy.num_replicas_in_sync).prefetch(32)\nfid_monet_ds = load_dataset(MONET_FILENAMES).batch(32*strategy.num_replicas_in_sync).prefetch(32)","metadata":{"id":"73XSB9lI-xbw","execution":{"iopub.status.busy":"2022-09-22T17:57:34.766016Z","iopub.execute_input":"2022-09-22T17:57:34.766592Z","iopub.status.idle":"2022-09-22T17:57:35.310731Z","shell.execute_reply.started":"2022-09-22T17:57:34.766552Z","shell.execute_reply":"2022-09-22T17:57:35.309196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_gan_dataset(monet_files, photo_files, augment=None, repeat=True, shuffle=True, batch_size=1):\n  monet_ds = load_dataset(monet_files)\n  photo_ds = load_dataset(photo_files)\n\n  if augment:\n    monet_ds = monet_ds.map(augment, num_parallel_calls=AUTOTUNE)\n    photo_ds = photo_ds.map(augment, num_parallel_calls=AUTOTUNE)\n\n  if repeat:\n    monet_ds = monet_ds.repeat()\n    photo_ds = photo_ds.repeat()\n\n  if shuffle:\n    monet_ds = monet_ds.shuffle(2048)\n    photo_ds = photo_ds.shuffle(2048)\n  \n  monet_ds = monet_ds.batch(batch_size, drop_remainder=True)\n  photo_ds = photo_ds.batch(batch_size, drop_remainder=True)\n# monet_ds = monet_ds.cache()\n# photo_ds = photo_ds.cache()\n  monet_ds = monet_ds.prefetch(AUTOTUNE)\n  photo_ds = photo_ds.prefetch(AUTOTUNE)\n\n  gan_ds = tf.data.Dataset.zip((monet_ds, photo_ds))\n  return gan_ds","metadata":{"id":"GtD6Nt4RNSCb","execution":{"iopub.status.busy":"2022-09-22T17:57:35.312307Z","iopub.execute_input":"2022-09-22T17:57:35.312677Z","iopub.status.idle":"2022-09-22T17:57:35.323764Z","shell.execute_reply.started":"2022-09-22T17:57:35.312646Z","shell.execute_reply":"2022-09-22T17:57:35.322124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fid_photo_ds","metadata":{"id":"lkvsd9dFNGaT","outputId":"18fde8c8-390f-4847-d2ed-25029794234b","execution":{"iopub.status.busy":"2022-09-22T17:57:35.325237Z","iopub.execute_input":"2022-09-22T17:57:35.325906Z","iopub.status.idle":"2022-09-22T17:57:35.339349Z","shell.execute_reply.started":"2022-09-22T17:57:35.325869Z","shell.execute_reply":"2022-09-22T17:57:35.338126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_dataset = get_gan_dataset(MONET_FILENAMES, PHOTO_FILENAMES, augment=None, repeat=True, shuffle=True, batch_size=BATCH_SIZE)","metadata":{"id":"V2mOHHvzSqWs","execution":{"iopub.status.busy":"2022-09-22T17:57:35.341262Z","iopub.execute_input":"2022-09-22T17:57:35.342020Z","iopub.status.idle":"2022-09-22T17:57:35.401410Z","shell.execute_reply.started":"2022-09-22T17:57:35.341971Z","shell.execute_reply":"2022-09-22T17:57:35.400423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build FID evaluator","metadata":{"id":"zvDregNnLZjC"}},{"cell_type":"markdown","source":"[FID evaluation explain](https://www.kaggle.com/competitions/gan-getting-started/overview/evaluation)","metadata":{"id":"tOAu4NAimn-t"}},{"cell_type":"code","source":"with strategy.scope():\n  inception_model = tf.keras.applications.InceptionV3(input_shape=(256, 256, 3), pooling='avg', include_top=False)\n\n  mix3 = inception_model.get_layer('mixed9').output\n  f0 = tf.keras.layers.GlobalAveragePooling2D()(mix3)\n\n  inception_model = tf.keras.Model(inputs=inception_model.input, outputs=f0)\n  inception_model.trainable = False\n\n  def calculate_activation_satistics_mod(images, fid_model):\n    act = tf.cast(fid_model.predict(images), tf.float32)\n\n    mu = tf.reduce_mean(act, axis=0)\n    mean_x = tf.reduce_mean(act, axis=0, keepdims=True)\n    mx = tf.matmul(tf.transpose(mean_x), mean_x)\n    vx = tf.matmul(tf.transpose(act), act)/tf.cast(tf.shape(act)[0], tf.float32)\n    sigma = vx - mx\n    return mu, sigma\n  myFID_mu2, myFID_sigma2 = calculate_activation_satistics_mod(fid_monet_ds, inception_model)","metadata":{"id":"WsjuArdoUOhx","outputId":"2f0bbadd-f65d-413c-f592-85f431316cc2","execution":{"iopub.status.busy":"2022-09-22T17:57:35.402625Z","iopub.execute_input":"2022-09-22T17:57:35.403596Z","iopub.status.idle":"2022-09-22T17:58:03.596553Z","shell.execute_reply.started":"2022-09-22T17:57:35.403532Z","shell.execute_reply":"2022-09-22T17:58:03.595427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n  def calculate_frechet_distance(mu1, sigma1, mu2, sigma2):\n    fid_epsilon = 1e-14\n\n    covmean = tf.linalg.sqrtm(tf.cast(tf.matmul(sigma1, sigma2), tf.complex64))\n#   isgood = tf.cast(tf.math.is_finite(covmean), tf.int32)\n#   if tf.size(isgood) != tf.math.reduce_sum(isgood)\n#     return 0\n    covmean = tf.cast(tf.math.real(covmean), tf.float32)\n\n    tr_covmean = tf.linalg.trace(covmean)\n\n    return tf.matmul(tf.expand_dims(mu1 - mu2, axis=0), tf.expand_dims(mu1 - mu2, axis=1)) + tf.linalg.trace(sigma1) + tf.linalg.trace(sigma2) - 2 * tr_covmean\n  \n  def FID(images, gen_model, inception_model=inception_model, myFID_mu2=myFID_mu2, myFID_sigma2=myFID_sigma2):\n    inp = layers.Input(shape=[256, 256, 3], name='input_image')\n    x = gen_model(inp)\n    x = inception_model(x)\n    fid_model = tf.keras.Model(inputs=inp, outputs=x)\n\n    mu1, sigma1 = calculate_activation_satistics_mod(images, fid_model)\n\n    fid_value = calculate_frechet_distance(mu1, sigma1, myFID_mu2, myFID_sigma2)\n\n    return fid_value","metadata":{"id":"xf7dmgj4c9h7","execution":{"iopub.status.busy":"2022-09-22T17:58:03.598032Z","iopub.execute_input":"2022-09-22T17:58:03.598398Z","iopub.status.idle":"2022-09-22T17:58:03.611580Z","shell.execute_reply.started":"2022-09-22T17:58:03.598367Z","shell.execute_reply":"2022-09-22T17:58:03.610460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build generator","metadata":{"id":"FEVMi0BdgKYL"}},{"cell_type":"code","source":"OUTPUT_CHANNELS = 3\ndef down_sample(filters, size, apply_instancenorm=True):\n  initializer = tf.random_normal_initializer(0., 0.02)\n  gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n  layer = keras.Sequential()\n  layer.add(layers.Conv2D(filters, size, strides=2, padding='same', kernel_initializer=initializer, use_bias=False))\n\n  if apply_instancenorm:\n    layer.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n  layer.add(layers.LeakyReLU())\n  return layer","metadata":{"id":"sLZRwgSFgL38","execution":{"iopub.status.busy":"2022-09-22T17:58:03.615253Z","iopub.execute_input":"2022-09-22T17:58:03.616228Z","iopub.status.idle":"2022-09-22T17:58:03.649615Z","shell.execute_reply.started":"2022-09-22T17:58:03.616185Z","shell.execute_reply":"2022-09-22T17:58:03.648347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def up_sample(filters, size, apply_dropout=False):\n  initializer = tf.random_normal_initializer(0., 0.02)\n  gamma_init = keras.initializers.RandomNormal(mean=0, stddev=0.02)\n\n  layer = keras.Sequential()\n  layer.add(layers.Conv2DTranspose(filters, size, strides=2, padding='same', kernel_initializer=initializer, use_bias=False))\n  layer.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n\n  if apply_dropout:\n    layer.add(layers.Dropout(0.5))\n  layer.add(layers.ReLU())\n  return layer","metadata":{"id":"8weETCO9h7IT","execution":{"iopub.status.busy":"2022-09-22T17:58:03.656700Z","iopub.execute_input":"2022-09-22T17:58:03.657399Z","iopub.status.idle":"2022-09-22T17:58:03.665946Z","shell.execute_reply.started":"2022-09-22T17:58:03.657349Z","shell.execute_reply":"2022-09-22T17:58:03.664783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.ops.gen_data_flow_ops import padding_fifo_queue\ndef Generator():\n  inputs = layers.Input(shape=[256, 256, 3])\n  down_stack = [\n      down_sample(64, 4, apply_instancenorm=False), # (size, 128, 128, 64)\n      down_sample(128, 4),                          # (size, 64, 64, 128)\n      down_sample(256, 4),                          # (size, 32, 32, 256)\n      down_sample(512, 4),                          # (size, 16, 16, 512)\n      down_sample(512, 4),                          # (size, 8, 8, 512)\n      down_sample(512, 4),                          # (size, 4, 4, 512)\n      down_sample(512, 4),                          # (size, 2, 2, 512)\n      down_sample(512, 4)                           # (size, 1, 1, 512)\n  ]\n\n  up_stack = [\n      up_sample(512, 4, apply_dropout=True),        # (size, 2, 2, 1024) : deep = 512 + 512(skip connect)\n      up_sample(512, 4, apply_dropout=True),        # (size, 4, 4, 1024)\n      up_sample(512, 4, apply_dropout=True),        # (size, 8, 8, 1024)\n      up_sample(512, 4),                            # (size, 16, 16, 1024)\n      up_sample(256, 4),                            # (size, 32, 32, 512)\n      up_sample(128, 4),                            # (size, 64, 64, 256)\n      up_sample(64, 4)                              # (size, 128, 128, 128)\n  ]\n  initializer = tf.random_normal_initializer(0., 0.02)\n  last = layers.Conv2DTranspose(3, 4, strides=2, padding='same', kernel_initializer=initializer, activation='tanh') # (size, 256, 256, 3)\n\n  x = inputs\n\n  #Downsampling through the model\n  skips = []\n  for down in down_stack:\n    x = down(x)\n    skips.append(x)\n  \n  skips = reversed(skips[:-1])\n\n  #Upsampling and establishing the skip connections\n  for up, skip in zip(up_stack, skips):\n    x = up(x)\n    x = layers.Concatenate()([x, skip])\n  \n  x = last(x)\n  return keras.Model(inputs=inputs, outputs=x)","metadata":{"id":"g0OZDqi4jKDp","execution":{"iopub.status.busy":"2022-09-22T17:58:03.668222Z","iopub.execute_input":"2022-09-22T17:58:03.669387Z","iopub.status.idle":"2022-09-22T17:58:03.682144Z","shell.execute_reply.started":"2022-09-22T17:58:03.669344Z","shell.execute_reply":"2022-09-22T17:58:03.681067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model","metadata":{"id":"YB3R2jDPOBgO","execution":{"iopub.status.busy":"2022-09-22T17:58:03.683683Z","iopub.execute_input":"2022-09-22T17:58:03.684417Z","iopub.status.idle":"2022-09-22T17:58:03.697534Z","shell.execute_reply.started":"2022-09-22T17:58:03.684361Z","shell.execute_reply":"2022-09-22T17:58:03.696186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen_viz = Generator()\nplot_model(gen_viz, show_shapes=True)","metadata":{"id":"-vfkMUjsOImo","outputId":"be68301d-989b-412d-a2bf-21697a418dd2","execution":{"iopub.status.busy":"2022-09-22T17:58:03.699231Z","iopub.execute_input":"2022-09-22T17:58:03.699938Z","iopub.status.idle":"2022-09-22T17:58:07.364139Z","shell.execute_reply.started":"2022-09-22T17:58:03.699900Z","shell.execute_reply":"2022-09-22T17:58:07.362856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen_viz.summary()","metadata":{"id":"yKe7x7SZAuLM","outputId":"fd2d1e79-9984-4c94-bc1d-ef85fce31961","execution":{"iopub.status.busy":"2022-09-22T17:58:07.366323Z","iopub.execute_input":"2022-09-22T17:58:07.367033Z","iopub.status.idle":"2022-09-22T17:58:07.379928Z","shell.execute_reply.started":"2022-09-22T17:58:07.366991Z","shell.execute_reply":"2022-09-22T17:58:07.378734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Discriminator","metadata":{"id":"OAdMT3VvSQ1k"}},{"cell_type":"markdown","source":"![](https://drive.google.com/uc?export=view&id=1M3pp5QlYFk1F8D9-YY-R3CxFg8_TFhAA)","metadata":{"id":"85gVKp2wKBDn"}},{"cell_type":"markdown","source":"Monet Discriminator","metadata":{"id":"nN9RKYMURwQW"}},{"cell_type":"markdown","source":"## Headless discriminator (shared part)","metadata":{"id":"OvYVT3avSkyJ"}},{"cell_type":"code","source":"def Discriminator():\n  initializer = tf.random_normal_initializer(0., 0.02)\n  gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n  inp = layers.Input(shape=[256, 256, 3], name='input_image')\n  x = inp\n  down1 = down_sample(64, 4, False)(x)        # (size, 128, 128, 64)\n  down2 = down_sample(128, 4)(down1)          # (size, 64, 64, 128)\n  down3 = down_sample(256, 4)(down2)          # (size, 32, 32, 256)\n\n  zero_pad1 = layers.ZeroPadding2D()(down3)   # (size, 34, 34, 256)\n  conv = layers.Conv2D(512, 4, strides=1, kernel_initializer=initializer, use_bias=False)(zero_pad1) # (size, 31, 31, 512)\n\n  norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init)(conv)\n  leaky_relu = layers.LeakyReLU()(norm1)\n  zero_pad2 = layers.ZeroPadding2D()(leaky_relu) # (size, 33, 33, 512)\n\n  return keras.Model(inputs=inp, outputs=zero_pad2)","metadata":{"id":"0i8tYKuUSW4E","execution":{"iopub.status.busy":"2022-09-22T17:58:07.382215Z","iopub.execute_input":"2022-09-22T17:58:07.383052Z","iopub.status.idle":"2022-09-22T17:58:07.392359Z","shell.execute_reply.started":"2022-09-22T17:58:07.383011Z","shell.execute_reply":"2022-09-22T17:58:07.391190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Discriminator_viz = Discriminator()\nplot_model(Discriminator_viz, show_shapes=True,show_dtype=True,show_layer_names=False , expand_nested=True)","metadata":{"id":"ighDTfBzcfNs","outputId":"83cdabcc-2ef5-4e0a-9237-a0628754f0e6","execution":{"iopub.status.busy":"2022-09-22T17:58:07.394110Z","iopub.execute_input":"2022-09-22T17:58:07.394890Z","iopub.status.idle":"2022-09-22T17:58:07.994035Z","shell.execute_reply.started":"2022-09-22T17:58:07.394849Z","shell.execute_reply":"2022-09-22T17:58:07.992856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Discriminator_viz.summary()","metadata":{"id":"MgdUi2b7eFX1","outputId":"532cc4c3-02c7-4e86-fd3e-e2cc34905137","execution":{"iopub.status.busy":"2022-09-22T17:58:07.996239Z","iopub.execute_input":"2022-09-22T17:58:07.996891Z","iopub.status.idle":"2022-09-22T17:58:08.006012Z","shell.execute_reply.started":"2022-09-22T17:58:07.996847Z","shell.execute_reply":"2022-09-22T17:58:08.004886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Head for two-objective discriminator for Monet","metadata":{"id":"RkkW4ks0W3K3"}},{"cell_type":"code","source":"def DHead():\n  initializer = tf.random_normal_initializer(0., 0.02)\n\n  inp = layers.Input(shape=[33, 33, 512], name='input_image')\n  x = inp\n\n  last = layers.Conv2D(1, 4, strides=1, kernel_initializer=initializer)(x) # (size, 30, 30, 1)\n\n  return keras.Model(inputs=inp, outputs=last)","metadata":{"id":"TPslknrGTCJ_","execution":{"iopub.status.busy":"2022-09-22T17:58:08.007613Z","iopub.execute_input":"2022-09-22T17:58:08.007978Z","iopub.status.idle":"2022-09-22T17:58:08.016881Z","shell.execute_reply.started":"2022-09-22T17:58:08.007944Z","shell.execute_reply":"2022-09-22T17:58:08.015573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DHead_viz = DHead()\nplot_model(DHead_viz, show_shapes=True,show_dtype=True,show_layer_names=False , expand_nested=True)","metadata":{"id":"AQmuQGHHeAN1","outputId":"59fb66b1-5e82-409b-bd44-b0f872068a89","execution":{"iopub.status.busy":"2022-09-22T17:58:08.018826Z","iopub.execute_input":"2022-09-22T17:58:08.019206Z","iopub.status.idle":"2022-09-22T17:58:08.437432Z","shell.execute_reply.started":"2022-09-22T17:58:08.019173Z","shell.execute_reply":"2022-09-22T17:58:08.435869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DHead_viz.summary()","metadata":{"id":"xxd1mug5eVOL","outputId":"f9aeea9f-4964-494b-a458-171c18073c83","execution":{"iopub.status.busy":"2022-09-22T17:58:08.439069Z","iopub.execute_input":"2022-09-22T17:58:08.440169Z","iopub.status.idle":"2022-09-22T17:58:08.448004Z","shell.execute_reply.started":"2022-09-22T17:58:08.440113Z","shell.execute_reply":"2022-09-22T17:58:08.446818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Discriminator for Photos (unchanged)","metadata":{"id":"3nAl0FU6X62h"}},{"cell_type":"code","source":"def DiscriminatorP():\n  initializer = tf.random_normal_initializer(0., 0.02)\n  gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n  inp = layers.Input(shape=[256, 256, 3], name='input_image')\n  x = inp\n\n  down1 = down_sample(64, 4, False)(x)        # (size, 128, 128, 64)\n  down2 = down_sample(128, 4)(down1)          # (size, 64, 64, 128)\n  down3 = down_sample(256, 4)(down2)          # (size, 32, 32, 256)\n\n  zero_pad1 = layers.ZeroPadding2D()(down3) # (size, 34, 34, 256)\n  conv = layers.Conv2D(512, 4, strides=1, kernel_initializer=initializer, use_bias=False)(zero_pad1) # (size, 31, 31, 512)\n\n  norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init)(conv)\n  leaky_relu = layers.LeakyReLU()(norm1)\n  zero_pad2 = layers.ZeroPadding2D()(leaky_relu) # (size, 33, 33, 512)\n  last = layers.Conv2D(1, 4, strides=1, kernel_initializer=initializer)(zero_pad2) # (size, 30, 30, 1)\n\n  return tf.keras.Model(inputs=inp, outputs=last)","metadata":{"id":"EiaeG3i1E8dJ","execution":{"iopub.status.busy":"2022-09-22T17:58:08.450167Z","iopub.execute_input":"2022-09-22T17:58:08.450649Z","iopub.status.idle":"2022-09-22T17:58:08.479148Z","shell.execute_reply.started":"2022-09-22T17:58:08.450614Z","shell.execute_reply":"2022-09-22T17:58:08.477700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n  monet_generator = Generator() # transforms photos to Monet-esque paintings\n  photo_generator = Generator() # transforms Monet paintings to be more like photos\n\n  monet_discriminator = Discriminator() # differentiates real and fake Monet paintings\n  photo_discriminator = DiscriminatorP() # differentiates real and fake photo\n\n  dHead1 = DHead()  # Head for BCE\n  dHead2 = DHead()  # Head for hinge loss","metadata":{"id":"6tgxA9QoNs1R","execution":{"iopub.status.busy":"2022-09-22T17:58:08.481072Z","iopub.execute_input":"2022-09-22T17:58:08.481989Z","iopub.status.idle":"2022-09-22T17:58:12.541998Z","shell.execute_reply.started":"2022-09-22T17:58:08.481945Z","shell.execute_reply":"2022-09-22T17:58:12.540413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CycleGAN with DiffAugment and two-objective discriminator","metadata":{"id":"AtIYWcdXQUM3"}},{"cell_type":"code","source":"class CycleGAN(keras.Model):\n  def __init__(\n      self,\n      monet_generator,\n      photo_generator,\n      monet_discriminator,\n      photo_discriminator,\n      dhead1,\n      dhead2,\n      lambda_cycle=3,\n      lambda_id=3\n  ):\n    super(CycleGAN, self).__init__()\n    self.m_gen = monet_generator\n    self.p_gen = photo_generator\n    self.m_disc = monet_discriminator\n    self.p_disc = photo_discriminator\n    self.lambda_cycle = lambda_cycle\n    self.lambda_id = lambda_id\n    self.dhead1 = dhead1\n    self.dhead2 = dhead2\n  def compile(\n      self,\n      m_gen_optimizer,\n      p_gen_optimizer,\n      m_disc_optimizer,\n      p_disc_optimizer,\n      gen_loss_fn1,\n      gen_loss_fn2,\n      disc_loss_fn1,\n      disc_loss_fn2,\n      cycle_loss_fn,\n      identity_loss_fn,\n      aug_fn\n  ):\n    super(CycleGAN, self).compile()\n    self.m_gen_optimizer = m_gen_optimizer\n    self.p_gen_optimizer = p_gen_optimizer\n    self.m_disc_optimizer = m_disc_optimizer\n    self.p_disc_optimizer = p_disc_optimizer\n    self.gen_loss_fn1 = gen_loss_fn1\n    self.gen_loss_fn2 = gen_loss_fn2\n    self.disc_loss_fn1 = disc_loss_fn1\n    self.disc_loss_fn2 = disc_loss_fn2\n    self.cycle_loss_fn = cycle_loss_fn\n    self.identity_loss_fn = identity_loss_fn\n    self.aug_fn = aug_fn\n\n    self.step_num = 0\n  def train_step(self, batch_data):\n    real_monet, real_photo = batch_data\n    batch_size = tf.shape(real_monet)[0]\n\n    #-----------------------------------\n    # Define and calculate loss function\n    #-----------------------------------\n\n    with tf.GradientTape(persistent=True) as tape:\n\n      #-------------------\n      # Training generator\n      #-------------------\n\n      # photo to monet back to photo\n      fake_monet = self.m_gen(real_photo, training=True)\n      cycled_photo = self.p_gen(fake_monet, training=True)\n\n      # monet to photo back to monet\n      fake_photo = self.p_gen(real_monet, training=True)\n      cycled_monet = self.m_gen(fake_photo, training=True)\n\n      # generating itself\n      same_photo = self.p_gen(real_photo, training=True)\n      same_monet = self.m_gen(real_monet, training=True)\n\n      # Diffaugment\n      both_monet = tf.concat([real_monet, fake_monet], axis=0)\n\n      aug_monet = self.aug_fn(both_monet)\n\n      aug_real_monet = aug_monet[:batch_size]\n      aug_fake_monet = aug_monet[batch_size:]\n\n      # Monet two-objective discriminator\n      disc_fake_monet1 = self.dhead1(self.m_disc(aug_fake_monet, training=True), training=True)\n      disc_real_monet1 = self.dhead1(self.m_disc(aug_real_monet, training=True), training=True)\n      disc_fake_monet2 = self.dhead2(self.m_disc(aug_fake_monet, training=True), training=True)\n      disc_real_monet2 = self.dhead2(self.m_disc(aug_real_monet, training=True), training=True)\n\n      # Monet GAN Loss\n      monet_gen_loss1 = self.gen_loss_fn1(disc_fake_monet1)\n      monet_gen_loss2 = self.gen_loss_fn2(disc_fake_monet2)\n      monet_gen_loss = (monet_gen_loss1 + monet_gen_loss2) * 0.4\n\n      # Photo discriminator\n      disc_real_photo = self.p_disc(real_photo, training=True)\n      disc_fake_photo = self.p_disc(fake_photo, training=True)\n\n      # Photo GAN Loss\n      photo_gen_loss = self.gen_loss_fn1(disc_fake_photo)\n\n      # Cycle Loss\n      monet_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle / tf.cast(batch_size, tf.float32))\n      photo_cycle_loss = self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle / tf.cast(batch_size, tf.float32))\n      total_cycle_loss = monet_cycle_loss + photo_cycle_loss\n\n      # Identity Loss\n      monet_identity_loss = self.identity_loss_fn(real_monet, same_monet, self.lambda_id / tf.cast(batch_size, tf.float32))\n      photo_identity_loss = self.identity_loss_fn(real_photo, same_photo, self.lambda_id / tf.cast(batch_size, tf.float32))\n\n      #------>Total Generator Loss\n      total_monet_gen_loss = monet_gen_loss + total_cycle_loss + monet_identity_loss\n      total_photo_gen_loss = photo_gen_loss + total_cycle_loss + photo_identity_loss\n\n      #---------------------\n      # Traing Discriminator\n      #---------------------\n\n      # Monet Discriminator Loss\n      monet_head_loss1 = self.disc_loss_fn1(disc_real_monet1, disc_fake_monet1)\n      monet_head_loss2 = self.disc_loss_fn2(disc_real_monet2, disc_fake_monet2)\n      monet_disc_loss = monet_head_loss1 + monet_head_loss2\n\n      # Photo Discriminator Loss\n      photo_disc_loss = self.disc_loss_fn1(disc_real_photo, disc_fake_photo)\n    \n    #------------------------\n    # Calculate the gradients\n    #------------------------\n\n    # for generator\n    monet_generator_gradients = tape.gradient(total_monet_gen_loss, self.m_gen.trainable_variables)\n    photo_generator_gradients = tape.gradient(total_photo_gen_loss, self.p_gen.trainable_variables)\n\n    # for discriminator\n    monet_discriminator_gradients = tape.gradient(monet_disc_loss, self.m_disc.trainable_variables)\n    photo_discriminator_gradients = tape.gradient(photo_disc_loss, self.p_disc.trainable_variables)\n\n    # for heads\n    monet_head_gradients1 = tape.gradient(monet_head_loss1, self.dhead1.trainable_variables)\n    monet_head_gradients2 = tape.gradient(monet_head_loss2, self.dhead2.trainable_variables)\n\n    #-------------------------------------\n    # Apply the gradients to the optimizer\n    #-------------------------------------\n    \n    # for head\n    self.m_disc_optimizer.apply_gradients(zip(monet_head_gradients1, self.dhead1.trainable_variables))\n    self.m_disc_optimizer.apply_gradients(zip(monet_head_gradients2, self.dhead2.trainable_variables))\n\n    # for generator\n    self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients, self.m_gen.trainable_variables))\n    self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients, self.p_gen.trainable_variables))\n\n    # for discriminator\n    self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients, self.m_disc.trainable_variables))\n    self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients, self.p_disc.trainable_variables))\n\n    return {\n        'monet_gen_loss': monet_gen_loss,\n        'monet_identity_loss': monet_identity_loss,\n        'photo_gen_loss': photo_gen_loss,\n        'photo_identity_loss': photo_identity_loss,\n        'total_cycle_loss': total_cycle_loss,\n        'total_monet_gen_loss': total_monet_gen_loss,\n        'total_photo_gen_loss': total_photo_gen_loss,\n        'monet_head_loss1': monet_head_loss1,\n        'monet_head_loss2': monet_head_loss2,\n        'monet_disc_loss': monet_disc_loss,\n        'photo_disc_loss': photo_disc_loss,\n    }\n    ","metadata":{"id":"td6lyDgXQQok","execution":{"iopub.status.busy":"2022-09-22T17:58:12.543873Z","iopub.execute_input":"2022-09-22T17:58:12.544657Z","iopub.status.idle":"2022-09-22T17:58:12.576655Z","shell.execute_reply.started":"2022-09-22T17:58:12.544604Z","shell.execute_reply":"2022-09-22T17:58:12.574148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate Loss","metadata":{"id":"a3bYWMTljBMM"}},{"cell_type":"markdown","source":"## Discriminator loss","metadata":{"id":"LJ84kJ1TjIc0"}},{"cell_type":"code","source":"with strategy.scope(): #for TPU\n  def discriminator_loss1(real, generated):\n    real_loss = tf.math.minimum(tf.zeros_like(real), real-tf.ones_like(real))\n\n    generated_loss = tf.math.minimum(tf.zeros_like(generated), -generated-tf.ones_like(generated))\n\n    total_disc_loss = real_loss + generated_loss\n\n    return tf.reduce_mean(-total_disc_loss * 0.5)\n  \n  def discriminator_loss2(real, generated):\n    generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)\\\n    (tf.ones_like(generated), generated)\n\n    real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)\\\n    (tf.ones_like(real), real)\n\n    total_disc_loss = real_loss + generated_loss\n\n    return tf.reduce_mean(total_disc_loss * 0.5)","metadata":{"id":"badhn-W6jHzC","execution":{"iopub.status.busy":"2022-09-22T17:58:12.578104Z","iopub.execute_input":"2022-09-22T17:58:12.578450Z","iopub.status.idle":"2022-09-22T17:58:12.865571Z","shell.execute_reply.started":"2022-09-22T17:58:12.578419Z","shell.execute_reply":"2022-09-22T17:58:12.864337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generator Loss","metadata":{"id":"nzbu7wdXpVM1"}},{"cell_type":"code","source":"with strategy.scope():\n  def generator_loss1(generated):\n    return tf.reduce_mean(-generated)\n  \n  def generator_loss2(generated):\n    return tf.reduce_mean(tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)\\\n                          (tf.zeros_like(generated), generated))","metadata":{"id":"_Rnf9lq8og9H","execution":{"iopub.status.busy":"2022-09-22T17:58:12.867700Z","iopub.execute_input":"2022-09-22T17:58:12.868515Z","iopub.status.idle":"2022-09-22T17:58:12.880829Z","shell.execute_reply.started":"2022-09-22T17:58:12.868433Z","shell.execute_reply":"2022-09-22T17:58:12.879823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cycle Loss","metadata":{"id":"u7m53SKIq1p0"}},{"cell_type":"code","source":"with strategy.scope():\n  def calc_cycle_loss(real_image, cycled_image, LAMBDA):\n    loss1 = tf.reduce_sum(tf.abs(real_image-cycled_image))\n\n    return LAMBDA * loss1 * 0.0000152587890625","metadata":{"id":"MqyMgSV2qyuH","execution":{"iopub.status.busy":"2022-09-22T17:58:12.882139Z","iopub.execute_input":"2022-09-22T17:58:12.883350Z","iopub.status.idle":"2022-09-22T17:58:12.892572Z","shell.execute_reply.started":"2022-09-22T17:58:12.883310Z","shell.execute_reply":"2022-09-22T17:58:12.891407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Identity Loss","metadata":{"id":"VPVmcnJ1rSsE"}},{"cell_type":"code","source":"with strategy.scope():\n  def identity_loss(real_image, same_image, LAMBDA):\n    loss = tf.reduce_sum(tf.abs(real_image-same_image))\n    return LAMBDA * 0.5 * loss * 0.0000152587890625","metadata":{"id":"lorob2YYrRwa","execution":{"iopub.status.busy":"2022-09-22T17:58:12.895999Z","iopub.execute_input":"2022-09-22T17:58:12.897238Z","iopub.status.idle":"2022-09-22T17:58:12.902774Z","shell.execute_reply.started":"2022-09-22T17:58:12.897195Z","shell.execute_reply":"2022-09-22T17:58:12.901790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Differentiable Augmentation","metadata":{"id":"lpPaO0ZsuX0b"}},{"cell_type":"markdown","source":"[Official code](https://github.com/mit-han-lab/data-efficient-gans/tree/master/DiffAugment-stylegan2)","metadata":{"id":"vx8SqDU00h4e"}},{"cell_type":"code","source":"with strategy.scope():\n  def rand_brightness(x):\n    magnitude = tf.random.uniform([tf.shape(x)[0], 1, 1, 1]) - 0.5\n    x = x + magnitude\n    return x\n  \n  def rand_saturation(x):\n    magnitude = tf.random.uniform([tf.shape(x)[0], 1, 1, 1]) * 2\n    x_mean = tf.reduce_sum(x, axis=3, keepdims=True) * 0.3333333333333333333\n\n    x = (x - x_mean) * magnitude + x_mean\n    return x\n  \n  def rand_contrast(x):\n    magnitude = tf.random.uniform([tf.shape(x)[0], 1, 1, 1]) + 0.5\n    x_mean = tf.reduce_sum(x, axis=[1, 2, 3], keepdims=True) * 5.086e-6\n    x = (x - x_mean) * magnitude + x_mean\n    return x\n  \n  def rand_translation(x, ratio=0.125):\n    batch_size = tf.shape(x)[0]\n    image_size = tf.shape(x)[1:3]\n    shift = tf.cast(tf.cast(image_size, tf.float32) * ratio + 0.5, tf.int32)\n    translation_x = tf.random.uniform(shape=[batch_size, 1], minval=-shift[0], maxval=shift[0] + 1, dtype=tf.int32)\n    translation_y = tf.random.uniform(shape=[batch_size, 1], minval=-shift[1], maxval=shift[1] + 1, dtype=tf.int32)\n    grid_x = tf.clip_by_value(tf.expand_dims(tf.range(image_size[0], dtype=tf.int32), 0) + translation_x + 1, 0, image_size[0] + 1)\n    grid_y = tf.clip_by_value(tf.expand_dims(tf.range(image_size[1], dtype=tf.int32), 0) + translation_y + 1, 0, image_size[1] + 1)\n    x = tf.gather_nd(tf.pad(x, [[0, 0], [1, 1], [0, 0], [0, 0]]), tf.expand_dims(grid_x, -1), batch_dims=1)\n    x = tf.transpose(tf.gather_nd(tf.pad(tf.transpose(x, [0, 2, 1, 3]), [[0, 0], [1, 1], [0, 0], [0, 0]]), tf.expand_dims(grid_y, -1),\n                                  batch_dims=1), [0, 2, 1, 3])\n    return x\n\n  def rand_cutout(x, ratio=0.5):\n    batch_size = tf.shape(x)[0]\n    image_size = tf.shape(x)[1:3]\n    cutout_size = tf.cast(tf.cast(image_size, tf.float32) * ratio + 0.5, tf.int32)\n    offset_x = tf.random.uniform([tf.shape(x)[0], 1, 1], maxval=image_size[0] + (1 - cutout_size[0] % 2), dtype=tf.int32)\n    offset_y = tf.random.uniform([tf.shape(x)[0], 1, 1], maxval=image_size[1] + (1 - cutout_size[1] % 2), dtype=tf.int32)\n    grid_batch, grid_x, grid_y = tf.meshgrid(tf.range(batch_size, dtype=tf.int32), tf.range(cutout_size[0], dtype=tf.int32),\n                                             tf.range(cutout_size[1], dtype=tf.int32), indexing='ij')\n    cutout_grid = tf.stack([grid_batch, grid_x + offset_x - cutout_size[0] // 2, grid_y + offset_y - cutout_size[1] // 2], axis=-1)\n    mask_shape = tf.stack([batch_size, image_size[0], image_size[1]])\n    cutout_grid = tf.maximum(cutout_grid, 0)\n    cutout_grid = tf.minimum(cutout_grid, tf.reshape(mask_shape - 1, [1, 1, 1, 3]))\n    mask = tf.maximum(1 - tf.scatter_nd(cutout_grid, tf.ones([batch_size, cutout_size[0], cutout_size[1]], dtype=tf.float32), mask_shape), 0)\n    x = x * tf.expand_dims(mask, axis=3)\n    return x\n\n  AUGMENT_FNS = {\n      'color': [rand_brightness, rand_saturation, rand_contrast],\n      'translation': [rand_translation],\n      'cutout': [rand_cutout],\n  }\n  def DiffAugment(x, policy='', channels_first=False):\n    if policy:\n      if channels_first:\n        x = tf.transpose(x, [0, 2, 3, 1])\n      for p in policy.split(','):\n        for f in AUGMENT_FNS[p]:\n          x = f(x)\n      if channels_first:\n        x = tf.transpose(x, [0, 3, 1, 2])\n    return x\n  \n  def aug_fn(image):\n    return DiffAugment(image, \"color,translation,cutout\")","metadata":{"id":"oGhOTM_mudwE","execution":{"iopub.status.busy":"2022-09-22T17:58:12.904669Z","iopub.execute_input":"2022-09-22T17:58:12.905380Z","iopub.status.idle":"2022-09-22T17:58:12.936171Z","shell.execute_reply.started":"2022-09-22T17:58:12.905331Z","shell.execute_reply":"2022-09-22T17:58:12.934960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define optimizer","metadata":{"id":"1ynDpaeUGxkM"}},{"cell_type":"code","source":"with strategy.scope():\n  monet_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\n  photo_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\n\n  monet_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\n  photo_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)","metadata":{"id":"nEJFef3uGsg0","execution":{"iopub.status.busy":"2022-09-22T17:58:12.937478Z","iopub.execute_input":"2022-09-22T17:58:12.938192Z","iopub.status.idle":"2022-09-22T17:58:12.954834Z","shell.execute_reply.started":"2022-09-22T17:58:12.938059Z","shell.execute_reply":"2022-09-22T17:58:12.953669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initial cycleGAN","metadata":{"id":"xboR7Zj6HjXD"}},{"cell_type":"code","source":"with strategy.scope():\n  cycle_gan_model = CycleGAN(\n      monet_generator, photo_generator, monet_discriminator, photo_discriminator, dHead1, dHead2\n  )","metadata":{"id":"VrAITmEBHij_","execution":{"iopub.status.busy":"2022-09-22T17:58:12.956406Z","iopub.execute_input":"2022-09-22T17:58:12.957573Z","iopub.status.idle":"2022-09-22T17:58:12.973812Z","shell.execute_reply.started":"2022-09-22T17:58:12.957531Z","shell.execute_reply":"2022-09-22T17:58:12.972605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{"id":"xD4btniaIEMV"}},{"cell_type":"markdown","source":"## 30 epochs with lr 2e-4","metadata":{"id":"08p6PULDILc5"}},{"cell_type":"code","source":"with strategy.scope():\n  cycle_gan_model.compile(\n      m_gen_optimizer = monet_generator_optimizer,\n      p_gen_optimizer = photo_generator_optimizer,\n      m_disc_optimizer = monet_discriminator_optimizer,\n      p_disc_optimizer = photo_discriminator_optimizer,\n      gen_loss_fn1 = generator_loss1,\n      gen_loss_fn2 = generator_loss2,\n      disc_loss_fn1 = discriminator_loss1,\n      disc_loss_fn2 = discriminator_loss2,\n      cycle_loss_fn = calc_cycle_loss,\n      identity_loss_fn = identity_loss,\n      aug_fn = aug_fn\n  )","metadata":{"id":"rGyJ1zYQIDOF","execution":{"iopub.status.busy":"2022-09-22T17:58:12.975719Z","iopub.execute_input":"2022-09-22T17:58:12.976406Z","iopub.status.idle":"2022-09-22T17:58:12.998248Z","shell.execute_reply.started":"2022-09-22T17:58:12.976367Z","shell.execute_reply":"2022-09-22T17:58:12.997133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history1 = cycle_gan_model.fit(final_dataset, steps_per_epoch=1407, epochs=1)","metadata":{"id":"aA7WK8mRJiUD","outputId":"87901c27-02e4-4c85-ecb3-cb2c42f31b91","execution":{"iopub.status.busy":"2022-09-22T17:58:13.000040Z","iopub.execute_input":"2022-09-22T17:58:13.000741Z","iopub.status.idle":"2022-09-22T17:58:13.005327Z","shell.execute_reply.started":"2022-09-22T17:58:13.000703Z","shell.execute_reply":"2022-09-22T17:58:13.004244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history1=np.load('../input/save-model/Monet_pretrain/history1.npy',allow_pickle='TRUE').item()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.007499Z","iopub.execute_input":"2022-09-22T17:58:13.008116Z","iopub.status.idle":"2022-09-22T17:58:13.035071Z","shell.execute_reply.started":"2022-09-22T17:58:13.008045Z","shell.execute_reply":"2022-09-22T17:58:13.033653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history1_df = pd.DataFrame(history1)\nhistory1_df.loc[:, ['monet_gen_loss', 'photo_disc_loss', 'monet_head_loss1', 'monet_head_loss2']].plot()\nprint('Minimum monet_gen_loss: {}'.format(history1_df['monet_gen_loss'].min()))\nprint('Minimum photo_disc_loss: {}'.format(history1_df['photo_disc_loss'].min()))\nprint('Minimum monet_head_loss1: {}'.format(history1_df['monet_head_loss1'].min()))\nprint('Minimum monet_head_loss2: {}'.format(history1_df['monet_head_loss2'].min()))","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.036667Z","iopub.execute_input":"2022-09-22T17:58:13.037363Z","iopub.status.idle":"2022-09-22T17:58:13.393809Z","shell.execute_reply.started":"2022-09-22T17:58:13.037300Z","shell.execute_reply":"2022-09-22T17:58:13.392423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8 epochs with lr 1e-4","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    monet_generator_optimizer = tf.keras.optimizers.Adam(1e-4, beta_1=0.5)\n    photo_generator_optimizer = tf.keras.optimizers.Adam(1e-4, beta_1=0.5)\n\n    monet_discriminator_optimizer = tf.keras.optimizers.Adam(1e-4, beta_1=0.5)\n    photo_discriminator_optimizer = tf.keras.optimizers.Adam(1e-4, beta_1=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.395602Z","iopub.execute_input":"2022-09-22T17:58:13.396862Z","iopub.status.idle":"2022-09-22T17:58:13.405335Z","shell.execute_reply.started":"2022-09-22T17:58:13.396811Z","shell.execute_reply":"2022-09-22T17:58:13.404192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n\n    cycle_gan_model.compile(\n        m_gen_optimizer = monet_generator_optimizer,\n        p_gen_optimizer = photo_generator_optimizer,\n        m_disc_optimizer = monet_discriminator_optimizer,\n        p_disc_optimizer = photo_discriminator_optimizer,\n        gen_loss_fn1 = generator_loss1,\n        gen_loss_fn2 = generator_loss2,\n        disc_loss_fn1 = discriminator_loss1,\n        disc_loss_fn2 = discriminator_loss2,\n        cycle_loss_fn = calc_cycle_loss,\n        identity_loss_fn = identity_loss,\n        aug_fn = aug_fn ,\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.407047Z","iopub.execute_input":"2022-09-22T17:58:13.407803Z","iopub.status.idle":"2022-09-22T17:58:13.424810Z","shell.execute_reply.started":"2022-09-22T17:58:13.407749Z","shell.execute_reply":"2022-09-22T17:58:13.423471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history2 = cycle_gan_model.fit(final_dataset,steps_per_epoch=1407, epochs=18)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.432509Z","iopub.execute_input":"2022-09-22T17:58:13.433182Z","iopub.status.idle":"2022-09-22T17:58:13.440044Z","shell.execute_reply.started":"2022-09-22T17:58:13.433140Z","shell.execute_reply":"2022-09-22T17:58:13.438892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2=np.load('../input/save-model/Monet_pretrain/history2.npy',allow_pickle='TRUE').item()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.441589Z","iopub.execute_input":"2022-09-22T17:58:13.442045Z","iopub.status.idle":"2022-09-22T17:58:13.482858Z","shell.execute_reply.started":"2022-09-22T17:58:13.442011Z","shell.execute_reply":"2022-09-22T17:58:13.481567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2_df = pd.DataFrame(history2)\nhistory2_df.loc[:, ['monet_gen_loss', 'photo_disc_loss', 'monet_head_loss1', 'monet_head_loss2']].plot()\nprint('Minimum monet_gen_loss: {}'.format(history2_df['monet_gen_loss'].min()))\nprint('Minimum photo_disc_loss: {}'.format(history2_df['photo_disc_loss'].min()))\nprint('Minimum monet_head_loss1: {}'.format(history2_df['monet_head_loss1'].min()))\nprint('Minimum monet_head_loss2: {}'.format(history2_df['monet_head_loss2'].min()))","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.484446Z","iopub.execute_input":"2022-09-22T17:58:13.484948Z","iopub.status.idle":"2022-09-22T17:58:13.789129Z","shell.execute_reply.started":"2022-09-22T17:58:13.484902Z","shell.execute_reply":"2022-09-22T17:58:13.786711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8 epochs with lr 1e-5","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    monet_generator_optimizer = tf.keras.optimizers.Adam(1e-5, beta_1=0.5)\n    photo_generator_optimizer = tf.keras.optimizers.Adam(1e-5, beta_1=0.5)\n\n    monet_discriminator_optimizer = tf.keras.optimizers.Adam(1e-5, beta_1=0.5)\n    photo_discriminator_optimizer = tf.keras.optimizers.Adam(1e-5, beta_1=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.790721Z","iopub.execute_input":"2022-09-22T17:58:13.791154Z","iopub.status.idle":"2022-09-22T17:58:13.798623Z","shell.execute_reply.started":"2022-09-22T17:58:13.791119Z","shell.execute_reply":"2022-09-22T17:58:13.797657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n\n    cycle_gan_model.compile(\n        m_gen_optimizer = monet_generator_optimizer,\n        p_gen_optimizer = photo_generator_optimizer,\n        m_disc_optimizer = monet_discriminator_optimizer,\n        p_disc_optimizer = photo_discriminator_optimizer,\n        gen_loss_fn1 = generator_loss1,\n        gen_loss_fn2 = generator_loss2,\n        disc_loss_fn1 = discriminator_loss1,\n        disc_loss_fn2 = discriminator_loss2,\n        cycle_loss_fn = calc_cycle_loss,\n        identity_loss_fn = identity_loss,\n        aug_fn = aug_fn ,\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.800009Z","iopub.execute_input":"2022-09-22T17:58:13.801112Z","iopub.status.idle":"2022-09-22T17:58:13.820431Z","shell.execute_reply.started":"2022-09-22T17:58:13.801035Z","shell.execute_reply":"2022-09-22T17:58:13.819247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history3 = cycle_gan_model.fit(final_dataset,steps_per_epoch=1407, epochs=12)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.821835Z","iopub.execute_input":"2022-09-22T17:58:13.822251Z","iopub.status.idle":"2022-09-22T17:58:13.834123Z","shell.execute_reply.started":"2022-09-22T17:58:13.822194Z","shell.execute_reply":"2022-09-22T17:58:13.832519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"monet_generator.load_weights('../input/save-model/Monet_pretrain/ckpt_monet_generator.h5')\nphoto_generator.load_weights('../input/save-model/Monet_pretrain/ckpt_photo_generator.h5')\nmonet_discriminator.load_weights('../input/save-model/Monet_pretrain/monet_discriminator.h5')\nphoto_discriminator.load_weights('../input/save-model/Monet_pretrain/photo_discriminator.h5')\ndHead1.load_weights('../input/save-model/Monet_pretrain/dHead1.h5')\ndHead2.load_weights('../input/save-model/Monet_pretrain/dHead2.h5')\nhistory3=np.load('../input/save-model/Monet_pretrain/history3.npy',allow_pickle='TRUE').item()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:13.835957Z","iopub.execute_input":"2022-09-22T17:58:13.836356Z","iopub.status.idle":"2022-09-22T17:58:18.279144Z","shell.execute_reply.started":"2022-09-22T17:58:13.836323Z","shell.execute_reply":"2022-09-22T17:58:18.278172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#monet_generator.save_weights('ckpt_monet_generator.h5')\n#photo_generator.save_weights('ckpt_photo_generator.h5')\n#monet_discriminator.save_weights('monet_discriminator.h5')\n#photo_discriminator.save_weights('photo_discriminator.h5')\n#dHead1.save_weights('dHead1.h5')\n#dHead2.save_weights('dHead2.h5')\n#np.save('history3.npy',history3.history)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:18.281059Z","iopub.execute_input":"2022-09-22T17:58:18.281471Z","iopub.status.idle":"2022-09-22T17:58:18.286432Z","shell.execute_reply.started":"2022-09-22T17:58:18.281438Z","shell.execute_reply":"2022-09-22T17:58:18.285360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_iter = iter(photo_ds)\nfor n_sample in range(8):\n        example_sample = next(ds_iter)\n        generated_sample = monet_generator(example_sample)\n        \n        f = plt.figure(figsize=(32, 32))\n        \n        plt.subplot(121)\n        plt.title('Input image')\n        plt.imshow(example_sample[0] * 0.5 + 0.5)\n        plt.axis('off')\n        \n        plt.subplot(122)\n        plt.title('Generated image')\n        plt.imshow(generated_sample[0] * 0.5 + 0.5)\n        plt.axis('off')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:18.288157Z","iopub.execute_input":"2022-09-22T17:58:18.289574Z","iopub.status.idle":"2022-09-22T17:58:26.574111Z","shell.execute_reply.started":"2022-09-22T17:58:18.289532Z","shell.execute_reply":"2022-09-22T17:58:26.573266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_iter = iter(monet_ds)\nfor n_sample in range(8):\n\n        example_sample = next(ds_iter)\n        generated_sample = photo_generator(example_sample)\n        \n        f = plt.figure(figsize=(24, 24))\n        \n        plt.subplot(121)\n        plt.title('Input image')\n        plt.imshow(example_sample[0] * 0.5 + 0.5)\n        plt.axis('off')\n        \n        plt.subplot(122)\n        plt.title('Generated image')\n        plt.imshow(generated_sample[0] * 0.5 + 0.5)\n        plt.axis('off')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:26.575359Z","iopub.execute_input":"2022-09-22T17:58:26.575848Z","iopub.status.idle":"2022-09-22T17:58:39.512460Z","shell.execute_reply.started":"2022-09-22T17:58:26.575816Z","shell.execute_reply":"2022-09-22T17:58:39.511153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history3_df = pd.DataFrame(history3)\nhistory3_df.loc[:, ['monet_gen_loss', 'photo_gen_loss', 'total_monet_gen_loss' , 'total_photo_gen_loss' , 'monet_disc_loss', 'photo_disc_loss']].plot()\nprint('Minimum monet_gen_loss: {}'.format(history3_df['monet_gen_loss'].min()))\nprint('Minimum photo_gen_loss: {}'.format(history3_df['photo_gen_loss'].min()))\nprint('Minimum total_monet_gen_loss: {}'.format(history3_df['total_monet_gen_loss'].min()))\nprint('Minimum total_photo_gen_loss: {}'.format(history3_df['total_photo_gen_loss'].min()))\nprint('Minimum monet_disc_loss: {}'.format(history3_df['monet_disc_loss'].min()))\nprint('Minimum photo_disc_loss: {}'.format(history3_df['photo_disc_loss'].min()))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-22T17:58:39.514193Z","iopub.execute_input":"2022-09-22T17:58:39.514598Z","iopub.status.idle":"2022-09-22T17:58:39.837549Z","shell.execute_reply.started":"2022-09-22T17:58:39.514549Z","shell.execute_reply":"2022-09-22T17:58:39.836212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}