{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\nHello people, welcome to this kernel. In this kernel I am going to generate monet paintings using monet paintings (300) and random normal images (7K+) \n\nBefore starting, I want to say that, I wrote this kernel for learning what is CycleGAN and how it works, so I've used this kernel:\n\nhttps://www.kaggle.com/amyjang/monet-cyclegan-tutorial\n\nAll the code of this kernel is same with the tutorial. So, if you don't know what is CycleGAN and want to learn, you should check the tutorial.\n\nSo let's get started."},{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nimport tensorflow_addons as tfa\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom glob import glob\n\nimport cv2\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* In this kernel, we'll use Tensorflow as deep learning library."},{"metadata":{},"cell_type":"markdown","source":"# Loading Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nGCS_PATH = KaggleDatasets().get_gcs_path()\n\nMONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))\nprint('Monet TFRecord Files:', len(MONET_FILENAMES))\n\nPHOTO_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))\nprint('Photo TFRecord Files:', len(PHOTO_FILENAMES))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [256, 256]\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    return image\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nmonet_ds = load_dataset(MONET_FILENAMES, labeled=True).batch(1)\nphoto_ds = load_dataset(PHOTO_FILENAMES, labeled=True).batch(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"# rgb\nOUT_CHANNELS = 3\n\ndef downsample(filters,size,apply_instancenorm=True):\n    initializer = tf.random_normal_initializer(0.,0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0,stddev=0.02)\n    \n    result = keras.Sequential()\n    result.add(layers.Conv2D(filters,size,strides=2,padding=\"same\",kernel_initializer=initializer,use_bias=False))\n    \n    if apply_instancenorm:\n        result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n        \n    result.add(layers.LeakyReLU())\n    \n    return result\n\n\ndef upsample(filters,size,apply_dropout=False):\n    initializer = tf.random_normal_initializer(0.,0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0,stddev=0.02)\n    \n    result = keras.Sequential()\n    result.add(layers.Conv2DTranspose(filters,size,strides=2,padding=\"same\",kernel_initializer=initializer,use_bias=False))\n    \n    result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n    \n    if apply_dropout:\n        result.add(layers.Dropout(0.5))\n        \n    result.add(layers.ReLU())\n    \n    return result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First, we defined downsample and upsample layers, thanks to these layers we downsample and upsample data in generator and discriminator."},{"metadata":{"trusted":true},"cell_type":"code","source":"def Generator():\n    inputs = layers.Input([256,256,3])\n    \n    # bs = batch size\n    down_stack = [\n        downsample(64,4,apply_instancenorm=False),\n        downsample(128, 4), \n        downsample(256, 4), \n        downsample(512, 4), \n        downsample(512, 4), \n        downsample(512, 4), \n        downsample(512, 4), \n        downsample(512, 4), \n        \n    ]\n    \n    up_stack = [\n        upsample(512, 4, apply_dropout=True), # (bs, 2, 2, 1024)\n        upsample(512, 4, apply_dropout=True), # (bs, 4, 4, 1024)\n        upsample(512, 4, apply_dropout=True), # (bs, 8, 8, 1024)\n        upsample(512, 4), # (bs, 16, 16, 1024)\n        upsample(256, 4), # (bs, 32, 32, 512)\n        upsample(128, 4), # (bs, 64, 64, 256)\n        upsample(64, 4), # (bs, 128, 128, 128)\n    ]\n    \n    initializer = tf.random_normal_initializer(0.,0.02)\n    last = layers.Conv2DTranspose(OUT_CHANNELS,4,strides=2,padding=\"same\",kernel_initializer=initializer,activation=\"tanh\")\n    \n    x = inputs\n    \n    skips = []\n    for down in down_stack:\n        x = down(x)\n        skips.append(x)\n        \n    skips = reversed(skips[:-1])\n    \n    for up,skip in zip(up_stack,skips):\n        x = up(x)\n        x = layers.Concatenate()([x,skip])\n        \n        \n    x = last(x)\n    \n    return tf.keras.Model(inputs=inputs,outputs=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* In order to avoid vanishing gradient problem we concatenated layers using skip connections."},{"metadata":{},"cell_type":"markdown","source":"# Building Discriminator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def Discriminator():\n    initializer = tf.random_normal_initializer(0.,0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0,stddev=0.02)\n    \n    inp = layers.Input(shape=(256,256,3),name=\"input_image\")\n    \n    x = inp\n    \n    down1 = downsample(64,4,False)(x)\n    down2 = downsample(128,4)(down1)\n    down3 = downsample(256,4)(down2)\n    \n    zero_pad1 = layers.ZeroPadding2D()(down3)\n    \n    conv = layers.Conv2D(512,4,strides=1,kernel_initializer=initializer,use_bias=False)(zero_pad1)\n    \n    norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init)(conv)\n    \n    leaky_relu = layers.LeakyReLU()(norm1)\n    \n    zero_pad2 = layers.ZeroPadding2D()(leaky_relu)\n    \n    last = layers.Conv2D(1,4,strides=1,kernel_initializer=initializer)(zero_pad2)\n    \n    return tf.keras.Model(inputs=inp,outputs=last)\n\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Discriminator is a CNN based classifier, it takes an image with shape 256x256x3 and determines whether the image is real or not.\n\n* We used LeakyReLU in order to avoid vanishing gradient problem.\n* And we used Instance normalization instead of BatchNormalization."},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating generators and discriminators using functions\n\nmonet_generator = Generator()\nphoto_generator = Generator()\n\nmonet_discriminator = Discriminator()\nphoto_discriminator = Discriminator()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There are 4 networks in a CycleGAN:\n\n    1. Monet Generator: Creates monet style images using photos.\n    1. Photo Geneartor: Creates photos using monet stlye images\n    1. Monet Discriminator: Determines whether the monet image is real or generated\n    1. Photo Discriminator: Determines whether the photo is real or generated."},{"metadata":{},"cell_type":"markdown","source":"# CycleGAN Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CycleGan(keras.Model):\n    def __init__(\n        self,\n        monet_generator,\n        photo_generator,\n        monet_discriminator,\n        photo_discriminator,\n        lambda_cycle=10,\n    ):\n        super(CycleGan, self).__init__()\n        self.m_gen = monet_generator\n        self.p_gen = photo_generator\n        self.m_disc = monet_discriminator\n        self.p_disc = photo_discriminator\n        self.lambda_cycle = lambda_cycle\n        \n    def compile(\n        self,\n        m_gen_optimizer,\n        p_gen_optimizer,\n        m_disc_optimizer,\n        p_disc_optimizer,\n        gen_loss_fn,\n        disc_loss_fn,\n        cycle_loss_fn,\n        identity_loss_fn\n    ):\n        super(CycleGan, self).compile()\n        self.m_gen_optimizer = m_gen_optimizer\n        self.p_gen_optimizer = p_gen_optimizer\n        self.m_disc_optimizer = m_disc_optimizer\n        self.p_disc_optimizer = p_disc_optimizer\n        self.gen_loss_fn = gen_loss_fn\n        self.disc_loss_fn = disc_loss_fn\n        self.cycle_loss_fn = cycle_loss_fn\n        self.identity_loss_fn = identity_loss_fn\n        \n    def train_step(self, batch_data):\n        real_monet, real_photo = batch_data\n        \n        with tf.GradientTape(persistent=True) as tape:\n            # photo to monet back to photo\n            fake_monet = self.m_gen(real_photo, training=True)\n            cycled_photo = self.p_gen(fake_monet, training=True)\n\n            # monet to photo back to monet\n            fake_photo = self.p_gen(real_monet, training=True)\n            cycled_monet = self.m_gen(fake_photo, training=True)\n\n            # generating itself\n            same_monet = self.m_gen(real_monet, training=True)\n            same_photo = self.p_gen(real_photo, training=True)\n\n            # discriminator used to check, inputing real images\n            disc_real_monet = self.m_disc(real_monet, training=True)\n            disc_real_photo = self.p_disc(real_photo, training=True)\n\n            # discriminator used to check, inputing fake images\n            disc_fake_monet = self.m_disc(fake_monet, training=True)\n            disc_fake_photo = self.p_disc(fake_photo, training=True)\n\n            # evaluates generator loss\n            monet_gen_loss = self.gen_loss_fn(disc_fake_monet)\n            photo_gen_loss = self.gen_loss_fn(disc_fake_photo)\n\n            # evaluates total cycle consistency loss\n            total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle) + self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle)\n\n            # evaluates total generator loss\n            total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle)\n            total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle)\n\n            # evaluates discriminator loss\n            monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)\n            photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)\n\n        # Calculate the gradients for generator and discriminator\n        monet_generator_gradients = tape.gradient(total_monet_gen_loss,\n                                                  self.m_gen.trainable_variables)\n        photo_generator_gradients = tape.gradient(total_photo_gen_loss,\n                                                  self.p_gen.trainable_variables)\n\n        monet_discriminator_gradients = tape.gradient(monet_disc_loss,\n                                                      self.m_disc.trainable_variables)\n        photo_discriminator_gradients = tape.gradient(photo_disc_loss,\n                                                      self.p_disc.trainable_variables)\n\n        # Apply the gradients to the optimizer\n        self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,\n                                                 self.m_gen.trainable_variables))\n\n        self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,\n                                                 self.p_gen.trainable_variables))\n\n        self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,\n                                                  self.m_disc.trainable_variables))\n\n        self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,\n                                                  self.p_disc.trainable_variables))\n        \n        return {\n            \"monet_gen_loss\": total_monet_gen_loss,\n            \"photo_gen_loss\": total_photo_gen_loss,\n            \"monet_disc_loss\": monet_disc_loss,\n            \"photo_disc_loss\": photo_disc_loss\n        }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There are 8 type of data in a CycleGAN:\n    1. Real Photos: Real photos from the dataset\n    1. Real Monet Style Images: Real monet paintings from the dataset\n    1. Generated Photos (using Monet Style Images)\n    1. Generated Monet Style Images (using Photos)\n    1. Cycled Photos: Real Photo => Monet Generator => Generated Monet => Photo Generator => Photo\n    1. Cycled Monet Style Images: Real Monet => Photo Generator => Generated Photo => Monet Generator => Monet\n    1. Same Photos: Real Photo => Photo Generator => Photo\n    1. Same Monet Style Images: Real Monet => Monet Generator => Monet"},{"metadata":{},"cell_type":"markdown","source":"# Defining Loss Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def discriminator_loss(real,generated):\n    real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True,reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(real),real)\n    generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True,reduction=tf.keras.losses.Reduction.NONE)(tf.zeros_like(generated),generated)\n    \n    total_disc_loss = real_loss + generated_loss\n    \n    return total_disc_loss * 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator_loss(generated):\n    return tf.keras.losses.BinaryCrossentropy(from_logits=True,reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(generated),generated)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_cycle_loss(real_image,cycled_image,LAMBDA):\n    loss1 = tf.reduce_mean(tf.abs(real_image-cycled_image))\n    \n    return LAMBDA * loss1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def identity_loss(real_image,same_image,LAMBDA):\n    loss = tf.reduce_mean(tf.abs(real_image - same_image))\n    return LAMBDA * loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There are 4 types of loss functions in a CycleGAN:\n    1. Discriminator loss: Total of real_image loss and fake_image loss (BCE loss)\n    1. Generator loss: Fake image loss (BCE loss)\n    1. Cycle Loss\n    1. Identity Loss\n    "},{"metadata":{},"cell_type":"markdown","source":"# CycleGAN Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"monet_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\nphoto_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\n\nmonet_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\nphoto_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cycle_gan_model = CycleGan(monet_generator, photo_generator, monet_discriminator, photo_discriminator)\n\ncycle_gan_model.compile(\n        m_gen_optimizer = monet_generator_optimizer,\n        p_gen_optimizer = photo_generator_optimizer,\n        m_disc_optimizer = monet_discriminator_optimizer,\n        p_disc_optimizer = photo_discriminator_optimizer,\n        gen_loss_fn = generator_loss,\n        disc_loss_fn = discriminator_loss,\n        cycle_loss_fn = calc_cycle_loss,\n        identity_loss_fn = identity_loss\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_dataset = tf.data.Dataset.zip((monet_ds, photo_ds))\n\ncycle_gan_model.fit(\n    final_dataset,\n    epochs=2\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I've trained model for two epochs, but when I created the submission file I've trained for 25 epochs."},{"metadata":{"trusted":true},"cell_type":"code","source":"_, ax = plt.subplots(5, 2, figsize=(12, 12))\nfor i, img in enumerate(photo_ds.take(5)):\n    prediction = monet_generator(img, training=False)[0].numpy()\n    prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n    img = (img[0] * 127.5 + 127.5).numpy().astype(np.uint8)\n\n    ax[i, 0].imshow(img)\n    ax[i, 1].imshow(prediction)\n    ax[i, 0].set_title(\"Input Photo\")\n    ax[i, 1].set_title(\"Monet-esque\")\n    ax[i, 0].axis(\"off\")\n    ax[i, 1].axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import PIL\n! mkdir ../images","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating A Submission File"},{"metadata":{"trusted":true},"cell_type":"code","source":"#i = 1\n#for img in photo_ds:\n    #prediction = monet_generator(img, training=False)[0].numpy()\n    #prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n    #im = PIL.Image.fromarray(prediction)\n    #im.save(\"../images/\" + str(i) + \".jpg\")\n    #i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import shutil\n#shutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}