{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"duration":3295.767085,"end_time":"2020-08-29T09:21:52.609073","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2020-08-29T08:26:56.841988","version":"2.1.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Environemnt Setup\n\nIn this code, we're setting up the environment to use a CycleGAN architecture to apply Monet-style to photos. We'll be working with a TFRecord dataset. We import the necessary packages and configure the accelerator to use TPU for faster processing.\n\nFor more information, check out [TensorFlow](https://www.tensorflow.org/tutorials/generative/cyclegan) and [Keras](https://keras.io/examples/generative/cyclegan/) CycleGAN documentation pages.","metadata":{"papermill":{"duration":0.013215,"end_time":"2020-08-29T08:27:01.892915","exception":false,"start_time":"2020-08-29T08:27:01.879700","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n    \nprint(tf.__version__)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":10.623182,"end_time":"2020-08-29T08:27:12.529697","exception":false,"start_time":"2020-08-29T08:27:01.906515","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-06T03:26:42.898663Z","iopub.execute_input":"2023-09-06T03:26:42.899024Z","iopub.status.idle":"2023-09-06T03:26:42.906967Z","shell.execute_reply.started":"2023-09-06T03:26:42.898993Z","shell.execute_reply":"2023-09-06T03:26:42.906066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading and Pre-Processing\n\nWe want to keep our photo dataset and our Monet dataset separate. First, load in the filenames of the TFRecords.","metadata":{"papermill":{"duration":0.010743,"end_time":"2020-08-29T08:27:12.552004","exception":false,"start_time":"2020-08-29T08:27:12.541261","status":"completed"},"tags":[]}},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path()\nTFREC_DIR = os.path.join(GCS_PATH, 'monet_tfrec')","metadata":{"papermill":{"duration":1.027954,"end_time":"2020-08-29T08:27:13.591219","exception":false,"start_time":"2020-08-29T08:27:12.563265","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-06T03:26:45.570189Z","iopub.execute_input":"2023-09-06T03:26:45.571098Z","iopub.status.idle":"2023-09-06T03:26:46.664345Z","shell.execute_reply.started":"2023-09-06T03:26:45.571051Z","shell.execute_reply":"2023-09-06T03:26:46.663377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MONET_FILENAMES = tf.io.gfile.glob(os.path.join(TFREC_DIR, '*.tfrec'))\nprint('Monet TFRecord Files:', len(MONET_FILENAMES))\n\nTFREC_DIR = os.path.join(GCS_PATH, 'photo_tfrec')\nPHOTO_FILENAMES = tf.io.gfile.glob(os.path.join(TFREC_DIR, '*.tfrec'))\nprint('Photo TFRecord Files:', len(PHOTO_FILENAMES))","metadata":{"papermill":{"duration":0.329473,"end_time":"2020-08-29T08:27:13.932267","exception":false,"start_time":"2020-08-29T08:27:13.602794","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:42.110185Z","iopub.execute_input":"2023-09-05T18:49:42.110564Z","iopub.status.idle":"2023-09-05T18:49:43.530731Z","shell.execute_reply.started":"2023-09-05T18:49:42.110529Z","shell.execute_reply":"2023-09-05T18:49:43.529635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The code is preparing the images for a competition. These images are already 256x256 pixels and have 3 color channels (RGB). The code scales the pixel values to be within the range of -1 to 1, which is a common practice for generative models. It also removes any unnecessary information like labels or image IDs and only retains the image data from the TFRecord.","metadata":{"papermill":{"duration":0.011089,"end_time":"2020-08-29T08:27:13.954949","exception":false,"start_time":"2020-08-29T08:27:13.943860","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\n\n# Define the image size\nIMAGE_SIZE = [256, 256]\n\ndef decode_image(image: tf.Tensor) -> tf.Tensor:\n    \"\"\"\n    Decode and preprocess a JPEG image from a TFRecord.\n\n    Args:\n        image (tf.Tensor): A string tensor containing the encoded image data.\n\n    Returns:\n        tf.Tensor: A preprocessed image tensor with shape [256, 256, 3].\n    \"\"\"\n    # Decode the JPEG image\n    image = tf.image.decode_jpeg(image, channels=3)\n    \n    # Normalize pixel values to the range [-1, 1]\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    \n    # Reshape the image\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    \n    return image\n\ndef read_tfrecord(example: tf.Tensor) -> tf.Tensor:\n    \"\"\"\n    Read and parse a single example from a TFRecord dataset.\n\n    Args:\n        example (tf.Tensor): A string tensor containing the TFRecord example.\n\n    Returns:\n        tf.Tensor: A preprocessed image tensor with shape [256, 256, 3].\n    \"\"\"\n    # Define the TFRecord format\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    \n    # Parse the TFRecord example\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    \n    # Decode and preprocess the image\n    image = decode_image(example['image'])\n    \n    return image\n","metadata":{"papermill":{"duration":0.026749,"end_time":"2020-08-29T08:27:13.993244","exception":false,"start_time":"2020-08-29T08:27:13.966495","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:43.532122Z","iopub.execute_input":"2023-09-05T18:49:43.532571Z","iopub.status.idle":"2023-09-05T18:49:43.542654Z","shell.execute_reply.started":"2023-09-05T18:49:43.532535Z","shell.execute_reply":"2023-09-05T18:49:43.541761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the function to extract the image from the files.","metadata":{"papermill":{"duration":0.011183,"end_time":"2020-08-29T08:27:14.016170","exception":false,"start_time":"2020-08-29T08:27:14.004987","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Lets load and pre-process the data stored in TFRecord files\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Create a dataset from TFRecord files\n    dataset = tf.data.TFRecordDataset(filenames)\n    # Map the decoding function to parse and preprocess the TFRecords\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return dataset","metadata":{"papermill":{"duration":0.021327,"end_time":"2020-08-29T08:27:14.049161","exception":false,"start_time":"2020-08-29T08:27:14.027834","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:43.544494Z","iopub.execute_input":"2023-09-05T18:49:43.545151Z","iopub.status.idle":"2023-09-05T18:49:43.553881Z","shell.execute_reply.started":"2023-09-05T18:49:43.545117Z","shell.execute_reply":"2023-09-05T18:49:43.552960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's  visualize a photo example and a Monet example.","metadata":{}},{"cell_type":"code","source":"import random\n# Load and batch the datasets\nmonet_ds = load_dataset(MONET_FILENAMES, labeled=True).batch(1)\nphoto_ds = load_dataset(PHOTO_FILENAMES, labeled=True).batch(1)\n\n# Fetch a single example from each dataset\nexample_monet = next(iter(monet_ds))\nexample_photo = next(iter(photo_ds))\n\n# Create a 1x2 grid for visualization\nplt.figure(figsize=(12, 6))\n\n# Display the Photo image\nplt.subplot(121)\nplt.title('Photo')\nplt.imshow(example_photo[0] * 0.5 + 0.5)\nplt.axis('off')\n\n# Display the Monet image\nplt.subplot(122)\nplt.title('Monet')\nplt.imshow(example_monet[0] * 0.5 + 0.5)\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:49:43.557426Z","iopub.execute_input":"2023-09-05T18:49:43.557800Z","iopub.status.idle":"2023-09-05T18:49:47.687471Z","shell.execute_reply.started":"2023-09-05T18:49:43.557765Z","shell.execute_reply":"2023-09-05T18:49:47.686630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Sampling (Downsample & Upsample)","metadata":{}},{"cell_type":"markdown","source":"This code prepares the UNET architecture for a CycleGAN model. It defines two functions, downsample and upsample, which change the image size. They use instance normalization to adjust image values. The code uses TensorFlow Add-ons to implement instance normalization.\n\nThe \"Upsample\" function in this code increases the size of an image, reversing the effect of the \"Downsample\" operation. It uses a layer called \"Conv2DTranspose\" to achieve this. \"Conv2DTranspose\" is essentially the opposite of the \"Conv2D\" layer, and it helps enlarge the image dimensions.","metadata":{}},{"cell_type":"code","source":"# Lets do some downsample and upsample functions before building the generator in a CycleGAN architecture. \nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\nOUTPUT_CHANNELS = 3\n\ndef downsample(filters, size, apply_instancenorm=True):\n    # Initialize weights with random values\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    # Define the downsample block\n    block = keras.Sequential()\n    block.add(layers.Conv2D(filters, size, strides=2, padding='same',\n                            kernel_initializer=initializer, use_bias=False))\n\n    if apply_instancenorm:\n        block.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n\n    block.add(layers.LeakyReLU(alpha=0.2))  # Use a small slope for LeakyReLU\n\n    return block\n\ndef upsample(filters, size, apply_dropout=False):\n    # Initialize weights with random values\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    # Define the upsample block\n    block = keras.Sequential()\n    block.add(layers.Conv2DTranspose(filters, size, strides=2,\n                                      padding='same',\n                                      kernel_initializer=initializer,\n                                      use_bias=False))\n\n    block.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n\n    if apply_dropout:\n        block.add(layers.Dropout(0.5))\n\n    block.add(layers.ReLU())  # Use ReLU activation for upsample\n\n    return block\n","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:49:47.688922Z","iopub.execute_input":"2023-09-05T18:49:47.689853Z","iopub.status.idle":"2023-09-05T18:49:47.702406Z","shell.execute_reply.started":"2023-09-05T18:49:47.689818Z","shell.execute_reply":"2023-09-05T18:49:47.701626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The generator first downsamples the input image and then upsample while establishing long skip connections. Skip connections are a way to help bypass the vanishing gradient problem by concatenating the output of a layer to multiple layers instead of only one. Here we concatenate the output of the downsample layer to the upsample layer in a symmetrical fashion.","metadata":{"papermill":{"duration":0.012895,"end_time":"2020-08-29T08:27:16.418061","exception":false,"start_time":"2020-08-29T08:27:16.405166","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Generator","metadata":{}},{"cell_type":"code","source":"def Generator():\n    inputs = layers.Input(shape=[256, 256, 3])\n    x = inputs\n\n    # Define the architecture with increased capacity\n    down_stack = [\n        downsample(64, 4, apply_instancenorm=False),\n        downsample(128, 4),\n        downsample(256, 4),\n        downsample(512, 4),\n        downsample(512, 4),\n    ]\n\n    up_stack = [\n        upsample(512, 4, apply_dropout=True),\n        upsample(512, 4, apply_dropout=True),\n        upsample(512, 4, apply_dropout=True),\n        upsample(256, 4),\n        upsample(128, 4),\n    ]\n\n    initializer = tf.random_normal_initializer(0., 0.02)\n    last = layers.Conv2DTranspose(OUTPUT_CHANNELS, 4,\n                                  strides=2,\n                                  padding='same',\n                                  kernel_initializer=initializer,\n                                  activation='tanh')\n\n    # Downsampling through the model\n    skips = []\n    for down in down_stack:\n        x = down(x)\n        skips.append(x)\n\n    skips = reversed(skips[:-1])\n\n    # Upsampling and establishing the skip connections\n    for up, skip in zip(up_stack, skips):\n        x = up(x)\n        x = layers.Concatenate()([x, skip])\n\n    x = last(x)\n\n    return keras.Model(inputs=inputs, outputs=x)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:49:47.703940Z","iopub.execute_input":"2023-09-05T18:49:47.704955Z","iopub.status.idle":"2023-09-05T18:49:47.717067Z","shell.execute_reply.started":"2023-09-05T18:49:47.704919Z","shell.execute_reply":"2023-09-05T18:49:47.716126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Discriminator\n\nThe discriminator is like a detective for images. When it sees an image, it doesn't just say \"real\" or \"fake.\" Instead, it looks at the entire image and creates a new image where certain areas have high values if it thinks they are real and low values if it thinks they are fake. So, it's not a single yes/no answer; it provides a detailed map of its decision across the image.","metadata":{"papermill":{"duration":0.012989,"end_time":"2020-08-29T08:27:16.491503","exception":false,"start_time":"2020-08-29T08:27:16.478514","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\ndef Discriminator():\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    inp = layers.Input(shape=[256, 256, 3], name='input_image')\n\n    x = inp\n\n    # Downsample layers\n    down1 = downsample(64, 4, False)(x)  # (bs, 128, 128, 64)\n    down2 = downsample(128, 4)(down1)   # (bs, 64, 64, 128)\n    down3 = downsample(256, 4)(down2)   # (bs, 32, 32, 256)\n\n    # Additional convolutional layers for increased capacity\n    conv1 = layers.Conv2D(512, 4, strides=1, padding='same',\n                          kernel_initializer=initializer, use_bias=False)(down3)  # (bs, 32, 32, 512)\n    norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init)(conv1)\n    leaky_relu1 = layers.LeakyReLU()(norm1)\n\n    conv2 = layers.Conv2D(512, 4, strides=1, padding='same',\n                          kernel_initializer=initializer, use_bias=False)(leaky_relu1)  # (bs, 32, 32, 512)\n    norm2 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init)(conv2)\n    leaky_relu2 = layers.LeakyReLU()(norm2)\n\n    # Output layer\n    last = layers.Conv2D(1, 4, strides=1,\n                         kernel_initializer=initializer)(leaky_relu2)  # (bs, 29, 29, 1)\n\n    return tf.keras.Model(inputs=inp, outputs=last)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:49:47.718602Z","iopub.execute_input":"2023-09-05T18:49:47.719397Z","iopub.status.idle":"2023-09-05T18:49:47.730619Z","shell.execute_reply.started":"2023-09-05T18:49:47.719357Z","shell.execute_reply":"2023-09-05T18:49:47.729695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    monet_generator = Generator() # Transforms photos to Monet-esque paintings\n    photo_generator = Generator() # Transforms Monet paintings to be more like photos\n\n    monet_discriminator = Discriminator() # Differentiates real Monet paintings and generated Monet paintings\n    photo_discriminator = Discriminator() # Differentiates real photos and generated photos","metadata":{"papermill":{"duration":9.978841,"end_time":"2020-08-29T08:27:26.527361","exception":false,"start_time":"2020-08-29T08:27:16.548520","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:47.731865Z","iopub.execute_input":"2023-09-05T18:49:47.732184Z","iopub.status.idle":"2023-09-05T18:49:49.917251Z","shell.execute_reply.started":"2023-09-05T18:49:47.732145Z","shell.execute_reply":"2023-09-05T18:49:49.916285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"generators = {\n    'monet': monet_generator,\n    'photo': photo_generator\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since our generators are not trained yet, the generated Monet-esque photo does not show what is expected at this point. Lets see what it displayes","metadata":{"papermill":{"duration":0.013578,"end_time":"2020-08-29T08:27:26.554960","exception":false,"start_time":"2020-08-29T08:27:26.541382","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Generate Monet-esque photo from example_photo using the 'monet' generator\nto_monet = generators['monet'](example_photo)\n\n# Create a single figure with subplots\nplt.figure(figsize=(12, 5))\n\n# Plot the original photo\nplt.subplot(1, 3, 1)\nplt.imshow(example_photo[0] * 0.5 + 0.5)\nplt.title(\"Original Photo\")\nplt.axis('off')\n\n# Plot the generated Monet-esque photo\nplt.subplot(1, 3, 2)\nplt.imshow(to_monet[0] * 0.5 + 0.5)\nplt.title(\"Monet-esque Photo\")\nplt.axis('off')\n\n# Plot a side-by-side comparison\nplt.subplot(1, 3, 3)\nplt.imshow(np.concatenate([example_photo[0], to_monet[0]], axis=1) * 0.5 + 0.5)\nplt.title(\"Comparison\")\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:49:50.470991Z","iopub.execute_input":"2023-09-05T18:49:50.472003Z","iopub.status.idle":"2023-09-05T18:49:50.930510Z","shell.execute_reply.started":"2023-09-05T18:49:50.471964Z","shell.execute_reply":"2023-09-05T18:49:50.929599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CycleGAN model\n\nThis part of the code is setting the stage for building a CycleGAN model. The model will learn to turn photos into Monet-style images and ensure that the transformation is consistent, meaning that the final image after two transformations is similar to the original photo. And specifics of how this is achieved, including the definition of loss functions","metadata":{"papermill":{"duration":0.013939,"end_time":"2020-08-29T08:27:27.319522","exception":false,"start_time":"2020-08-29T08:27:27.305583","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CycleGan(keras.Model):\n    def __init__(\n        self,\n        monet_generator,\n        photo_generator,\n        monet_discriminator,\n        photo_discriminator,\n        lambda_cycle=10,\n    ):\n        super(CycleGan, self).__init__()\n        self.m_gen = monet_generator\n        self.p_gen = photo_generator\n        self.m_disc = monet_discriminator\n        self.p_disc = photo_discriminator\n        self.lambda_cycle = lambda_cycle\n\n    def compile(\n        self,\n        m_gen_optimizer,\n        p_gen_optimizer,\n        m_disc_optimizer,\n        p_disc_optimizer,\n        gen_loss_fn,\n        disc_loss_fn,\n        cycle_loss_fn,\n        identity_loss_fn\n    ):\n        super(CycleGan, self).compile()\n        self.m_gen_optimizer = m_gen_optimizer\n        self.p_gen_optimizer = p_gen_optimizer\n        self.m_disc_optimizer = m_disc_optimizer\n        self.p_disc_optimizer = p_disc_optimizer\n        self.gen_loss_fn = gen_loss_fn\n        self.disc_loss_fn = disc_loss_fn\n        self.cycle_loss_fn = cycle_loss_fn\n        self.identity_loss_fn = identity_loss_fn\n\n    def train_step(self, batch_data):\n        real_monet, real_photo = batch_data\n\n        with tf.GradientTape(persistent=True) as tape:\n            # photo to monet back to photo\n            fake_monet = self.m_gen(real_photo, training=True)\n            cycled_photo = self.p_gen(fake_monet, training=True)\n\n            # monet to photo back to monet\n            fake_photo = self.p_gen(real_monet, training=True)\n            cycled_monet = self.m_gen(fake_photo, training=True)\n\n            # generating itself\n            same_monet = self.m_gen(real_monet, training=True)\n            same_photo = self.p_gen(real_photo, training=True)\n\n            # discriminator used to check, inputting real images\n            disc_real_monet = self.m_disc(real_monet, training=True)\n            disc_real_photo = self.p_disc(real_photo, training=True)\n\n            # discriminator used to check, inputting fake images\n            disc_fake_monet = self.m_disc(fake_monet, training=True)\n            disc_fake_photo = self.p_disc(fake_photo, training=True)\n\n            # evaluates generator loss\n            monet_gen_loss = self.gen_loss_fn(disc_fake_monet)\n            photo_gen_loss = self.gen_loss_fn(disc_fake_photo)\n\n            # evaluates total cycle consistency loss\n            total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle) + self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle)\n\n            # evaluates total generator loss\n            total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle)\n            total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle)\n\n            # evaluates discriminator loss\n            monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)\n            photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)\n\n        # Calculate the gradients for generator and discriminator\n        monet_generator_gradients = tape.gradient(total_monet_gen_loss,\n                                                  self.m_gen.trainable_variables)\n        photo_generator_gradients = tape.gradient(total_photo_gen_loss,\n                                                  self.p_gen.trainable_variables)\n\n        monet_discriminator_gradients = tape.gradient(monet_disc_loss,\n                                                      self.m_disc.trainable_variables)\n        photo_discriminator_gradients = tape.gradient(photo_disc_loss,\n                                                      self.p_disc.trainable_variables)\n\n        # Apply the gradients to the optimizer\n        self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,\n                                                 self.m_gen.trainable_variables))\n\n        self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,\n                                                 self.p_gen.trainable_variables))\n\n        self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,\n                                                  self.m_disc.trainable_variables))\n\n        self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,\n                                                  self.p_disc.trainable_variables))\n\n        return {\n            \"monet_gen_loss\": total_monet_gen_loss,\n            \"photo_gen_loss\": total_photo_gen_loss,\n            \"monet_disc_loss\": monet_disc_loss,\n            \"photo_disc_loss\": photo_disc_loss\n        }\n","metadata":{"papermill":{"duration":0.049519,"end_time":"2020-08-29T08:27:27.383553","exception":false,"start_time":"2020-08-29T08:27:27.334034","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:50.931806Z","iopub.execute_input":"2023-09-05T18:49:50.932965Z","iopub.status.idle":"2023-09-05T18:49:50.952956Z","shell.execute_reply.started":"2023-09-05T18:49:50.932929Z","shell.execute_reply":"2023-09-05T18:49:50.951873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss functions\n\nThe discriminator loss function below compares real images to a matrix of 1s and fake images to a matrix of 0s. The perfect discriminator will output all 1s for real images and all 0s for fake images. The discriminator loss outputs the average of the real and generated loss.","metadata":{"papermill":{"duration":0.01417,"end_time":"2020-08-29T08:27:27.412256","exception":false,"start_time":"2020-08-29T08:27:27.398086","status":"completed"},"tags":[]}},{"cell_type":"code","source":"with strategy.scope():\n    # Use BinaryCrossentropy reduction to compute mean loss\n    bce_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.SUM)\n\n    def discriminator_loss(real, generated):\n        # Compute the loss for real samples\n        real_loss = bce_loss(tf.ones_like(real), real)\n\n        # Compute the loss for generated samples\n        generated_loss = bce_loss(tf.zeros_like(generated), generated)\n\n        # Calculate the total discriminator loss as the average of real and generated losses\n        total_disc_loss = (real_loss + generated_loss) * 0.5\n\n        return total_disc_loss\n","metadata":{"papermill":{"duration":0.02693,"end_time":"2020-08-29T08:27:27.453371","exception":false,"start_time":"2020-08-29T08:27:27.426441","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:50.954260Z","iopub.execute_input":"2023-09-05T18:49:50.954611Z","iopub.status.idle":"2023-09-05T18:49:50.966660Z","shell.execute_reply.started":"2023-09-05T18:49:50.954575Z","shell.execute_reply":"2023-09-05T18:49:50.965689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The generator's job is to create images that are so convincing that the discriminator can't tell they're fake. To do this, it tries to make the discriminator's output as close to 1 (indicating \"real\") as possible for every pixel in the generated image. So, it checks how different the discriminator's output is from a matrix of 1s (which represents a perfect \"real\" image), and this difference is the loss that the generator tries to minimize.","metadata":{"papermill":{"duration":0.014183,"end_time":"2020-08-29T08:27:27.482436","exception":false,"start_time":"2020-08-29T08:27:27.468253","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\n\ndef generator_loss(generated):\n    return tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(generated), generated)","metadata":{"papermill":{"duration":0.025103,"end_time":"2020-08-29T08:27:27.522023","exception":false,"start_time":"2020-08-29T08:27:27.496920","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:50.971698Z","iopub.execute_input":"2023-09-05T18:49:50.972779Z","iopub.status.idle":"2023-09-05T18:49:50.978504Z","shell.execute_reply.started":"2023-09-05T18:49:50.972711Z","shell.execute_reply":"2023-09-05T18:49:50.977586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We want the original photo and the photo that has been transformed twice (e.g., photo to Monet painting and back to photo) to look very similar. So, we calculate the difference between these two images and take the average of those differences. This average difference is our cycle consistency loss, and we want to minimize it during training.","metadata":{"papermill":{"duration":0.014145,"end_time":"2020-08-29T08:27:27.550631","exception":false,"start_time":"2020-08-29T08:27:27.536486","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def calc_cycle_loss(real_image, cycled_image, lambda_cycle):\n    loss = tf.reduce_mean(tf.abs(real_image - cycled_image))\n    return lambda_cycle * loss","metadata":{"papermill":{"duration":0.023987,"end_time":"2020-08-29T08:27:27.589167","exception":false,"start_time":"2020-08-29T08:27:27.565180","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:50.979645Z","iopub.execute_input":"2023-09-05T18:49:50.980594Z","iopub.status.idle":"2023-09-05T18:49:50.987700Z","shell.execute_reply.started":"2023-09-05T18:49:50.980555Z","shell.execute_reply":"2023-09-05T18:49:50.986764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The identity loss checks if the generator can take an image and transform it into itself. For example, if we input a photo, we want the generator to produce an output that is very similar to the original photo. The identity loss measures how similar the generated image is to the input image, and we aim to minimize this loss during training.","metadata":{"papermill":{"duration":0.014192,"end_time":"2020-08-29T08:27:27.617994","exception":false,"start_time":"2020-08-29T08:27:27.603802","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def identity_loss(real_image, same_image, lambda_value=1.0):\n    \"\"\"\n    Calculate identity loss.\n\n    Parameters:\n        real_image (tf.Tensor): Real input image.\n        same_image (tf.Tensor): Transformed image.\n        lambda_value (float): Weighting factor for the loss.\n\n    Returns:\n        tf.Tensor: Identity loss.\n    \"\"\"\n    loss = tf.reduce_mean(tf.abs(real_image - same_image))\n    return lambda_value * 0.5 * loss","metadata":{"papermill":{"duration":0.024098,"end_time":"2020-08-29T08:27:27.656443","exception":false,"start_time":"2020-08-29T08:27:27.632345","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:50.989288Z","iopub.execute_input":"2023-09-05T18:49:50.989749Z","iopub.status.idle":"2023-09-05T18:49:50.996335Z","shell.execute_reply.started":"2023-09-05T18:49:50.989686Z","shell.execute_reply":"2023-09-05T18:49:50.995315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train CycleGAN Model\n\nLet's compile our model. Since we used `tf.keras.Model` to build our CycleGAN, we can just ude the `fit` function to train our model.","metadata":{"papermill":{"duration":0.014013,"end_time":"2020-08-29T08:27:27.685063","exception":false,"start_time":"2020-08-29T08:27:27.671050","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Define the learning rate and beta_1 for the optimizers\nlearning_rate = 2e-4\nbeta_1 = 0.5\n\n# Create optimizers for generators and discriminators\nwith strategy.scope():\n    monet_generator_optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, beta_1=beta_1)\n    photo_generator_optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, beta_1=beta_1)\n\n    monet_discriminator_optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, beta_1=beta_1)\n    photo_discriminator_optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, beta_1=beta_1)\n","metadata":{"papermill":{"duration":0.025521,"end_time":"2020-08-29T08:27:27.724855","exception":false,"start_time":"2020-08-29T08:27:27.699334","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T18:49:50.997819Z","iopub.execute_input":"2023-09-05T18:49:50.998151Z","iopub.status.idle":"2023-09-05T18:49:51.011990Z","shell.execute_reply.started":"2023-09-05T18:49:50.998116Z","shell.execute_reply":"2023-09-05T18:49:51.011059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create and compile the CycleGAN model\nwith strategy.scope():\n    cycle_gan_model = CycleGan(\n        monet_generator=monet_generator,\n        photo_generator=photo_generator,\n        monet_discriminator=monet_discriminator,\n        photo_discriminator=photo_discriminator\n    )\n\n    cycle_gan_model.compile(\n        m_gen_optimizer=monet_generator_optimizer,\n        p_gen_optimizer=photo_generator_optimizer,\n        m_disc_optimizer=monet_discriminator_optimizer,\n        p_disc_optimizer=photo_discriminator_optimizer,\n        gen_loss_fn=generator_loss,\n        disc_loss_fn=discriminator_loss,\n        cycle_loss_fn=calc_cycle_loss,\n        identity_loss_fn=identity_loss\n    )\n","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:49:51.013349Z","iopub.execute_input":"2023-09-05T18:49:51.013870Z","iopub.status.idle":"2023-09-05T18:49:51.033089Z","shell.execute_reply.started":"2023-09-05T18:49:51.013837Z","shell.execute_reply":"2023-09-05T18:49:51.032223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cycle_gan_model.fit(\n    tf.data.Dataset.zip((monet_ds, photo_ds)),\n    epochs=25\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:49:51.034180Z","iopub.execute_input":"2023-09-05T18:49:51.034779Z","iopub.status.idle":"2023-09-05T20:33:41.300198Z","shell.execute_reply.started":"2023-09-05T18:49:51.034700Z","shell.execute_reply":"2023-09-05T20:33:41.298996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize our Monet-esque photos","metadata":{"papermill":{"duration":0.520583,"end_time":"2020-08-29T08:45:17.377538","exception":false,"start_time":"2020-08-29T08:45:16.856955","status":"completed"},"tags":[]}},{"cell_type":"code","source":"_, ax = plt.subplots(5, 2, figsize=(12, 20))  # Increase the figure size for better visualization\nfor i, img in enumerate(photo_ds.take(5)):\n    prediction = monet_generator(img, training=False)[0].numpy()\n    prediction = (prediction * 0.5 + 0.5)  # Denormalize\n    img = (img[0] * 0.5 + 0.5).numpy()\n\n    ax[i, 0].imshow(img)\n    ax[i, 1].imshow(prediction)\n    ax[i, 0].set_title(\"Input Photo\")\n    ax[i, 1].set_title(\"Monet-esque\")\n    ax[i, 0].axis(\"off\")\n    ax[i, 1].axis(\"off\")\n\nplt.tight_layout()  # Adjust subplot layout for better spacing\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-05T20:33:41.302113Z","iopub.execute_input":"2023-09-05T20:33:41.302480Z","iopub.status.idle":"2023-09-05T20:33:45.346861Z","shell.execute_reply.started":"2023-09-05T20:33:41.302445Z","shell.execute_reply":"2023-09-05T20:33:45.342816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create submission file","metadata":{"papermill":{"duration":0.589149,"end_time":"2020-08-29T08:45:21.943852","exception":false,"start_time":"2020-08-29T08:45:21.354703","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import PIL\n! mkdir ../images","metadata":{"papermill":{"duration":1.320397,"end_time":"2020-08-29T08:45:23.794591","exception":false,"start_time":"2020-08-29T08:45:22.474194","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T20:33:45.348455Z","iopub.execute_input":"2023-09-05T20:33:45.349050Z","iopub.status.idle":"2023-09-05T20:33:46.418476Z","shell.execute_reply.started":"2023-09-05T20:33:45.349013Z","shell.execute_reply":"2023-09-05T20:33:46.417195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nfor img in photo_ds:\n    prediction = monet_generator(img, training=False)[0].numpy()\n    prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n    im = PIL.Image.fromarray(prediction)\n    im.save(\"../images/\" + str(i) + \".jpg\")\n    i += 1","metadata":{"papermill":{"duration":2180.855246,"end_time":"2020-08-29T09:21:45.222099","exception":false,"start_time":"2020-08-29T08:45:24.366853","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T20:33:46.420944Z","iopub.execute_input":"2023-09-05T20:33:46.421349Z","iopub.status.idle":"2023-09-05T20:40:14.996755Z","shell.execute_reply.started":"2023-09-05T20:33:46.421310Z","shell.execute_reply":"2023-09-05T20:40:14.995701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{"papermill":{"duration":5.118423,"end_time":"2020-08-29T09:21:50.874795","exception":false,"start_time":"2020-08-29T09:21:45.756372","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-05T20:40:14.998382Z","iopub.execute_input":"2023-09-05T20:40:14.998748Z","iopub.status.idle":"2023-09-05T20:40:19.430631Z","shell.execute_reply.started":"2023-09-05T20:40:14.998694Z","shell.execute_reply":"2023-09-05T20:40:19.429625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.523647,"end_time":"2020-08-29T09:21:51.928992","exception":false,"start_time":"2020-08-29T09:21:51.405345","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}