{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21755,"databundleVersionId":1475600,"sourceType":"competition"}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"colab":{"provenance":[{"file_id":"1-JUIft8u_-A9IWaucgUcMw6Wt6_zvLTf","timestamp":1710596330412}],"machine_shape":"hm","gpuType":"V100"},"accelerator":"GPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### <font color='#DC7633'>**Introduction**</font>\n\n**This project uses CycleGAN to convert pictures to monet-styled drawings. We will use the gan-getting-started dataset from kaggle that contains regular pictures and monet drawings, in jpg format and in tfrec format**.","metadata":{"id":"9DvVj2Jy8sYQ"}},{"cell_type":"code","source":"# !pip install kaggle\n# !pip install tensorflow-addons","metadata":{"id":"545fxEDEUWlU","executionInfo":{"status":"ok","timestamp":1710594739056,"user_tz":-120,"elapsed":13588,"user":{"displayName":"דור בוכריס","userId":"12233430204438784905"}},"outputId":"0457771b-ecd6-4acb-c4b9-ea2cdf793371","execution":{"iopub.status.busy":"2024-03-19T19:52:32.881488Z","iopub.execute_input":"2024-03-19T19:52:32.881723Z","iopub.status.idle":"2024-03-19T19:52:43.102302Z","shell.execute_reply.started":"2024-03-19T19:52:32.881697Z","shell.execute_reply":"2024-03-19T19:52:43.101113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","id":"_VbM1HVh8sYU","executionInfo":{"status":"ok","timestamp":1710594742626,"user_tz":-120,"elapsed":3572,"user":{"displayName":"דור בוכריס","userId":"12233430204438784905"}},"outputId":"5f964503-8363-478a-dd6e-838cd0cf5ffd","execution":{"iopub.status.busy":"2024-03-19T20:30:40.362115Z","iopub.execute_input":"2024-03-19T20:30:40.362773Z","iopub.status.idle":"2024-03-19T20:30:40.368157Z","shell.execute_reply.started":"2024-03-19T20:30:40.362742Z","shell.execute_reply":"2024-03-19T20:30:40.367021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T20:23:09.871544Z","iopub.execute_input":"2024-03-19T20:23:09.871957Z","iopub.status.idle":"2024-03-19T20:23:17.386902Z","shell.execute_reply.started":"2024-03-19T20:23:09.871928Z","shell.execute_reply":"2024-03-19T20:23:17.386154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nAUTO = tf.data.experimental.AUTOTUNE    \nprint(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T20:31:28.033411Z","iopub.execute_input":"2024-03-19T20:31:28.033920Z","iopub.status.idle":"2024-03-19T20:31:28.039099Z","shell.execute_reply.started":"2024-03-19T20:31:28.033883Z","shell.execute_reply":"2024-03-19T20:31:28.038172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**Load The Data**</font>\n\n\n**Load in the filenames of the TFRecords.**","metadata":{"id":"Dhr25FqH8sYV"}},{"cell_type":"code","source":"ds_path = '/kaggle/input/gan-getting-started'\n\nmonet_files = tf.io.gfile.glob(str(ds_path + '/monet_tfrec/*.tfrec'))\nprint('Monet TFRecord Files:', len(monet_files))\n\nregular_files = tf.io.gfile.glob(str(ds_path + '/photo_tfrec/*.tfrec'))\nprint('Photo TFRecord Files:', len(regular_files))","metadata":{"id":"60-FjEI38sYW","executionInfo":{"status":"ok","timestamp":1710594763177,"user_tz":-120,"elapsed":404,"user":{"displayName":"דור בוכריס","userId":"12233430204438784905"}},"outputId":"363c03b0-1025-4170-d9bf-821b8a0995ca","execution":{"iopub.status.busy":"2024-03-19T20:31:30.207502Z","iopub.execute_input":"2024-03-19T20:31:30.208302Z","iopub.status.idle":"2024-03-19T20:31:30.225517Z","shell.execute_reply.started":"2024-03-19T20:31:30.208269Z","shell.execute_reply":"2024-03-19T20:31:30.224639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The images sized to 320x320 RGB format. We need to scale the images to a [-1, 1] scale - to return only the image itself and not the id or label.**","metadata":{"id":"MMHNYbk18sYX"}},{"cell_type":"code","source":"IMAGE_SIZE = [320, 320]","metadata":{"id":"NHwFb4hDUagd","execution":{"iopub.status.busy":"2024-03-19T20:31:31.957090Z","iopub.execute_input":"2024-03-19T20:31:31.957475Z","iopub.status.idle":"2024-03-19T20:31:31.962058Z","shell.execute_reply.started":"2024-03-19T20:31:31.957445Z","shell.execute_reply":"2024-03-19T20:31:31.960965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.image.resize(image, IMAGE_SIZE)\n    return image\n\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    return image","metadata":{"id":"m-pek83k8sYX","execution":{"iopub.status.busy":"2024-03-19T20:31:33.499992Z","iopub.execute_input":"2024-03-19T20:31:33.500359Z","iopub.status.idle":"2024-03-19T20:31:33.507736Z","shell.execute_reply.started":"2024-03-19T20:31:33.500330Z","shell.execute_reply":"2024-03-19T20:31:33.506532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Function to apply some aguments.**","metadata":{"id":"eGSBITMWobor"}},{"cell_type":"code","source":"def augment_image(image):\n  # resizing to 350 x 350 x 3\n  image = tf.image.resize(image, [350, 350],\n                          method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n\n  # randomly cropping to 256 x 256 x 3\n  image = tf.image.random_crop(image, size=[*IMAGE_SIZE, 3])\n\n  # random mirroring\n  # image = tf.image.random_flip_left_right(image)\n\n  return image","metadata":{"id":"M5mdw5JXodL8","execution":{"iopub.status.busy":"2024-03-19T20:31:36.204788Z","iopub.execute_input":"2024-03-19T20:31:36.205115Z","iopub.status.idle":"2024-03-19T20:31:36.210405Z","shell.execute_reply.started":"2024-03-19T20:31:36.205090Z","shell.execute_reply":"2024-03-19T20:31:36.209439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Function to extract the image from the files.**","metadata":{"id":"e9nJ7b-M8sYY"}},{"cell_type":"code","source":"def load_dataset(filenames, with_aguments=False):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE) #autotune - allows TensorFlow to automatically choose the optimal number of parallel calls for the map transformation\n    if with_aguments:\n      augmented_monet_dataset = dataset.map(augment_image, num_parallel_calls=tf.data.AUTOTUNE)\n      dataset = dataset.concatenate(augmented_monet_dataset) #merges the original dataset with the augmented one\n      dataset = dataset.shuffle(buffer_size=1024, reshuffle_each_iteration=True)\n    return dataset","metadata":{"id":"W7IGU2_L8sYY","execution":{"iopub.status.busy":"2024-03-19T20:31:39.482547Z","iopub.execute_input":"2024-03-19T20:31:39.482936Z","iopub.status.idle":"2024-03-19T20:31:39.488568Z","shell.execute_reply.started":"2024-03-19T20:31:39.482908Z","shell.execute_reply":"2024-03-19T20:31:39.487719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load in our datasets.**","metadata":{"id":"3TWFYMfA8sYY"}},{"cell_type":"code","source":"monet_ds = load_dataset(monet_files, with_aguments=True).batch(1)\nphoto_ds = load_dataset(regular_files).batch(1)","metadata":{"id":"BtvzJmXY8sYZ","execution":{"iopub.status.busy":"2024-03-19T20:31:42.311172Z","iopub.execute_input":"2024-03-19T20:31:42.311517Z","iopub.status.idle":"2024-03-19T20:31:42.451501Z","shell.execute_reply.started":"2024-03-19T20:31:42.311488Z","shell.execute_reply":"2024-03-19T20:31:42.450781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_monet = next(iter(monet_ds))\nexample_photo = next(iter(photo_ds))","metadata":{"id":"Ki6RqKAW8sYZ","execution":{"iopub.status.busy":"2024-03-19T20:31:44.301355Z","iopub.execute_input":"2024-03-19T20:31:44.301755Z","iopub.status.idle":"2024-03-19T20:31:45.522903Z","shell.execute_reply.started":"2024-03-19T20:31:44.301721Z","shell.execute_reply":"2024-03-19T20:31:45.521802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualize a regular photo example and a Monet example.**","metadata":{"id":"TDXI18Vg8sYZ"}},{"cell_type":"code","source":"plt.subplot(121)\nplt.title('Photo')\nplt.imshow(example_photo[0] * 0.5 + 0.5)\n\nplt.subplot(122)\nplt.title('Monet')\nplt.imshow(example_monet[0] * 0.5 + 0.5)","metadata":{"id":"wGh4__Eg8sYZ","executionInfo":{"status":"ok","timestamp":1710520903941,"user_tz":-120,"elapsed":1304,"user":{"displayName":"Rom Ben Aharon","userId":"04194826883711623755"}},"outputId":"d1d4d7e3-1b03-4155-ee9f-5877064cf1d2","execution":{"iopub.status.busy":"2024-03-19T20:31:46.644073Z","iopub.execute_input":"2024-03-19T20:31:46.644404Z","iopub.status.idle":"2024-03-19T20:31:47.243539Z","shell.execute_reply.started":"2024-03-19T20:31:46.644378Z","shell.execute_reply":"2024-03-19T20:31:47.242707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**The Generator**</font>\n\n**Define `downsample` and `upsample` functions:**\n\n**`downsample` reduces the width and height of the image by the stride. The stride is the length of the step the filter takes.**\n\n**We are using an instance normalization instead of batch normalization.**","metadata":{"id":"2KriW7vQ8sYa"}},{"cell_type":"code","source":"OUTPUT_CHANNELS = 3 #for RGB images\n\n\nclass InstanceNormalization(tf.keras.layers.Layer):\n    def __init__(self, epsilon=1e-5, **kwargs):\n        super(InstanceNormalization, self).__init__(**kwargs)\n        self.epsilon = epsilon\n\n    def build(self, input_shape):\n        self.gamma = self.add_weight(\n            name='gamma',\n            shape=(input_shape[-1],),\n            initializer='ones',\n            trainable=True\n        )\n        self.beta = self.add_weight(\n            name='beta',\n            shape=(input_shape[-1],),\n            initializer='zeros',\n            trainable=True\n        )\n\n    def call(self, inputs):\n        mean, variance = tf.nn.moments(inputs, axes=[1, 2], keepdims=True)\n        inv_sqrt_var = tf.math.rsqrt(variance + self.epsilon)\n        normalized = (inputs - mean) * inv_sqrt_var\n        return self.gamma * normalized + self.beta\n\ndef downsample(filters, size, apply_instancenorm=True):\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    result = keras.Sequential()\n    result.add(layers.Conv2D(filters, size, strides=2, padding='same',\n                             kernel_initializer=initializer, use_bias=False))\n    \n    if apply_instancenorm:\n        result.add(InstanceNormalization())\n    \n    result.add(layers.LeakyReLU())\n\n    return result","metadata":{"id":"JWiQ5zui8sYa","execution":{"iopub.status.busy":"2024-03-19T20:31:54.004085Z","iopub.execute_input":"2024-03-19T20:31:54.004430Z","iopub.status.idle":"2024-03-19T20:31:54.015752Z","shell.execute_reply.started":"2024-03-19T20:31:54.004404Z","shell.execute_reply":"2024-03-19T20:31:54.014584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**`Upsample` is the opposite of downsample - increases the dimensions of the of the image.**","metadata":{"id":"lanQ6KKr8sYa"}},{"cell_type":"code","source":"def upsample(filters, size, apply_dropout=False):\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    result = keras.Sequential()\n    result.add(layers.Conv2DTranspose(filters, size, strides=2,\n                                      padding='same',\n                                      kernel_initializer=initializer,\n                                      use_bias=False))\n\n    result.add(InstanceNormalization())\n\n    if apply_dropout:\n        result.add(layers.Dropout(0.5))\n\n    result.add(layers.ReLU())\n\n    return result","metadata":{"id":"CBp8dNuY8sYa","execution":{"iopub.status.busy":"2024-03-19T20:31:56.525015Z","iopub.execute_input":"2024-03-19T20:31:56.525374Z","iopub.status.idle":"2024-03-19T20:31:56.532266Z","shell.execute_reply.started":"2024-03-19T20:31:56.525345Z","shell.execute_reply":"2024-03-19T20:31:56.531308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The generator uses downsamples on input image, and then upsamples it.**\n\n**It also uses \"skips\" (also called \"residual connections\") to connect the output of a layer to multiple layers instead of only one, making better generation and prevent loss of information.**\n\n**Basically, G(x) = F(x) + x => where x is the original input, F(x) is the upsample and G(x) is the result with the skip.**","metadata":{"id":"8PhVbe188sYa"}},{"cell_type":"code","source":"def Generator():\n    inputs = layers.Input(shape=[*IMAGE_SIZE,3])\n\n        # bs = batch size\n    down_stack = [\n        downsample(64, 4, apply_instancenorm=False),  # (bs, 160, 160, 64)\n        downsample(128, 4),                          # (bs, 80, 80, 128)\n        downsample(256, 4),                          # (bs, 40, 40, 256)\n        downsample(512, 4),                          # (bs, 20, 20, 512)\n        downsample(512, 4),                          # (bs, 10, 10, 512)\n        downsample(512, 4),                          # (bs, 5, 5, 512)\n\n        # Adjust further if needed\n    ]\n\n    up_stack = [\n        upsample(512, 4, apply_dropout=True),        # (bs, 10, 10, 1024)\n        upsample(512, 4),                            # (bs, 20, 20, 1024)\n        upsample(512, 4),                            # (bs, 40, 40, 1024)\n        upsample(256, 4),                            # (bs, 80, 80, 512)\n        upsample(128, 4),                            # (bs, 160, 160, 256)\n        upsample(64, 4),                             # (bs, 320, 320, 128)\n        # Adjust further if needed\n    ]\n\n    initializer = tf.random_normal_initializer(0., 0.02)\n    last = layers.Conv2DTranspose(OUTPUT_CHANNELS, 4,\n                                  strides=2,\n                                  padding='same',\n                                  kernel_initializer=initializer,\n                                  activation='tanh') # (bs, 320, 320, 3)\n\n    x = inputs\n\n    # Downsampling through the model\n    skips = []\n    for down in down_stack:\n        x = down(x)\n        skips.append(x)\n\n    skips = reversed(skips[:-1])\n\n    # Upsampling and making skip connections\n    for up, skip in zip(up_stack, skips):\n        x = up(x)\n        x = layers.Concatenate()([x, skip])\n\n    x = last(x)\n\n    return keras.Model(inputs=inputs, outputs=x)","metadata":{"id":"FbvO5G0Z8sYa","execution":{"iopub.status.busy":"2024-03-19T20:31:58.855816Z","iopub.execute_input":"2024-03-19T20:31:58.856171Z","iopub.status.idle":"2024-03-19T20:31:58.866900Z","shell.execute_reply.started":"2024-03-19T20:31:58.856141Z","shell.execute_reply":"2024-03-19T20:31:58.865830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**The Discriminator**</font>\n\n\nThe discriminator takes in the input image and classifies it as real image or a generated image. The discriminator outputs a smaller 2D image with higher pixel values indicating a real classification and lower values indicating a fake classification.","metadata":{"id":"eQ1SejeV8sYb"}},{"cell_type":"code","source":"def Discriminator():\n    # Initialize parameters for the network layers\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    # Define input layer for the discriminator\n    input = layers.Input(shape=[*IMAGE_SIZE, 3], name='input_image')\n\n    # Pass the input through the initial layer\n    x = input\n\n    # Downsample the input through three convolutional layers\n    down1 = downsample(64, 4, False)(x) # (bs, 128, 128, 64)\n    down2 = downsample(128, 4)(down1) # (bs, 64, 64, 128)\n    down3 = downsample(256, 4)(down2) # (bs, 32, 32, 256)\n\n    # Pad the output of the third convolutional layer\n    zero_pad1 = layers.ZeroPadding2D()(down3) # (bs, 34, 34, 256)\n\n    # Apply convolutional layer with Instance Normalization and Leaky ReLU activation\n    conv = layers.Conv2D(512, 4, strides=1,\n                         kernel_initializer=initializer,\n                         use_bias=False)(zero_pad1) # (bs, 31, 31, 512)\n    norm1 = InstanceNormalization()(conv)\n    leaky_relu = layers.LeakyReLU()(norm1)\n\n    # Pad the output of the Leaky ReLU layer\n    zero_pad2 = layers.ZeroPadding2D()(leaky_relu) # (bs, 33, 33, 512)\n\n    # Apply final convolutional layer to get the discriminator output\n    last = layers.Conv2D(1, 4, strides=1,\n                         kernel_initializer=initializer)(zero_pad2) # (bs, 30, 30, 1)\n\n    return tf.keras.Model(inputs=input, outputs=last)","metadata":{"id":"NYbW--5j8sYb","execution":{"iopub.status.busy":"2024-03-19T20:32:01.548142Z","iopub.execute_input":"2024-03-19T20:32:01.548795Z","iopub.status.idle":"2024-03-19T20:32:01.557763Z","shell.execute_reply.started":"2024-03-19T20:32:01.548762Z","shell.execute_reply":"2024-03-19T20:32:01.556965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating instances of generators and discriminators**","metadata":{"id":"F8UlQoWxUHFY"}},{"cell_type":"code","source":"monet_generator = Generator() # transforms photos to Monet-esque paintings\nphoto_generator = Generator() # transforms Monet paintings to be more like photos\n\nmonet_discriminator = Discriminator() # differentiates real Monet paintings and generated Monet paintings\nphoto_discriminator = Discriminator() # differentiates real photos and generated photos","metadata":{"id":"F51zgB_e8sYb","execution":{"iopub.status.busy":"2024-03-19T20:32:03.810560Z","iopub.execute_input":"2024-03-19T20:32:03.810926Z","iopub.status.idle":"2024-03-19T20:32:04.515320Z","shell.execute_reply.started":"2024-03-19T20:32:03.810899Z","shell.execute_reply":"2024-03-19T20:32:04.514546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The generator is not trained yet, but this is how we will use it to generate:**","metadata":{"id":"WfgoElDM8sYb"}},{"cell_type":"code","source":"to_monet = monet_generator(example_photo)\n\nplt.subplot(1, 2, 1)\nplt.title(\"Original Regular Photo\")\nplt.imshow(example_photo[0] * 0.5 + 0.5)\n\nplt.subplot(1, 2, 2)\nplt.title(\"Generated Monet Photo\")\nplt.imshow(to_monet[0] * 0.5 + 0.5)\nplt.show()","metadata":{"id":"pnVRyur88sYb","executionInfo":{"status":"ok","timestamp":1710595674782,"user_tz":-120,"elapsed":4748,"user":{"displayName":"דור בוכריס","userId":"12233430204438784905"}},"outputId":"c7445793-aeb3-4c40-f754-a5bd087c810e","execution":{"iopub.status.busy":"2024-03-19T20:32:05.446232Z","iopub.execute_input":"2024-03-19T20:32:05.446958Z","iopub.status.idle":"2024-03-19T20:32:08.139935Z","shell.execute_reply.started":"2024-03-19T20:32:05.446926Z","shell.execute_reply":"2024-03-19T20:32:08.138880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**The CycleGAN Model**</font>\n\n**We will now create the cyclegan model itself, using the generators and discriminators from before. The class will derive from Model - so we can run the function \"fit\" that Model class have.**\n\n**The model transforms a photo to a monet painting and back to photo. The loss is the difference between the original photo and the result of those transfers. The more similar, the better and less loss.**","metadata":{"id":"Vwlgn7Vk8sYb"}},{"cell_type":"code","source":"class CycleGan(keras.Model):\n    def __init__(\n        self,\n        monet_generator,\n        photo_generator,\n        monet_discriminator,\n        photo_discriminator,\n        lambda_cycle=10,\n    ):\n        super(CycleGan, self).__init__()\n        # Initialize the CycleGAN model with generators, discriminators, and hyperparameters\n        self.m_gen = monet_generator  # Monet generator\n        self.p_gen = photo_generator  # Photo generator\n        self.m_disc = monet_discriminator  # Monet discriminator\n        self.p_disc = photo_discriminator  # Photo discriminator\n        self.lambda_cycle = lambda_cycle  # Used for the cycle consistency loss weight, bigger value => making it more significate\n\n    def compile(\n        self,\n        m_gen_optimizer,\n        p_gen_optimizer,\n        m_disc_optimizer,\n        p_disc_optimizer,\n        gen_loss_fn,\n        disc_loss_fn,\n        cycle_loss_fn,\n        identity_loss_fn,\n        SSIM_loss_fn,\n        style_loss_fn\n    ):\n        super(CycleGan, self).compile()\n        # Configure the optimizers and loss functions for training\n        self.m_gen_optimizer = m_gen_optimizer  # Monet generator optimizer\n        self.p_gen_optimizer = p_gen_optimizer  # Photo generator optimizer\n        self.m_disc_optimizer = m_disc_optimizer  # Monet discriminator optimizer\n        self.p_disc_optimizer = p_disc_optimizer  # Photo discriminator optimizer\n        self.gen_loss_fn = gen_loss_fn  # Generator loss function\n        self.disc_loss_fn = disc_loss_fn  # Discriminator loss function\n        self.cycle_loss_fn = cycle_loss_fn  # Cycle consistency loss function\n        self.identity_loss_fn = identity_loss_fn  # Identity loss function\n        self.style_loss_fn = style_loss_fn # style loss function\n        self.SSIM_loss_fn = SSIM_loss_fn # SSIM loss function\n\n    def train_step(self, batch_data):\n        # Unpack the batch data into real Monet and real Photo images\n        real_monet, real_photo = batch_data\n\n        # Use GradientTape to record gradients for training\n        with tf.GradientTape(persistent=True) as tape:\n            # Forward pass through generators and discriminators\n            fake_monet = self.m_gen(real_photo, training=True)\n            cycled_photo = self.p_gen(fake_monet, training=True)\n            fake_photo = self.p_gen(real_monet, training=True)\n            cycled_monet = self.m_gen(fake_photo, training=True)\n            same_monet = self.m_gen(real_monet, training=True)\n            same_photo = self.p_gen(real_photo, training=True)\n            disc_real_monet = self.m_disc(real_monet, training=True)\n            disc_real_photo = self.p_disc(real_photo, training=True)\n            disc_fake_monet = self.m_disc(fake_monet, training=True)\n            disc_fake_photo = self.p_disc(fake_photo, training=True)\n\n            # Calculate losses\n            # TODO: check if style transfer was used apropriatly!(maybe move to a diffrent var)\n            monet_gen_loss = self.gen_loss_fn(disc_fake_monet)\n            photo_gen_loss = self.gen_loss_fn(disc_fake_photo)\n            total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle) \\\n              + self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle) \\\n              + self.SSIM_loss_fn(real_monet, cycled_monet, self.lambda_cycle) \\\n              + self.SSIM_loss_fn(real_photo, cycled_photo, self.lambda_cycle)\n            total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle) + self.style_loss_fn(real_monet, cycled_monet, self.lambda_cycle)\n            total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle)\n            monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)\n            photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)\n\n        # Calculate gradients for generators and discriminators\n        monet_generator_gradients = tape.gradient(total_monet_gen_loss, self.m_gen.trainable_variables)\n        photo_generator_gradients = tape.gradient(total_photo_gen_loss, self.p_gen.trainable_variables)\n        monet_discriminator_gradients = tape.gradient(monet_disc_loss, self.m_disc.trainable_variables)\n        photo_discriminator_gradients = tape.gradient(photo_disc_loss, self.p_disc.trainable_variables)\n\n        # Apply gradients to the optimizer\n        self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients, self.m_gen.trainable_variables))\n        self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients, self.p_gen.trainable_variables))\n        self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients, self.m_disc.trainable_variables))\n        self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients, self.p_disc.trainable_variables))\n\n        # Return the losses for monitoring\n        return {\n            \"monet_gen_loss\": total_monet_gen_loss,\n            \"photo_gen_loss\": total_photo_gen_loss,\n            \"monet_disc_loss\": monet_disc_loss,\n            \"photo_disc_loss\": photo_disc_loss\n        }\n","metadata":{"id":"sFNKdzRw8sYb","execution":{"iopub.status.busy":"2024-03-19T20:32:22.309078Z","iopub.execute_input":"2024-03-19T20:32:22.309719Z","iopub.status.idle":"2024-03-19T20:32:22.328613Z","shell.execute_reply.started":"2024-03-19T20:32:22.309685Z","shell.execute_reply":"2024-03-19T20:32:22.327504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**Loss Functions**</font>\n\n**Four types of loss: discriminator loss, generator loss, cycle loss and identity loss.**\n\n**The discriminator loss compares real images to a matrix of 1s and fake images to a matrix of 0s - because the output of the discriminator model is a matrix that higher values means it thinks the image is more real.**","metadata":{"id":"H1dMn01N8sYc"}},{"cell_type":"code","source":"\ndef discriminator_loss(real, generated):\n    real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(real), real)\n    generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.zeros_like(generated), generated)\n\n    total_disc_loss = real_loss + generated_loss\n\n    return total_disc_loss * 0.5\n","metadata":{"id":"oX5-xLmP8sYc","execution":{"iopub.status.busy":"2024-03-19T20:32:25.805333Z","iopub.execute_input":"2024-03-19T20:32:25.806175Z","iopub.status.idle":"2024-03-19T20:32:25.811851Z","shell.execute_reply.started":"2024-03-19T20:32:25.806140Z","shell.execute_reply":"2024-03-19T20:32:25.810856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The generator loss compares the discrimanotr output of the generated image to a matrix of 1s to find the loss - because the closer to values of 1s, the best it will fool the discriminator model.**","metadata":{"id":"GIyP4aWZ8sYc"}},{"cell_type":"code","source":"\ndef generator_loss(generated):\n    return tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(generated), generated)\n\n","metadata":{"id":"KXP1eVtc8sYc","execution":{"iopub.status.busy":"2024-03-19T20:32:29.714302Z","iopub.execute_input":"2024-03-19T20:32:29.714673Z","iopub.status.idle":"2024-03-19T20:32:29.719894Z","shell.execute_reply.started":"2024-03-19T20:32:29.714627Z","shell.execute_reply":"2024-03-19T20:32:29.718905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The cycle loss making sure the original photo and the transformed photo be similar to one another. So it calculates the consistency of the cycle, by finding the average of their difference.**","metadata":{"id":"G8hMrohL8sYc"}},{"cell_type":"code","source":"def calc_cycle_loss(real_image, cycled_image, LAMBDA):\n    loss1 = tf.reduce_mean(tf.abs(real_image - cycled_image))\n\n    return LAMBDA * loss1","metadata":{"id":"tyIlpf6W8sYd","execution":{"iopub.status.busy":"2024-03-19T20:32:31.537899Z","iopub.execute_input":"2024-03-19T20:32:31.538291Z","iopub.status.idle":"2024-03-19T20:32:31.543367Z","shell.execute_reply.started":"2024-03-19T20:32:31.538263Z","shell.execute_reply":"2024-03-19T20:32:31.542210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The identity loss compares the image itself with its generator - meaning it is making sure the output of the generator will be the same as the original image.**","metadata":{"id":"X0b5Xye98sYd"}},{"cell_type":"code","source":"def identity_loss(real_image, same_image, LAMBDA):\n    loss = tf.reduce_mean(tf.abs(real_image - same_image))\n    return LAMBDA * 0.5 * loss","metadata":{"id":"k-lje4qN8sYd","execution":{"iopub.status.busy":"2024-03-19T20:32:33.338121Z","iopub.execute_input":"2024-03-19T20:32:33.338490Z","iopub.status.idle":"2024-03-19T20:32:33.343428Z","shell.execute_reply.started":"2024-03-19T20:32:33.338463Z","shell.execute_reply":"2024-03-19T20:32:33.342268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ssim_loss(y_true, y_pred, LAMBDA):\n    \"\"\"\n    Compute the Structural Similarity Index (SSIM) loss between y_true and y_pred.\n    \"\"\"\n    # SSIM parameters\n    K1 = 0.01\n    K2 = 0.03\n\n    # Flatten the images\n    y_true_flat = tf.keras.backend.flatten(y_true)\n    y_pred_flat = tf.keras.backend.flatten(y_pred)\n\n    # Compute mean of y_true and y_pred\n    mu_true = tf.reduce_mean(y_true_flat)\n    mu_pred = tf.reduce_mean(y_pred_flat)\n\n    # Compute variance of y_true and y_pred\n    var_true = tf.reduce_mean(tf.square(y_true_flat - mu_true))\n    var_pred = tf.reduce_mean(tf.square(y_pred_flat - mu_pred))\n\n    # Compute covariance\n    covar = tf.reduce_mean((y_true_flat - mu_true) * (y_pred_flat - mu_pred))\n\n    # Compute SSIM\n    numerator = (2 * mu_true * mu_pred + K1) * (2 * covar + K2)\n    denominator = (tf.square(mu_true) + tf.square(mu_pred) + K1) * (var_true + var_pred + K2)\n    ssim = numerator / denominator\n\n    return LAMBDA * (1 - ssim)  # Return 1 - SSIM as the loss","metadata":{"execution":{"iopub.status.busy":"2024-03-19T20:32:36.955841Z","iopub.execute_input":"2024-03-19T20:32:36.956628Z","iopub.status.idle":"2024-03-19T20:32:36.964239Z","shell.execute_reply.started":"2024-03-19T20:32:36.956595Z","shell.execute_reply":"2024-03-19T20:32:36.963357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gram_matrix(x):\n    \"\"\"\n    Calculate the Gram matrix for a given tensor.\n    \"\"\"\n    features = tf.keras.backend.batch_flatten(tf.keras.backend.permute_dimensions(x, (2, 0, 1, 3)))\n    gram = tf.keras.backend.dot(features, tf.keras.backend.transpose(features))\n    return gram","metadata":{"execution":{"iopub.status.busy":"2024-03-19T20:32:50.536755Z","iopub.execute_input":"2024-03-19T20:32:50.537366Z","iopub.status.idle":"2024-03-19T20:32:50.542698Z","shell.execute_reply.started":"2024-03-19T20:32:50.537334Z","shell.execute_reply":"2024-03-19T20:32:50.541609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"channels = 3\nsize = IMAGE_SIZE[0] * IMAGE_SIZE[1]\n\ndef style_loss(style, generated, LAMBDA):\n    \"\"\"\n    Calculate the style loss between the style reference and generated images.\n    \"\"\"\n    S = gram_matrix(style)\n    G = gram_matrix(generated)\n\n    return LAMBDA * 0.5 * tf.keras.backend.sum(tf.keras.backend.square(S - G)) / (4.0 * (channels ** 2) * (size ** 2))","metadata":{"execution":{"iopub.status.busy":"2024-03-19T20:32:52.785168Z","iopub.execute_input":"2024-03-19T20:32:52.785536Z","iopub.status.idle":"2024-03-19T20:32:52.792564Z","shell.execute_reply.started":"2024-03-19T20:32:52.785507Z","shell.execute_reply":"2024-03-19T20:32:52.791705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**CycleGAN Training**</font>\n\n**We defined all the models and losses, loaded the data - we can compile and fit our model.**","metadata":{"id":"jrql4ol68sYd"}},{"cell_type":"markdown","source":"**Creating our optimizers**","metadata":{"id":"lVlqoICQGEHf"}},{"cell_type":"code","source":"monet_generator_optimizer = tf.keras.optimizers.Adam(3e-4, beta_1=0.5)\nphoto_generator_optimizer = tf.keras.optimizers.Adam(3e-4, beta_1=0.5)\n\nmonet_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\nphoto_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)","metadata":{"id":"ZVuZ6Glm8sYd","execution":{"iopub.status.busy":"2024-03-19T20:32:56.444264Z","iopub.execute_input":"2024-03-19T20:32:56.444926Z","iopub.status.idle":"2024-03-19T20:32:56.458510Z","shell.execute_reply.started":"2024-03-19T20:32:56.444893Z","shell.execute_reply":"2024-03-19T20:32:56.457713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Compiling our model**","metadata":{"id":"WmAz8mLdGJoU"}},{"cell_type":"code","source":"cycle_gan_model = CycleGan(\n    monet_generator, photo_generator, monet_discriminator, photo_discriminator\n)\n\ncycle_gan_model.compile(\n    m_gen_optimizer = monet_generator_optimizer,\n    p_gen_optimizer = photo_generator_optimizer,\n    m_disc_optimizer = monet_discriminator_optimizer,\n    p_disc_optimizer = photo_discriminator_optimizer,\n    gen_loss_fn = generator_loss,\n    disc_loss_fn = discriminator_loss,\n    cycle_loss_fn = calc_cycle_loss,\n    identity_loss_fn = identity_loss,\n    style_loss_fn = style_loss,\n    SSIM_loss_fn = ssim_loss\n)","metadata":{"id":"oc8eRasz8sYd","execution":{"iopub.status.busy":"2024-03-19T20:32:59.075701Z","iopub.execute_input":"2024-03-19T20:32:59.076416Z","iopub.status.idle":"2024-03-19T20:32:59.091317Z","shell.execute_reply.started":"2024-03-19T20:32:59.076385Z","shell.execute_reply":"2024-03-19T20:32:59.090279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**And finally, training the CycleGAN**","metadata":{"id":"RT4N3FKIGO7g"}},{"cell_type":"code","source":"# with strategy.scope():\ncycle_gan_model.fit(\n    tf.data.Dataset.zip((monet_ds, photo_ds)),\n    epochs=35\n)","metadata":{"id":"WyGNCSgR8sYd","outputId":"482c9829-770b-4f41-c0d4-48833fbcb703","execution":{"iopub.status.busy":"2024-03-19T20:33:03.068194Z","iopub.execute_input":"2024-03-19T20:33:03.068846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**Visualizing Results**</font>","metadata":{"id":"pfMaiOmX8sYi"}},{"cell_type":"code","source":"_, ax = plt.subplots(10, 2, figsize=(24, 24))\nfor i, img in enumerate(photo_ds.take(10)):\n    prediction = monet_generator(img, training=False)[0].numpy()\n    prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n    img = (img[0] * 127.5 + 127.5).numpy().astype(np.uint8)\n\n    ax[i, 0].imshow(img)\n    ax[i, 1].imshow(prediction)\n    ax[i, 0].set_title(\"Input Photo\")\n    ax[i, 1].set_title(\"Monet-esque\")\n    ax[i, 0].axis(\"off\")\n    ax[i, 1].axis(\"off\")\nplt.show()","metadata":{"id":"1mvsx3SL8sYi","executionInfo":{"status":"ok","timestamp":1709404520584,"user_tz":-120,"elapsed":9019,"user":{"displayName":"Rom Ben Aharon","userId":"04194826883711623755"}},"outputId":"8f314c61-7364-48fd-c031-e0cb8ac0200d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='#DC7633'>**Creating Submission File**</font>\n","metadata":{"id":"2z-Q1SAZ8sYi"}},{"cell_type":"code","source":"import PIL\n! mkdir ../images","metadata":{"id":"7lYojHriOouY","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nfor img in photo_ds:\n    prediction = monet_generator(img, training=False)[0].numpy()\n    prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n    im = PIL.Image.fromarray(prediction)\n    im = im.resize((256, 256))\n    im.save(\"../images/\" + str(i) + \".jpg\")\n    i += 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")\n","metadata":{"id":"xhQyxnRU8sYi","trusted":true},"execution_count":null,"outputs":[]}]}