{"metadata":{"colab":{"provenance":[],"gpuType":"T4"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"accelerator":"GPU","kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21755,"databundleVersionId":1475600,"sourceType":"competition"},{"sourceId":17742,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":14777}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\nUsing GANs to Create Art - Monet CycleGan Project\n\nPreface\n\n- The project was created as part of \"Advanced Deep Learning\" of our deep learning specialization.\n- The model takes part of the \"I’m Something of a Painter Myself\" Kaggle competition: https://www.kaggle.com/competitions/gan-getting-started\n- The model was created using the following resources:\n - Based on our on research.\n - The knowledge we gained from Dr Moshe Butman the facultiy.\n - Information from a known research paper named \"Unpaired Image-to-Image Translation using Cycle-Consistent Adversarial Networks\". The research was published by Berkeley AI Research (BAIR) laboratory, Link: https://arxiv.org/pdf/1703.10593.pdf .\n - Information from a research paper \"CycleGAN with Better Cycles\" by Tongzhou Wang and Yihan Lin, Link:https://www.tongzhouwang.info/better_cycles/report.pdf .\n - The model was trained on RTX 4090 GPU.\n\n
\nWritten by:\n- Itamar Kirsch\n- Liel Pargamentnik\n- Chen Shiri\n- Matan Ofri","metadata":{"id":"Zv7MOwnYIAA1"}},{"cell_type":"markdown","source":"# Part 1 - Libraries and Requirements","metadata":{"id":"VGyqOSIqH2KN"}},{"cell_type":"markdown","source":"## 1.1 Importing the libraries","metadata":{"id":"7azQxIHVICQR"}},{"cell_type":"code","source":"import tensorflow as tf\nimport keras as ker\nimport numpy as np\nimport os\nimport cv2\nimport glob\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import initializers, layers, Input, backend, Model, losses, optimizers\nfrom tensorflow.keras.layers import Conv2D, Conv2DTranspose, ReLU, Add, Activation ,Concatenate, LeakyReLU,GroupNormalization\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import BinaryCrossentropy\nfrom tensorflow.keras.initializers import RandomNormal\nfrom tensorflow.keras.metrics import BinaryAccuracy\nfrom tensorflow.keras.utils import register_keras_serializable\nfrom tensorflow.keras.callbacks import Callback\nfrom matplotlib import pyplot as plt\nfrom numpy import load ,ones ,zeros ,asarray\nfrom numpy.random import randint\nimport cv2 as cv2_imshow\nimport matplotlib.pyplot as plt\nfrom random import random\nimport random as rand\n#import tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.utils import plot_model\n#Saving the output\nfrom tqdm import tqdm\nimport zipfile","metadata":{"id":"vE7N_Uj4BAgc","outputId":"844c5d6d-093f-4266-b26e-dbf9cf8dbedd","execution":{"iopub.status.busy":"2024-03-18T21:06:07.618680Z","iopub.execute_input":"2024-03-18T21:06:07.619336Z","iopub.status.idle":"2024-03-18T21:06:20.380539Z","shell.execute_reply.started":"2024-03-18T21:06:07.619299Z","shell.execute_reply":"2024-03-18T21:06:20.379559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.2 InstanceNormalization Class\nhttps://github.com/tensorflow/addons/blob/3380b3ccf906f20dbef49c05906e6b6dabf479cf/tensorflow_addons/layers/normalizations.py#L297","metadata":{}},{"cell_type":"code","source":"class InstanceNormalization(GroupNormalization):\n \"\"\"Instance normalization layer.\n\n Instance Normalization is an specific case of ```GroupNormalization```since\n it normalizes all features of one channel. The Groupsize is equal to the\n channel size. Empirically, its accuracy is more stable than batch norm in a\n wide range of small batch sizes, if learning rate is adjusted linearly\n with batch sizes.\n\n Arguments\n axis: Integer, the axis that should be normalized.\n epsilon: Small float added to variance to avoid dividing by zero.\n center: If True, add offset of `beta` to normalized tensor.\n If False, `beta` is ignored.\n scale: If True, multiply by `gamma`.\n If False, `gamma` is not used.\n beta_initializer: Initializer for the beta weight.\n gamma_initializer: Initializer for the gamma weight.\n beta_regularizer: Optional regularizer for the beta weight.\n gamma_regularizer: Optional regularizer for the gamma weight.\n beta_constraint: Optional constraint for the beta weight.\n gamma_constraint: Optional constraint for the gamma weight.\n\n Input shape\n Arbitrary. Use the keyword argument `input_shape`\n (tuple of integers, does not include the samples axis)\n when using this layer as the first layer in a model.\n\n Output shape\n Same shape as input.\n\n References\n - [Instance Normalization: The Missing Ingredient for Fast Stylization]\n (https://arxiv.org/abs/1607.08022)\n \"\"\"\n\n def __init__(self, **kwargs):\n if \"groups\" in kwargs:\n logging.warning(\"The given value for groups will be overwritten.\")\n\n kwargs[\"groups\"] = -1\n super().__init__(**kwargs)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:06:32.641621Z","iopub.execute_input":"2024-03-18T21:06:32.642257Z","iopub.status.idle":"2024-03-18T21:06:32.649515Z","shell.execute_reply.started":"2024-03-18T21:06:32.642226Z","shell.execute_reply":"2024-03-18T21:06:32.648581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3 Hyperparameters ","metadata":{"id":"0UyUxfA4IgtE"}},{"cell_type":"code","source":"img_height = 320\nimg_width = 320\nimg_channels = 3\nbatch_size = 4\nepochs = 100\nlearning_rate=0.0002\nbeta_val=0.5\nimg_shape = (img_width, img_height, img_channels)\nnum_of_resnet_blocks = 12\nprint(\"Done\") ","metadata":{"id":"2_-_ZdYtEy8u","execution":{"iopub.status.busy":"2024-03-18T21:06:55.280050Z","iopub.execute_input":"2024-03-18T21:06:55.280762Z","iopub.status.idle":"2024-03-18T21:06:55.286224Z","shell.execute_reply.started":"2024-03-18T21:06:55.280732Z","shell.execute_reply":"2024-03-18T21:06:55.285208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Part 2- Implementing the blocks, Layers","metadata":{"id":"977uC5cWLNie"}},{"cell_type":"markdown","source":"## 2.1 Reflection Padding Layer\nWe have used the code from keras documentation in order to implement the reflection padding layers. \nLink: https://keras.io/examples/generative/cyclegan/","metadata":{"id":"8xbzilNGO6DL"}},{"cell_type":"code","source":"class ReflectionPadding2D(layers.Layer):\n def __init__(self, padding=(1, 1), **kwargs):\n self.padding = tuple(padding)\n super().__init__(**kwargs)\n\n # Override in Layer\n def call(self, input_tensor, mask=None):\n padding_width, padding_height = self.padding\n padding_tensor = [\n [0, 0],\n [padding_height, padding_height],\n [padding_width, padding_width],\n [0, 0],\n ]\n return tf.pad(input_tensor, padding_tensor, mode=\"REFLECT\")\nprint(\"Done\") ","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:06:58.109946Z","iopub.execute_input":"2024-03-18T21:06:58.110312Z","iopub.status.idle":"2024-03-18T21:06:58.117726Z","shell.execute_reply.started":"2024-03-18T21:06:58.110283Z","shell.execute_reply":"2024-03-18T21:06:58.116650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Implementing the blocks for the generator\n\n- The **Encoder Block** is designed to downsample the input layer, extracting and compressing features through convolution, and optionally applying instance normalization for consistent feature scaling, followed by a ReLU activation for non-linearity.\n- The **ResNet Block** enhances feature representation without increasing the depth or complexity significantly by applying a series of convolutions within a residual learning framework, allowing the input to be directly added to the output, which helps mitigate the vanishing gradient problem in deep networks.\n- The **Decoder Block** performs the inverse operation of the encoder, upsampling the input layer and reconstructing the image or feature map dimensions through transposed convolution, with optional instance normalization and ReLU activation to maintain feature consistency and non-linearity.\n- The **Resized Convolution Block** increases the spatial dimensions of the input layer through bilinear resizing and symmetric padding, followed by convolutions to refine the upscaled features, optionally applying instance normalization and ReLU activation to ensure feature normalization and introduce non-linearity.","metadata":{"id":"B-Y_2DvZkrJ7"}},{"cell_type":"code","source":"conv_initializer = tf.random_normal_initializer(mean=0.0, stddev=0.02)\ngamma_initializer = tf.keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n# Encoder Block\ndef encoder_block(input_layer, filters, size=3, strides=2, apply_instancenorm=True, activation=ker.layers.ReLU(), name='encoder'):\n enc_layer = ker.layers.Conv2D(filters, size,\n strides=strides,\n padding='same',\n use_bias=False,\n kernel_initializer=conv_initializer,\n name=f'block-{name}')(input_layer)\n\n if apply_instancenorm:\n enc_layer = InstanceNormalization(gamma_initializer=gamma_initializer)(enc_layer)\n enc_layer = activation(enc_layer)\n return enc_layer\n\n# Resnet Block\ndef resnet_block(input_layer, size=3, strides=1, name='resnet'):\n filters = input_layer.shape[-1]\n res_layer = ReflectionPadding2D()(input_layer)\n res_layer = ker.layers.Conv2D(filters, size,\n strides=strides,\n padding='valid',\n use_bias=False,\n kernel_initializer=conv_initializer,\n name=f'block-{name}_1')(res_layer)\n res_layer = ker.layers.ReLU()(res_layer)\n res_layer = ReflectionPadding2D()(res_layer)\n res_layer = ker.layers.Conv2D(filters, size,\n strides=strides,\n padding='valid',\n use_bias=False,\n kernel_initializer=conv_initializer,\n name=f'block-{name}_2')(res_layer)\n res_layer = ker.layers.Add()([res_layer, input_layer])\n return res_layer\n\n# Decoder Block\ndef decoder_block(input_layer, filters, size=3, strides=2, apply_instancenorm=True, name='decoder'):\n dec_layer = ker.layers.Conv2DTranspose(filters, size,\n strides=strides,\n padding='same',\n use_bias=False,\n kernel_initializer=conv_initializer,\n name=f'block-{name}')(input_layer)\n\n if apply_instancenorm:\n dec_layer = InstanceNormalization(gamma_initializer=gamma_initializer)(dec_layer)\n dec_layer = ker.layers.ReLU()(dec_layer)\n return dec_layer\n\n# Resized convolution Block\ndef resized_conv_block(input_layer, filters, size=3, strides=1, apply_instancenorm=True, name='resized_conv'):\n rescon_layer = tf.image.resize(images=input_layer, method='bilinear', size=(input_layer.shape[1]*2, input_layer.shape[2]*2))\n\n rescon_layer = tf.pad(rescon_layer, [[0, 0], [1, 1], [1, 1], [0, 0]], \"SYMMETRIC\")\n rescon_layer = ker.layers.Conv2D(filters, size, strides=strides, padding='valid', use_bias=False)(rescon_layer)\n rescon_layer = ker.layers.Conv2D(filters, size,\n strides=strides,\n padding='same',\n use_bias=False,\n kernel_initializer=conv_initializer,\n name=f'block-{name}')(rescon_layer)\n\n if apply_instancenorm:\n rescon_layer = InstanceNormalization(gamma_initializer=gamma_initializer)(rescon_layer)\n rescon_layer = ker.layers.ReLU()(rescon_layer)\n return rescon_layer\nprint(\"Done\")","metadata":{"id":"hcCul_SOTPFD","execution":{"iopub.status.busy":"2024-03-18T21:07:40.486767Z","iopub.execute_input":"2024-03-18T21:07:40.487456Z","iopub.status.idle":"2024-03-18T21:07:40.507750Z","shell.execute_reply.started":"2024-03-18T21:07:40.487420Z","shell.execute_reply":"2024-03-18T21:07:40.506629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Part 3 - Implementing the Generator and Descriminator Models","metadata":{"id":"jM1fsR0CNRpb"}},{"cell_type":"markdown","source":"## 3.1 Generator Model","metadata":{"id":"emRelwk4JPAc"}},{"cell_type":"code","source":"def generator(height=img_height, width=img_width, channels=img_channels, resnet_blocks=num_of_resnet_blocks):\n # Input:\n inputs = ker.layers.Input(shape=[height, width, channels], name='input_image')\n\n # Encoder:\n # 1 - (batch_size, 256, 256, 64)\n enc_1 = encoder_block(inputs, 64, 7, 1, apply_instancenorm=False, activation=ker.layers.ReLU(), name='enc_1')\n # 2 - (batch_size, 128, 128, 128)\n enc_2 = encoder_block(enc_1, 128, 3, 2, apply_instancenorm=True, activation=ker.layers.ReLU(), name='enc_2')\n # 3 - (batch_size, 64, 64, 256)\n enc_3 = encoder_block(enc_2, 256, 3, 2, apply_instancenorm=True, activation=ker.layers.ReLU(), name='enc_3')\n\n # Resnet:\n # The value of \"resnet_blocks\" times, every Layer is (batch_size, 64, 64, 256)\n res = enc_3\n for i in range(resnet_blocks):\n res = resnet_block(res, 3, 1, name=f'resnet_block_{i+1}')\n\n # Decoder:\n skip_con = ker.layers.Concatenate(name='enc_dec_skip_1')([res, enc_3])\n # 1 - (batch_size, 128, 128, 128)\n dec_1 = decoder_block(skip_con, 128, 3, 2, apply_instancenorm=True, name='block_1')\n skip_con = ker.layers.Concatenate(name='enc_dec_skip_2')([dec_1, enc_2])\n # 2 - (batch_size, 256, 256, 64)\n dec_2 = decoder_block(skip_con, 64, 3, 2, apply_instancenorm=True, name='block_2')\n skip_con = ker.layers.Concatenate(name='enc_dec_skip_3')([dec_2, enc_1])\n\n # Output:\n # 1 - (batch_size, 256, 256, 3)\n outputs = last = ker.layers.Conv2D(channels, 7,\n strides=1, padding='same',\n kernel_initializer=conv_initializer,\n use_bias=False,\n activation='tanh',\n name='decoder_output_block')(skip_con)\n\n generator = Model(inputs, outputs)\n return generator\nprint(\"Done\")","metadata":{"id":"8GwQCyCxTM2F","execution":{"iopub.status.busy":"2024-03-18T21:08:06.206308Z","iopub.execute_input":"2024-03-18T21:08:06.206720Z","iopub.status.idle":"2024-03-18T21:08:06.218875Z","shell.execute_reply.started":"2024-03-18T21:08:06.206691Z","shell.execute_reply":"2024-03-18T21:08:06.217788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generator Model Layers Summary","metadata":{"id":"oCxRxYl9bqi-"}},{"cell_type":"code","source":"plot_model(generator(), to_file='/root/GeneratorModel.jpg', show_shapes=True)","metadata":{"id":"8wRNhLImb0gr","outputId":"842c14fe-bf97-4163-e6d8-fae2294526b2","execution":{"iopub.status.busy":"2024-03-18T21:08:16.794623Z","iopub.execute_input":"2024-03-18T21:08:16.795323Z","iopub.status.idle":"2024-03-18T21:08:18.763701Z","shell.execute_reply.started":"2024-03-18T21:08:16.795292Z","shell.execute_reply":"2024-03-18T21:08:18.762043Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Discriminator Model","metadata":{"id":"DXXssKpoJIBB"}},{"cell_type":"code","source":"def discriminator(height=img_height, width=img_width, channels=img_channels):\n # Input:\n inputs = ker.layers.Input(shape=[height, width, channels], name='input_image')\n inputs_patch = tf.keras.layers.RandomCrop(height=70, width=70, name='input_image_patch')(inputs)\n\n # Encoder:\n # 1 - (batch_size, 128, 128, 64)\n x = encoder_block(inputs_patch, 64, 4, 2, apply_instancenorm=False, activation=ker.layers.LeakyReLU(0.2), name='block_1')\n # 2 - (batch_size, 64, 64, 128)\n x = encoder_block(x, 128, 4, 2, apply_instancenorm=True, activation=ker.layers.LeakyReLU(0.2), name='block_2')\n # 3 - (batch_size, 32, 32, 256)\n x = encoder_block(x, 256, 4, 2, apply_instancenorm=True, activation=ker.layers.LeakyReLU(0.2), name='block_3')\n # 4 - (batch_size, 32, 32, 512)\n x = encoder_block(x, 512, 4, 1, apply_instancenorm=True, activation=ker.layers.LeakyReLU(0.2), name='block_4')\n\n # Output:\n # 1 - (batch_size, 29, 29, 1)\n outputs = ker.layers.Conv2D(1, 4, strides=1, padding='valid', kernel_initializer=conv_initializer)(x)\n\n discriminator = Model(inputs, outputs)\n return discriminator\nprint(\"Done\") ","metadata":{"id":"ikM_0sT-FrXq","execution":{"iopub.status.busy":"2024-03-18T21:08:49.960599Z","iopub.execute_input":"2024-03-18T21:08:49.960955Z","iopub.status.idle":"2024-03-18T21:08:49.970364Z","shell.execute_reply.started":"2024-03-18T21:08:49.960928Z","shell.execute_reply":"2024-03-18T21:08:49.969290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Discriminator Model Layers Summary","metadata":{"id":"oWSYjGHFcLks"}},{"cell_type":"code","source":"plot_model(discriminator(), to_file='/root/DiscriminatorModel.jpg', show_shapes=True)","metadata":{"id":"uvDjVuGtcQgW","outputId":"98796f0e-6a17-40af-9d4e-53c1b68449bb","execution":{"iopub.status.busy":"2024-03-18T21:08:57.862546Z","iopub.execute_input":"2024-03-18T21:08:57.863415Z","iopub.status.idle":"2024-03-18T21:08:58.032082Z","shell.execute_reply.started":"2024-03-18T21:08:57.863380Z","shell.execute_reply":"2024-03-18T21:08:58.031122Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Part 4 - Data and Augmentation","metadata":{"id":"I9d1RRY4i5Bu"}},{"cell_type":"markdown","source":"## 4.1 Loading the data\nThe model was trained on 7038 \"normal\" images from the competition's dataset and an additional 1072 drawings by Monet from the following repository: https://efrosgans.eecs.berkeley.edu/cyclegan/datasets/monet2photo.zip.","metadata":{"id":"6algYvM62PPg"}},{"cell_type":"code","source":"!mkdir /root/data/\n!mkdir /root/data/images/\n!mkdir /root/data/monet/\n!cp -r /kaggle/input/gan-getting-started/photo_jpg /root/data/images/photo_jpg/\n!cp -r /kaggle/input/gan-getting-started/monet_jpg /root/data/monet/monet_jpg/\n\nnormal_photos = '/root/data/images/'\nmonet_photos = '/root/data/monet/'","metadata":{"id":"gc4XnHpHnSmU","execution":{"iopub.status.busy":"2024-03-18T21:33:24.925240Z","iopub.execute_input":"2024-03-18T21:33:24.926261Z","iopub.status.idle":"2024-03-18T21:33:35.981391Z","shell.execute_reply.started":"2024-03-18T21:33:24.926220Z","shell.execute_reply":"2024-03-18T21:33:35.980081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.2 Initialization and Augmentation","metadata":{"id":"0fUw21o-2S8a"}},{"cell_type":"code","source":"def monet_prep(image):\n image = tf.image.resize(image, size=[*(img_height, img_width)], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n image = tf.image.random_crop(image, size=[*(img_height, img_width, img_channels)])\n image = tf.cast(image, dtype=tf.float32)\n image = (image / 127.5) - 1.0\n return image\n\ndef normal_prep(image):\n image = image.astype(np.float32) / 255.0\n image = (image - 0.5) * 2\n return image\n\ndatagen_normal = ImageDataGenerator(\n horizontal_flip=True,\n preprocessing_function=normal_prep,\n)\n\ndatagen_monet = ImageDataGenerator(\n horizontal_flip=True,\n preprocessing_function=monet_prep,\n)\n\nnormal_image_generator = datagen_normal.flow_from_directory(\n normal_photos,\n target_size=(img_height, img_width),\n batch_size=batch_size,\n class_mode='sparse',\n shuffle=True\n)\n\nmonet_image_generator = datagen_monet.flow_from_directory(\n monet_photos,\n target_size=(img_height, img_width),\n batch_size=batch_size,\n class_mode='sparse',\n shuffle=False\n)\n\nsteps = ((monet_image_generator.samples)//batch_size)","metadata":{"id":"ClRBkoUiYwN5","outputId":"01e92de0-adab-4c2b-e80a-ec6bedc4c2cd","execution":{"iopub.status.busy":"2024-03-18T21:34:40.353889Z","iopub.execute_input":"2024-03-18T21:34:40.354275Z","iopub.status.idle":"2024-03-18T21:34:40.549048Z","shell.execute_reply.started":"2024-03-18T21:34:40.354241Z","shell.execute_reply":"2024-03-18T21:34:40.548114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Part 5 - CycleGan Model\n- The model definition, written using the Keras documentation: https://keras.io/examples/generative/cyclegan/","metadata":{"id":"ZJMLijOuDTu3"}},{"cell_type":"markdown","source":"## 5.1 Creating the model","metadata":{"id":"2Hu96dDTN3vi"}},{"cell_type":"code","source":"# monet_generator transforms normal photos to Monet-esque paintings:\nmonet_generator = generator(height=None, width=None, resnet_blocks=num_of_resnet_blocks)\n# normal_generatort ransforms Monet paintings to be more like normal photos:\nnormal_generator = generator(height=None, width=None, resnet_blocks=num_of_resnet_blocks)\n# monet_discriminator differentiates real Monet paintings and generated Monet paintings:\nmonet_discriminator = discriminator(height=None, width=None)\n# normal_discriminator differentiates real photos and generated photos:\nnormal_discriminator = discriminator(height=None, width=None)\n\nclass CycleGan(Model):\n def __init__(\n self,\n monet_generator,\n normal_generator,\n monet_discriminator,\n normal_discriminator,\n lambda_cycle=10,\n ):\n super(CycleGan, self).__init__()\n self.monet_gen = monet_generator\n self.normal_gen = normal_generator\n self.monet_disc = monet_discriminator\n self.normal_disc = normal_discriminator\n self.lambda_cycle = lambda_cycle\n\n # Override in Model\n def compile(\n self,\n monet_gen_optimizer,\n normal_gen_optimizer,\n monet_disc_optimizer,\n normal_disc_optimizer,\n gen_loss,\n disc_loss,\n cycle_loss, # monet -> normal -> monet || normal -> monet -> normal\n identity_loss\n ):\n super(CycleGan, self).compile()\n self.monet_gen_optimizer = monet_gen_optimizer\n self.normal_gen_optimizer = normal_gen_optimizer\n self.monet_disc_optimizer = monet_disc_optimizer\n self.normal_disc_optimizer = normal_disc_optimizer\n self.gen_loss = calc_generator_loss\n self.disc_loss = calc_discriminator_loss\n self.cycle_loss = calc_cycle_loss\n self.identity_loss = calc_identity_loss\n\n # Override in Model\n def train_step(self, batch_data):\n real_monet, real_normal = batch_data\n\n with tf.GradientTape(persistent=True) as tape:\n # Normal photo to monet and back to normal photo\n fake_monet = self.monet_gen(real_normal, training=True)\n cycled_normal = self.normal_gen(fake_monet, training=True)\n\n # Monet to normal photo and back to Monet\n fake_normal = self.normal_gen(real_monet, training=True)\n cycled_monet = self.monet_gen(fake_normal, training=True)\n\n # generating itself\n complex_monet = self.monet_gen(real_monet, training=True)\n complex_normal = self.normal_gen(real_normal, training=True)\n\n # discriminator used to check, inputing real images\n disc_real_monet = self.monet_disc(real_monet, training=True)\n disc_real_normal = self.normal_disc(real_normal, training=True)\n\n # discriminator used to check, inputing fake images\n disc_fake_monet = self.monet_disc(fake_monet, training=True)\n disc_fake_normal = self.normal_disc(fake_normal, training=True)\n\n # evaluates generator loss\n monet_gen_loss = self.gen_loss(disc_fake_monet)\n normal_gen_loss = self.gen_loss(disc_fake_normal)\n\n # evaluates total cycle consistency loss\n total_cycle_loss = self.cycle_loss(real_monet, cycled_monet, self.lambda_cycle) + self.cycle_loss(real_normal, cycled_normal, self.lambda_cycle)\n\n # evaluates total generator loss\n total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss(real_monet, complex_monet, self.lambda_cycle)\n total_normal_gen_loss = normal_gen_loss + total_cycle_loss + self.identity_loss(real_normal, complex_normal, self.lambda_cycle)\n\n # evaluates discriminator loss\n monet_disc_loss = self.disc_loss(disc_real_monet, disc_fake_monet)\n normal_disc_loss = self.disc_loss(disc_real_normal, disc_fake_normal)\n\n # Calculate the gradients for generator and discriminator\n monet_generator_gradients = tape.gradient(total_monet_gen_loss, self.monet_gen.trainable_variables)\n self.monet_gen_optimizer.apply_gradients(zip(monet_generator_gradients, self.monet_gen.trainable_variables))\n\n normal_generator_gradients = tape.gradient(total_normal_gen_loss, self.normal_gen.trainable_variables)\n self.normal_gen_optimizer.apply_gradients(zip(normal_generator_gradients, self.normal_gen.trainable_variables))\n\n monet_discriminator_gradients = tape.gradient(monet_disc_loss, self.monet_disc.trainable_variables)\n self.monet_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients, self.monet_disc.trainable_variables))\n\n normal_discriminator_gradients = tape.gradient(normal_disc_loss, self.normal_disc.trainable_variables)\n self.normal_disc_optimizer.apply_gradients(zip(normal_discriminator_gradients, self.normal_disc.trainable_variables))\n\n\n return {'monet_gen_loss': total_monet_gen_loss,\n 'photo_gen_loss': total_normal_gen_loss,\n 'monet_disc_loss': monet_disc_loss,\n 'photo_disc_loss': normal_disc_loss\n }\nprint(\"Done\") ","metadata":{"id":"2Y1Wc9FLDSFu","execution":{"iopub.status.busy":"2024-03-18T21:34:58.773792Z","iopub.execute_input":"2024-03-18T21:34:58.774355Z","iopub.status.idle":"2024-03-18T21:34:59.394364Z","shell.execute_reply.started":"2024-03-18T21:34:58.774325Z","shell.execute_reply":"2024-03-18T21:34:59.393317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.2 Calculating Loss Values","metadata":{"id":"15hHq6tKCCsU"}},{"cell_type":"markdown","source":"About the loss:\n- **Discriminator Loss** (calc_discriminator_loss): The discriminator's goal is to distinguish between real and generated (fake) images.\nThe discriminator loss consists of two parts:\n 1. **Real Loss**: Measures how well the discriminator is able to recognize real images. It compares the discriminator's predictions for real images to an array of ones (since real images should ideally yield a score close to 1).\n\n 2. **Generated Loss**: Assesses the discriminator's ability to identify generated images. It compares the discriminator's predictions for generated images to an array of zeros (since generated images should ideally yield a score close to 0).\n\n - **The total discriminator loss** is the average of these two losses, encouraging the discriminator to correctly classify real and generated images.\n\n- **Generator Loss** (calc_generator_loss): The generator's objective is to create images that are indistinguishable from real images. The generator loss measures how well the generated images fool the discriminator by comparing the discriminator's predictions for generated images to an array of ones. A lower generator loss indicates that the generator is producing more realistic images.\n\n- **Cycle Loss** (calc_cycle_loss): This loss is specific to CycleGANs or similar models where the goal includes not just one-way translation (e.g., normal to Monet) but also a reverse translation (e.g., Monet back to normal) to ensure consistency. The cycle loss measures the difference between an original image and a \"cycled\" image (an image that has been translated and then translated back). It encourages the preservation of content through the translation processes. The loss in the original paper is MAE, we choose to follow the loss of the second paper \"CycleGAN with Better Cycles\" that uses MSE.\n\n- **Identity Loss** (calc_identity_loss): This loss is used to preserve color composition between the input and output images. When an image from the target domain is fed into the generator, the identity loss measures how much the generator changes this image. The idea is that if the generator is shown an image already in the target style, it should not change the image much.\n","metadata":{"id":"ELg5vQMo1pea"}},{"cell_type":"code","source":"def calc_discriminator_loss(real, generated):\n real_loss = losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE)(tf.ones_like(real), real)\n generated_loss = losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE)(tf.zeros_like(generated), generated)\n return (real_loss + generated_loss) * 0.5\n\ndef calc_generator_loss(generated):\n return losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE)(tf.ones_like(generated), generated)\n\ndef calc_cycle_loss(real_image, cycled_image, Lambda):\n loss = tf.reduce_mean(tf.square(real_image - cycled_image))\n return Lambda * loss\n\ndef calc_identity_loss(real_image, complex_image, Lambda):\n loss = tf.reduce_mean(tf.abs(real_image - complex_image))\n return Lambda * loss * 0.5\n\nprint(\"Done\")","metadata":{"id":"s8xPctSpXdz-","execution":{"iopub.status.busy":"2024-03-18T21:35:21.904024Z","iopub.execute_input":"2024-03-18T21:35:21.904802Z","iopub.status.idle":"2024-03-18T21:35:21.913290Z","shell.execute_reply.started":"2024-03-18T21:35:21.904769Z","shell.execute_reply":"2024-03-18T21:35:21.912256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.3 Optimizers and Compilation\n- Based on the first article \"Unpaired Image-to-Image Translation using Cycle-Consistent Adversarial Networks\", we choose to use the optimizer Adam with a change of two default values: learning rate, beta 1.","metadata":{"id":"d-cw0xGwN87q"}},{"cell_type":"code","source":"monet_generator_optimizer = optimizers.Adam(learning_rate=learning_rate, beta_1=beta_val)\nnormal_generator_optimizer = optimizers.Adam(learning_rate=learning_rate, beta_1=beta_val)\n\nmonet_discriminator_optimizer = optimizers.Adam(learning_rate=learning_rate, beta_1=beta_val)\nnormal_discriminator_optimizer = optimizers.Adam(learning_rate=learning_rate, beta_1=beta_val)\n\ngan_model = CycleGan(monet_generator, normal_generator, monet_discriminator, normal_discriminator)\ngan_model.compile(monet_gen_optimizer=monet_generator_optimizer,\n normal_gen_optimizer=normal_generator_optimizer,\n monet_disc_optimizer=monet_discriminator_optimizer,\n normal_disc_optimizer=normal_discriminator_optimizer,\n gen_loss=calc_generator_loss,\n disc_loss=calc_identity_loss,\n cycle_loss=calc_cycle_loss,\n identity_loss=calc_discriminator_loss\n )\ngan_model.built = True\nprint(\"Done\")","metadata":{"id":"72I8h_SpXkgV","execution":{"iopub.status.busy":"2024-03-18T21:35:50.650526Z","iopub.execute_input":"2024-03-18T21:35:50.650921Z","iopub.status.idle":"2024-03-18T21:35:50.676767Z","shell.execute_reply.started":"2024-03-18T21:35:50.650890Z","shell.execute_reply":"2024-03-18T21:35:50.675819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.4 Reduce Linearly Learning Rate\n- This function reduces the learning rate at every epoch starting from the defined epoch.\n- We choose to start reducing the learning rate at around epoke 50. The quality of the photos created by Monet Generator has started decresing.","metadata":{"id":"I_G3rVaIEfDk"}},{"cell_type":"code","source":"\"\"\"\nselected_epoch = 45\nclass LinearLRScheduler(tf.keras.callbacks.Callback):\n def __init__(self, optimizer, initial_lr, epoch_to_reduce):\n super(LinearLRScheduler, self).__init__()\n self.optimizer = optimizer\n self.initial_lr = initial_lr\n self.epoch_to_reduce = epoch_to_reduce\n\n # Override in Callback\n def on_epoch_begin(self, epoch, logs=None):\n print(\"current lr is\",print(K.eval(self.optimizer.learning_rate)))\n if epoch >= self.epoch_to_reduce:\n new_lr = self.initial_lr * (1 - (epoch - self.epoch_to_reduce) / (self.params['epochs'] - self.epoch_to_reduce))\n tf.keras.backend.set_value(self.optimizer.learning_rate, new_lr)\n print(\"changed lr is\",print(K.eval(self.optimizer.learning_rate)))\nlinear_lr_scheduler_monet_gen = LinearLRScheduler(monet_generator_optimizer, initial_lr=learning_rate, epoch_to_reduce=selected_epoch)\nlinear_lr_scheduler_photo_gen = LinearLRScheduler(normal_generator_optimizer, initial_lr=learning_rate, epoch_to_reduce=selected_epoch)\nlinear_lr_scheduler_monet_disc = LinearLRScheduler(monet_discriminator_optimizer, initial_lr=learning_rate, epoch_to_reduce=selected_epoch)\nlinear_lr_scheduler_photo_disc = LinearLRScheduler(normal_discriminator_optimizer, initial_lr=learning_rate, epoch_to_reduce=selected_epoch)\n\"\"\"","metadata":{"id":"5qGCatyx3Gct","execution":{"iopub.status.busy":"2024-03-18T20:20:11.726644Z","iopub.execute_input":"2024-03-18T20:20:11.727503Z","iopub.status.idle":"2024-03-18T20:20:11.736323Z","shell.execute_reply.started":"2024-03-18T20:20:11.727465Z","shell.execute_reply":"2024-03-18T20:20:11.735257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Part 5 -Training of the model","metadata":{"id":"aVyY4H6QFH1j"}},{"cell_type":"code","source":"\"\"\"\ndef get_from_directory(monet_generator, normal_generator, steps_per_epoch):\n while True:\n monet_images = monet_image_generator.next()[0]\n normal_images = normal_image_generator.next()[0]\n yield (monet_images, normal_images)\n\nplotter = GANMonitor()\ncheckpoint_filepath = \"/content/MonetModels11/cyclegan_checkpoints.{epoch:03d}\"\nmodel_checkpoint_callback = ker.callbacks.ModelCheckpoint(filepath=checkpoint_filepath, save_weights_only=True)\n\ngan_model.fit(\n get_from_directory(monet_image_generator, normal_image_generator, steps),\n epochs=epochs,\n steps_per_epoch=steps,\n callbacks=[plotter, linear_lr_scheduler_monet_gen, linear_lr_scheduler_photo_gen,\n linear_lr_scheduler_monet_disc, linear_lr_scheduler_photo_disc, model_checkpoint_callback],\n)\n\"\"\"","metadata":{"id":"xIJo8SahXoc7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Part 6 - Testing the model","metadata":{"id":"eMd9_XwUTXD4"}},{"cell_type":"markdown","source":"## 6.1 - Loading the model weights\nIt is possible to load the weights file that is attached to the Kaggle project.","metadata":{"id":"ZlT6CGHNUdtY"}},{"cell_type":"code","source":"gan_model.load_weights(\"/kaggle/input/monet/keras/v1/1/FinalWights.h5\")","metadata":{"id":"0-_elQrOUqGw","execution":{"iopub.status.busy":"2024-03-18T21:40:19.990813Z","iopub.execute_input":"2024-03-18T21:40:19.991213Z","iopub.status.idle":"2024-03-18T21:40:21.604764Z","shell.execute_reply.started":"2024-03-18T21:40:19.991181Z","shell.execute_reply":"2024-03-18T21:40:21.603966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6.2 - Testing the model on normal generic images","metadata":{"id":"qRFzk0keUuFi"}},{"cell_type":"markdown","source":"**Note:** The model was written to create images in the size of 320x320 as instructed the faculty Lecturer","metadata":{"id":"odviUqpvSrAY"}},{"cell_type":"code","source":"\n\nsrcimg = cv2.imread(\"/kaggle/input/gan-getting-started/photo_jpg/009d534136.jpg\")\nimg = cv2.resize(srcimg, (img_height, img_width))\nimg = img.astype('float32')\nimg = (img - 127.5) / 127.5\nimg = np.expand_dims(img, axis=0)\noutimg = gan_model.monet_gen(img, training=False)\noutimg = (outimg.numpy() * 127.5 + 127.5).astype(np.uint8)\nplt.figure(figsize=(10, 5))\nplt.subplot(1, 2, 1)\nplt.title('Normal Image')\n# Change \"srcimg\" to \"cropped_img\":\nplt.imshow(cv2.cvtColor(srcimg, cv2.COLOR_BGR2RGB))\nplt.axis('off')\nplt.subplot(1, 2, 2)\nplt.title('Monet Image')\nplt.imshow(cv2.cvtColor(outimg[0], cv2.COLOR_BGR2RGB))\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:40:49.825369Z","iopub.execute_input":"2024-03-18T21:40:49.825738Z","iopub.status.idle":"2024-03-18T21:40:53.046026Z","shell.execute_reply.started":"2024-03-18T21:40:49.825709Z","shell.execute_reply":"2024-03-18T21:40:53.045023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and Creating submission file","metadata":{}},{"cell_type":"code","source":"import zipfile\n\ndirectory = \"/kaggle/input/gan-getting-started/photo_jpg/\"\noutput_directory = \"/kaggle/working/\"\nzip_filename = os.path.join(output_directory, \"images.zip\")\n\nwith zipfile.ZipFile(zip_filename, 'w') as myzip:\n all_photos = os.listdir(directory)\n for filename in all_photos:\n try:\n srcimg = cv2.imread(os.path.join(directory, filename))\n if srcimg is None:\n continue\n\n img = cv2.resize(srcimg, (256, 256))\n img = img.astype('float32')\n img = (img - 127.5) / 127.5\n img = np.expand_dims(img, axis=0)\n outimg = gan_model.monet_gen(img, training=False)\n outimg = (outimg.numpy() * 127.5 + 127.5).astype(np.uint8)\n output_filename = f\"processed_{filename}\"\n\n temp_path = os.path.join(output_directory, output_filename)\n plt.figure(figsize=(2.56, 2.56), dpi=100) \n plt.imshow(cv2.cvtColor(outimg[0], cv2.COLOR_BGR2RGB))\n plt.axis('off')\n plt.subplots_adjust(left=0, right=1, top=1, bottom=0) \n plt.savefig(temp_path, bbox_inches='tight', pad_inches=0)\n plt.close()\n\n if os.path.exists(temp_path):\n myzip.write(temp_path, arcname=output_filename)\n os.remove(temp_path)\n else:\n print(f\"Failed to save image: {temp_path}\")\n except Exception as e:\n print(f\"An error occurred processing {filename}: {e}\")\n\nwith zipfile.ZipFile(zip_filename, 'r') as check_zip:\n print(\"Contents of the zip file:\", check_zip.namelist())","metadata":{"id":"x3w4LdQ5S1Lk","execution":{"iopub.status.busy":"2024-03-18T22:15:01.530288Z","iopub.execute_input":"2024-03-18T22:15:01.531436Z","iopub.status.idle":"2024-03-18T22:23:26.545302Z","shell.execute_reply.started":"2024-03-18T22:15:01.531398Z","shell.execute_reply":"2024-03-18T22:23:26.544317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning Files","metadata":{}},{"cell_type":"code","source":"!rm -rf /root/data","metadata":{"execution":{"iopub.status.busy":"2024-03-18T22:25:56.454268Z","iopub.execute_input":"2024-03-18T22:25:56.454659Z","iopub.status.idle":"2024-03-18T22:25:57.679837Z","shell.execute_reply.started":"2024-03-18T22:25:56.454632Z","shell.execute_reply":"2024-03-18T22:25:57.678462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results:\n\n","metadata":{}}]}