{"cells":[{"metadata":{},"cell_type":"markdown","source":"I have use @amyjang's wonderful code to run this on a TPU. I've modified the generator to have a resnet structure with residual blocks described in the original paper.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\n\ntry:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print('Device:', tpu.master())\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n \nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))\nprint('Monet TF records: ', len(MONET_FILENAMES))\n\nPHOTO_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))\nprint('Photo TF records: ', len(PHOTO_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [256, 256]\n\ndef decode_image(image):\n image = tf.image.decode_jpeg(image, channels=3)\n image = (tf.cast(image, tf.float32) / 127.5) - 1\n image = tf.reshape(image, [*IMAGE_SIZE, 3])\n return image\n\ndef read_tfrecord(example):\n tfrecord_format = {\n \"image_name\": tf.io.FixedLenFeature([], tf.string),\n \"image\": tf.io.FixedLenFeature([], tf.string),\n \"target\": tf.io.FixedLenFeature([], tf.string)\n }\n example = tf.io.parse_single_example(example, tfrecord_format)\n image = decode_image(example['image'])\n return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n dataset = tf.data.TFRecordDataset(filenames)\n dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"monet_ds = load_dataset(MONET_FILENAMES, labeled = True).batch(1)\nphoto_ds = load_dataset(PHOTO_FILENAMES, labeled = True).batch(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_monet = next(iter(monet_ds))\nexample_photo = next(iter(photo_ds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(1,2,1)\nplt.title('Photo')\nplt.imshow(example_photo[0] *0.5 + 0.5)\n\nplt.subplot(1,2,2)\nplt.title('Monet')\nplt.imshow(example_monet[0]*0.5 + 0.5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building up the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Weights initializer for the layers.\nkernel_init = keras.initializers.RandomNormal(mean = 0.0, stddev = 0.02)\n# Gamma initializer for instance normalization.\ngamma_init = keras.initializers.RandomNormal(mean = 0.0, stddev = 0.02)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConstantPadding2D(layers.Layer):\n '''Implements Reflection Padding as a layer.\n\n Args:\n padding(tuple): Amount of padding for the\n spatial dimensions.\n\n Returns:\n A padded tensor with the same type as the input tensor.\n '''\n \n def __init__(self, padding = (1,1), **kwargs):\n self.padding = padding\n super(ConstantPadding2D, self).__init__(**kwargs)\n \n def call(self, input_tensor, mask = None):\n padding_width, padding_height = self.padding\n padding_tensor = [\n [0,0],\n [padding_height, padding_height],\n [padding_width, padding_width],\n [0,0]\n ]\n return tf.pad(input_tensor, padding_tensor, mode = 'CONSTANT')\n \ndef residual_block(\n x,\n activation,\n kernel_initializer = kernel_init,\n kernel_size = (3,3),\n strides = (1,1),\n padding = 'valid',\n gamma_initializer = gamma_init,\n use_bias = False,\n):\n dim = x.shape[-1]\n input_tensor = x\n \n x = ConstantPadding2D()(input_tensor)\n x = layers.Conv2D(\n dim,\n kernel_size,\n strides = strides,\n kernel_initializer = kernel_initializer,\n padding = padding,\n use_bias = use_bias,\n )(x)\n x = tfa.layers.InstanceNormalization(gamma_initializer = gamma_initializer)(x)\n x = activation(x)\n x = ConstantPadding2D()(x)\n x = layers.Conv2D(\n dim,\n kernel_size,\n strides = strides,\n kernel_initializer = kernel_initializer,\n padding = padding,\n use_bias = use_bias,\n )(x)\n x = tfa.layers.InstanceNormalization(gamma_initializer = gamma_initializer)(x)\n x = layers.add([input_tensor, x])\n return x\n\ndef downsample(\n x,\n filters,\n activation,\n kernel_initializer = kernel_init,\n kernel_size = (3,3),\n strides = (2,2),\n padding = 'same',\n gamma_initializer = gamma_init,\n use_bias = False,\n):\n x = layers.Conv2D(\n filters,\n kernel_size,\n strides = strides,\n kernel_initializer = kernel_initializer,\n padding = padding,\n use_bias = use_bias,\n )(x)\n if activation:\n x = activation(x)\n return x\n\ndef upsample(\n x,\n filters,\n activation,\n kernel_initializer = kernel_init,\n kernel_size = (3,3),\n strides = (2,2),\n padding = 'same',\n gamma_initializer = gamma_init,\n use_bias = False,\n):\n x = layers.Conv2DTranspose(\n filters,\n kernel_size,\n strides = strides,\n padding = padding,\n kernel_initializer = kernel_initializer,\n use_bias = use_bias,\n )(x)\n x = tfa.layers.InstanceNormalization(gamma_initializer = gamma_initializer)(x)\n if activation:\n x = activation(x)\n return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building the generator","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def resnet_generator(\n filters = 64,\n num_downsampling_blocks = 2,\n num_residual_blocks = 9,\n num_upsampling_blocks = 2,\n gamma_initializer = gamma_init,\n name = None,\n):\n img_input = layers.Input(shape = (256,256,3), name = name + '_img_input')\n x = ConstantPadding2D(padding = (3,3))(img_input)\n x = layers.Conv2D(filters, (7,7), kernel_initializer = kernel_init, use_bias = False)(x)\n x = tfa.layers.InstanceNormalization(gamma_initializer = gamma_initializer)(x)\n x = layers.Activation('relu')(x)\n \n #Downsampling\n for _ in range(num_downsampling_blocks):\n filters *= 2\n x = downsample(x, filters = filters, activation = layers.Activation('relu'))\n \n #Residual blocks\n for _ in range(num_residual_blocks):\n x = residual_block(x, activation = layers.Activation('relu'))\n \n #Upsampling\n for _ in range(num_upsampling_blocks):\n filters //= 2\n x = upsample(x, filters = filters, activation = layers.Activation('relu'))\n \n #final block\n x = ConstantPadding2D(padding = (3,3))(x)\n x = layers.Conv2D(3, (7,7), padding = 'valid')(x)\n x = layers.Activation('tanh')(x)\n \n model = keras.models.Model(img_input, x, name = name)\n return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building the discriminator","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def discriminator(\n filters = 64,\n kernel_initializer = kernel_init,\n num_downsampling = 3,\n name = None,\n):\n img_input = layers.Input(shape = (256,256,3), name = name + '_img_input')\n x = layers.Conv2D(\n filters,\n (4,4),\n strides = (2,2),\n padding = 'same',\n kernel_initializer = kernel_initializer,\n )(img_input)\n x = layers.LeakyReLU(0.2)(x)\n \n num_filters = filters\n for num_downsample_block in range(3):\n num_filters *= 2\n if num_downsample_block < 2:\n x = downsample(\n x,\n filters = num_filters,\n activation = layers.LeakyReLU(0.2),\n kernel_size = (4,4),\n strides = (2,2),\n )\n else:\n x = downsample(\n x,\n filters = num_filters,\n activation = layers.LeakyReLU(0.2),\n kernel_size = (4,4),\n strides = (1,1),\n )\n x = layers.Conv2D(1, (4,4), strides = (1,1), padding = 'same', kernel_initializer = kernel_initializer)(x)\n model = keras.models.Model(img_input, x, name = name)\n return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building the GAN","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n monet_generator = resnet_generator(name = 'generator_monet') #transforms photos to monet\n photo_generator = resnet_generator(name = 'generator_photo') #transforms monet to photos\n monet_discriminator = discriminator(name = 'discriminator_monet') #differentiates between real monets and generated monets\n photo_discriminator = discriminator(name = 'discriminator_photo') #differentiates between real photos and generated photos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"to_monet = monet_generator(example_photo)\n\nplt.subplot(1,2,1)\nplt.title('Original Photo')\nplt.imshow(example_photo[0]*0.5 + 0.5)\n\nplt.subplot(1,2,2)\nplt.title('Generated Monet')\nplt.imshow(to_monet[0]*0.5 + 0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CycleGAN(keras.Model):\n def __init__(\n self,\n monet_generator,\n photo_generator,\n monet_discriminator,\n photo_discriminator,\n lambda_cycle = 10.0,\n lambda_identity = 0.5\n ):\n super(CycleGAN, self).__init__()\n self.m_gen = monet_generator\n self.p_gen = photo_generator\n self.m_disc = monet_discriminator\n self.p_disc = photo_discriminator\n self.lambda_cycle = lambda_cycle\n self.lambda_identity = lambda_identity\n \n def compile(\n self,\n m_gen_optimizer,\n p_gen_optimizer,\n m_disc_optimizer,\n p_disc_optimizer,\n gen_loss_fn,\n disc_loss_fn,\n cycle_loss_fn,\n identity_loss_fn\n ):\n super(CycleGAN, self).compile()\n self.m_gen_optimizer = m_gen_optimizer\n self.p_gen_optimizer = p_gen_optimizer\n self.m_disc_optimizer = m_disc_optimizer\n self.p_disc_optimizer = p_disc_optimizer\n self.gen_loss_fn = gen_loss_fn\n self.disc_loss_fn = disc_loss_fn\n self.cycle_loss_fn = cycle_loss_fn\n self.identity_loss_fn = identity_loss_fn\n \n def train_step(self, batch):\n real_monet, real_photo = batch\n with tf.GradientTape(persistent = True) as tape:\n #photo --> monet --> photo\n fake_monet = self.m_gen(real_photo, training = True)\n cycled_photo = self.p_gen(fake_monet, training = True)\n \n #monet --> photo --> monet\n fake_photo = self.p_gen(real_monet, training = True)\n cycled_monet = self.m_gen(fake_photo, training = True)\n \n #identity mapping\n same_monet = self.m_gen(real_monet, training = True)\n same_photo = self.p_gen(real_photo, training = True)\n \n #discriminator output\n disc_real_monet = self.m_disc(real_monet, training = True)\n disc_fake_monet = self.m_disc(fake_monet, training = True)\n \n disc_real_photo = self.p_disc(real_photo, training = True)\n disc_fake_photo = self.p_disc(fake_photo, training = True)\n \n #generator cycle losses\n cycled_loss_monet = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle)\n cycled_loss_photo = self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle)\n total_cycle_loss = cycled_loss_monet + cycled_loss_photo\n \n #generator losses\n monet_gen_loss = self.gen_loss_fn(disc_fake_monet)\n photo_gen_loss = self.gen_loss_fn(disc_fake_photo)\n \n #identity losses\n monet_id_loss = self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle, self.lambda_identity)\n photo_id_loss = self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle, self.lambda_identity)\n total_id_loss = monet_id_loss + photo_id_loss\n \n #total generator loss\n total_monet_gen_loss = monet_gen_loss + total_cycle_loss + total_id_loss\n total_photo_gen_loss = photo_gen_loss + total_cycle_loss + total_id_loss\n \n #discriminator losses\n monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)\n photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)\n \n #calculate gradients\n monet_generator_gradients = tape.gradient(total_monet_gen_loss, self.m_gen.trainable_variables)\n photo_generator_gradients = tape.gradient(total_photo_gen_loss, self.p_gen.trainable_variables)\n \n monet_discriminator_gradients = tape.gradient(monet_disc_loss, self.m_disc.trainable_variables)\n photo_discriminator_gradients = tape.gradient(photo_disc_loss, self.p_disc.trainable_variables)\n \n self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients, self.m_gen.trainable_variables))\n self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients, self.p_gen.trainable_variables))\n self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients, self.m_disc.trainable_variables))\n self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients, self.p_disc.trainable_variables))\n \n return {\n 'monet_gen_loss': total_monet_gen_loss,\n 'photo_gen_loss': total_photo_gen_loss,\n 'monet_disc_loss': monet_disc_loss,\n 'photo_disc_loss': photo_disc_loss\n }\n ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n def discriminator_loss(real, generated):\n real_loss = keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(real), real)\n fake_loss = keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.zeros_like(generated), generated)\n total_disc_loss = real_loss + fake_loss\n return total_disc_loss*0.5 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n def generator_loss(generated):\n return keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(generated), generated)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n def cycle_loss(real_image, cycled_image, LAMBDA):\n loss = tf.reduce_mean(tf.abs(real_image - cycled_image))\n return loss*LAMBDA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n def identity_loss(real_image, same_image, LAMBDA1, LAMBDA2):\n loss = tf.reduce_mean(tf.abs(real_image - same_image))\n return loss*LAMBDA1*LAMBDA2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n m_gen_optimizer = keras.optimizers.Adam(learning_rate = 2e-4, beta_1 = 0.5)\n p_gen_optimizer = keras.optimizers.Adam(learning_rate = 2e-4, beta_1 = 0.5)\n m_disc_optimizer = keras.optimizers.Adam(learning_rate = 2e-4, beta_1 = 0.5)\n p_disc_optimizer = keras.optimizers.Adam(learning_rate = 2e-4, beta_1 = 0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n cycle_gan_model = CycleGAN(monet_generator, photo_generator, monet_discriminator, photo_discriminator)\n cycle_gan_model.compile(\n m_gen_optimizer = m_gen_optimizer,\n p_gen_optimizer = p_gen_optimizer,\n m_disc_optimizer = m_disc_optimizer,\n p_disc_optimizer = p_disc_optimizer,\n gen_loss_fn = generator_loss,\n disc_loss_fn = discriminator_loss,\n cycle_loss_fn = cycle_loss,\n identity_loss_fn = identity_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = cycle_gan_model.fit(tf.data.Dataset.zip((monet_ds, photo_ds)), epochs = 50 )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualising predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"_, ax = plt.subplots(6,2,figsize = (20,20))\nfor i, img in enumerate(photo_ds.take(6)):\n prediction = monet_generator(img, training = False)[0].numpy()\n prediction = prediction*0.5 + 0.5\n img = (img[0]*0.5 + 0.5).numpy()\n ax[i, 0].imshow(img)\n ax[i, 1].imshow(prediction)\n ax[i, 0].set_title('Input Photo')\n ax[i, 1].set_title('Generated Photo')\n ax[i, 0].axis('off')\n ax[i, 1].axis('off')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import PIL\n! mkdir ../images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 1\nfor img in photo_ds:\n prediction = monet_generator(img, training=False)[0].numpy()\n prediction = (prediction*127.5 + 127.5).astype(np.uint8)\n im = PIL.Image.fromarray(prediction)\n im.save(\"../images/\" + str(i) + \".jpg\")\n i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}