{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook implement a CycleGAN with two-objective dualhead discriminator and DiffAugment.\nThe kernel source is : https://www.kaggle.com/code/unfriendlyai/two-objective-discriminator\n\nMy own contributions are comments adding all along the notebook to enable a better understanding when re-using this code, adapting the batch size to fit the capability of a TPU (thus decreasing the computing-time per epoch-step by a factor of 30 in comparisons of the initial kernel) and finally I made a function to easily test various learning_rate/epoch_number scenarios.","metadata":{}},{"cell_type":"markdown","source":"Install/Import the required libraries","metadata":{}},{"cell_type":"code","source":"from __future__ import absolute_import, division, print_function\nimport sys\nprint(\"Python version\")\nprint (sys.version)\nprint(\"Version info.\")\nprint (sys.version_info)\n\n#!{sys.executable} -m pip install --upgrade pip\n#!{sys.executable} -m pip install numpy\n#!{sys.executable} -m pip install pandas\n#!{sys.executable} -m pip install tensorflow_addons\n#!pip install PyDrive\n#! gdown --id 11dzPv9LxegCDJTNE0RVCJD9hi2sxjQb4\n#! mkdir ~/.kaggle\n#! cp kaggle.json ~/.kaggle/\n#! chmod 600 ~/.kaggle/kaggle.json\n#! kaggle competitions download -c gan-getting-started\n#! unzip gan-getting-started\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport re\nimport os\nimport gzip, pickle\nfrom scipy import linalg\nimport pathlib\nimport urllib\nimport warnings\nfrom tqdm import tqdm\nimport os.path\nfrom os import path\nfrom PIL import Image\ntry:\n  from kaggle_datasets import KaggleDatasets\nexcept:\n  pass\nimport PIL\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nfrom argparse import ArgumentParser, ArgumentDefaultsHelpFormatter\nimport imageio\nfrom torch.autograd import Variable\nfrom torch.nn.functional import adaptive_avg_pool2d\nfrom functools import reduce\nimport cv2\nimport pandas as pd\nimport shutil\ntry:\n    from torchvision.models.utils import load_state_dict_from_url\nexcept ImportError:\n    from torch.utils.model_zoo import load_url as load_state_dict_from_url","metadata":{"executionInfo":{"elapsed":30368,"status":"ok","timestamp":1648212331942,"user":{"displayName":"Jeff Vache","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"02900104580587288419"},"user_tz":-60},"id":"HT9esKZzJ7pN","outputId":"d4e4c508-f732-482e-d236-cd792df47069","execution":{"iopub.status.busy":"2022-03-30T13:16:14.625068Z","iopub.execute_input":"2022-03-30T13:16:14.625766Z","iopub.status.idle":"2022-03-30T13:16:14.646592Z","shell.execute_reply.started":"2022-03-30T13:16:14.625728Z","shell.execute_reply":"2022-03-30T13:16:14.645655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Connecting to the TPU ","metadata":{"id":"_5c0_bDKETMd"}},{"cell_type":"code","source":"try:\n    #Get the TPU address\n    tpu_resolver = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu_resolver.master())\nexcept ValueError:\n    tpu_resolver = None\n\nif tpu_resolver:\n    #Connecting to the TPU\n    tf.config.experimental_connect_to_cluster(tpu_resolver)\n    #TPU initialisazion\n    tf.tpu.experimental.initialize_tpu_system(tpu_resolver)\n    #TPU's trategy instanciation\n    strategy = tf.distribute.TPUStrategy(tpu_resolver)\n    \nelse:\n    strategy = tf.distribute.get_strategy() \n#Show replica number\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n#Set AUTOTUNE variable for optimization\nAUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"id":"20K3IZzuBlzj","executionInfo":{"status":"ok","timestamp":1648212386648,"user_tz":-60,"elapsed":54719,"user":{"displayName":"Jeff Vache","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"02900104580587288419"}},"outputId":"9cb3a855-1f72-404c-a9ef-9e46fb1c897c","execution":{"iopub.status.busy":"2022-03-30T13:16:14.648574Z","iopub.execute_input":"2022-03-30T13:16:14.649091Z","iopub.status.idle":"2022-03-30T13:16:14.663289Z","shell.execute_reply.started":"2022-03-30T13:16:14.648989Z","shell.execute_reply":"2022-03-30T13:16:14.662444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Set BATCH_SIZE and STEPS_PER_EPOCH variables xdepending on the available accelerator","metadata":{}},{"cell_type":"code","source":"PHOTO_DATASET_SIZE = 7028\n\nif tpu_resolver:\n    BATCH_SIZE = 1024*strategy.num_replicas_in_sync\n    STEPS_PER_EPOCH = (PHOTO_DATASET_SIZE // BATCH_SIZE + 1)*10\nelse:\n    BATCH_SIZE = 32\n    STEPS_PER_EPOCH = (PHOTO_DATASET_SIZE // BATCH_SIZE + 1)*10","metadata":{"id":"lkdL7sP30ndU","execution":{"iopub.status.busy":"2022-03-30T13:16:14.664313Z","iopub.execute_input":"2022-03-30T13:16:14.667407Z","iopub.status.idle":"2022-03-30T13:16:14.681236Z","shell.execute_reply.started":"2022-03-30T13:16:14.667356Z","shell.execute_reply":"2022-03-30T13:16:14.679943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Data loading and preprocessing","metadata":{"id":"H9Fh67dsZcwk"}},{"cell_type":"markdown","source":"Creating filepaths for accessing TFRecord (Tensorflow Records) format data","metadata":{"id":"lbyg6VBh-oLF"}},{"cell_type":"code","source":"datafolder = KaggleDatasets().get_gcs_path()\n\nMONET_FILENAMES = tf.io.gfile.glob(str(datafolder + '/monet_tfrec/*.tfrec'))\nprint('Monet TFRecord Files:', len(MONET_FILENAMES))\n\nPHOTO_FILENAMES = tf.io.gfile.glob(str(datafolder + '/photo_tfrec/*.tfrec'))\nprint('Photo TFRecord Files:', len(PHOTO_FILENAMES))","metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1648212386649,"user":{"displayName":"Jeff Vache","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"02900104580587288419"},"user_tz":-60},"id":"G3Y4fcybJ7pR","outputId":"c6eb75ee-61c9-4d67-b409-6228f113cb11","execution":{"iopub.status.busy":"2022-03-30T13:16:14.682686Z","iopub.execute_input":"2022-03-30T13:16:14.683177Z","iopub.status.idle":"2022-03-30T13:16:15.722222Z","shell.execute_reply.started":"2022-03-30T13:16:14.683103Z","shell.execute_reply":"2022-03-30T13:16:15.721301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Functions to decode image from TFRecord to JPEG format and to load an image-dataset","metadata":{"id":"kop3hyIRDoR_"}},{"cell_type":"code","source":"IMAGE_SIZE = [256, 256]\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    return image\n\ndef load_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return dataset","metadata":{"id":"WCHjEACO-zRi","execution":{"iopub.status.busy":"2022-03-30T13:16:15.723603Z","iopub.execute_input":"2022-03-30T13:16:15.724337Z","iopub.status.idle":"2022-03-30T13:16:15.735017Z","shell.execute_reply.started":"2022-03-30T13:16:15.724288Z","shell.execute_reply":"2022-03-30T13:16:15.734224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Showing some samples of images","metadata":{"id":"Cqt60TtAMODT"}},{"cell_type":"code","source":"monet_ds = load_dataset(MONET_FILENAMES).batch(1)\nphoto_ds = load_dataset(PHOTO_FILENAMES).batch(1)\nexample_monet = next(iter(monet_ds))\nexample_photo = next(iter(photo_ds))\n\nplt.subplot(121)\nplt.title('Photo')\nplt.imshow(example_photo[0] * 0.5 + 0.5)\n\nplt.subplot(122)\nplt.title('Monet')\nplt.imshow(example_monet[0] * 0.5 + 0.5)","metadata":{"id":"pb2XDnKXMONM","executionInfo":{"status":"ok","timestamp":1648212388486,"user_tz":-60,"elapsed":1541,"user":{"displayName":"Jeff Vache","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"02900104580587288419"}},"outputId":"22e329c4-5743-4408-c6a0-a1a0fd090e1c","execution":{"iopub.status.busy":"2022-03-30T13:16:15.736341Z","iopub.execute_input":"2022-03-30T13:16:15.737009Z","iopub.status.idle":"2022-03-30T13:16:17.251703Z","shell.execute_reply.started":"2022-03-30T13:16:15.736969Z","shell.execute_reply":"2022-03-30T13:16:17.250661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading and preparing dataset function","metadata":{}},{"cell_type":"code","source":"def get_gan_dataset(monet_files, photo_files, repeat=True, shuffle=True, batch_size=1,nb_repeat=None):\n    monet_ds = load_dataset(monet_files)\n    photo_ds = load_dataset(photo_files)\n\n    #Dédouble les datatasets\n    if repeat:\n        monet_ds = monet_ds.repeat(nb_repeat)\n        photo_ds = photo_ds.repeat(nb_repeat)\n\n    #Mélange les datasets\n    if shuffle:\n        monet_ds = monet_ds.shuffle(512, 2, True)\n        photo_ds = photo_ds.shuffle(512, 2, True)\n    \n    #batchage des datasets\n    monet_ds = monet_ds.batch(batch_size, drop_remainder=True)\n    photo_ds = photo_ds.batch(batch_size, drop_remainder=True)\n\n    #Pré-extraction d'un nombre \"AUTOTUNE\" de batchs\n    monet_ds = monet_ds.prefetch(AUTOTUNE)\n    photo_ds = photo_ds.prefetch(AUTOTUNE)\n    \n    gan_ds = tf.data.Dataset.zip((monet_ds, photo_ds))\n    return gan_ds","metadata":{"id":"q8TFAh_HJ7pS","execution":{"iopub.status.busy":"2022-03-30T13:16:17.253269Z","iopub.execute_input":"2022-03-30T13:16:17.253585Z","iopub.status.idle":"2022-03-30T13:16:17.264089Z","shell.execute_reply.started":"2022-03-30T13:16:17.253548Z","shell.execute_reply":"2022-03-30T13:16:17.263217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating final dataset","metadata":{}},{"cell_type":"code","source":"final_dataset = get_gan_dataset(MONET_FILENAMES, PHOTO_FILENAMES, repeat=True, shuffle=False, batch_size=64)","metadata":{"id":"_FRRvJxXJ7pT","execution":{"iopub.status.busy":"2022-03-30T13:16:17.265655Z","iopub.execute_input":"2022-03-30T13:16:17.266558Z","iopub.status.idle":"2022-03-30T13:16:17.525948Z","shell.execute_reply.started":"2022-03-30T13:16:17.266508Z","shell.execute_reply":"2022-03-30T13:16:17.524807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#CycleGAN Building","metadata":{"id":"E2Gx45eudCRv"}},{"cell_type":"markdown","source":"Convolution layers building function","metadata":{"id":"liS-YIXc57pI"}},{"cell_type":"code","source":"OUTPUT_CHANNELS = 3\n\ndef down_sample(filters, size, apply_instancenorm=True):\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    layer = keras.Sequential()\n    layer.add(layers.Conv2D(filters, size, strides=2, padding='same', kernel_initializer=initializer, use_bias=False))\n\n    if apply_instancenorm:\n        layer.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n\n    layer.add(layers.LeakyReLU())\n\n    return layer","metadata":{"id":"g8wG1nMWJ7pU","execution":{"iopub.status.busy":"2022-03-30T13:16:17.527697Z","iopub.execute_input":"2022-03-30T13:16:17.527977Z","iopub.status.idle":"2022-03-30T13:16:17.536255Z","shell.execute_reply.started":"2022-03-30T13:16:17.527943Z","shell.execute_reply":"2022-03-30T13:16:17.535156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transpose convolution layers building function","metadata":{"id":"hhLtEvezVjij"}},{"cell_type":"code","source":"def up_sample(filters, size, apply_dropout=False):\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n\n    layer = keras.Sequential()\n    layer.add(layers.Conv2DTranspose(filters, size, strides=2, padding='same', kernel_initializer=initializer,use_bias=False))\n    layer.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))\n\n    if apply_dropout:\n        layer.add(layers.Dropout(0.5))\n\n    layer.add(layers.ReLU())\n\n    return layer","metadata":{"id":"ZXJI4UQmJ7pV","execution":{"iopub.status.busy":"2022-03-30T13:16:17.537452Z","iopub.execute_input":"2022-03-30T13:16:17.537714Z","iopub.status.idle":"2022-03-30T13:16:17.550901Z","shell.execute_reply.started":"2022-03-30T13:16:17.537684Z","shell.execute_reply":"2022-03-30T13:16:17.549768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Generator building function","metadata":{"id":"bzVYFwNxVoGQ"}},{"cell_type":"code","source":"def Generator():\n    inputs = layers.Input(shape=[256,256,3])\n    down_stack = [\n        down_sample(64, 4, apply_instancenorm=False),\n        down_sample(128, 4),                        \n        down_sample(256, 4),                        \n        down_sample(512, 4),                        \n        down_sample(512, 4),                      \n        down_sample(512, 4),                      \n        down_sample(512, 4),                      \n        down_sample(512, 4),                      \n    ]\n\n    up_stack = [\n        up_sample(512, 4, apply_dropout=True),    \n        up_sample(512, 4, apply_dropout=True),    \n        up_sample(512, 4, apply_dropout=True),    \n        up_sample(512, 4),                          \n        up_sample(256, 4),                         \n        up_sample(128, 4),                           \n        up_sample(64, 4),                           \n    ]\n\n    initializer = tf.random_normal_initializer(0., 0.02)\n    last = layers.Conv2DTranspose(3, 4, strides=2, padding='same', kernel_initializer=initializer, activation='tanh') \n   \n\n    x = inputs\n\n    # Downsampling through the model\n    skips = []\n    for down in down_stack:\n        x = down(x)\n        skips.append(x)\n\n    skips = reversed(skips[:-1])\n\n    # Upsampling and establishing the skip connections\n    for up, skip in zip(up_stack, skips):\n        x = up(x)\n        x = layers.Concatenate()([x, skip])\n\n    x = last(x)\n\n    return keras.Model(inputs=inputs, outputs=x)","metadata":{"id":"cEqTPFLlJ7pW","execution":{"iopub.status.busy":"2022-03-30T13:16:17.552707Z","iopub.execute_input":"2022-03-30T13:16:17.553043Z","iopub.status.idle":"2022-03-30T13:16:17.568500Z","shell.execute_reply.started":"2022-03-30T13:16:17.553008Z","shell.execute_reply":"2022-03-30T13:16:17.567509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Discriminator building function for Monet Paint.\n\nThis kind of discriminator does not include the last layer as it is trained separately for each Discriminator of the dual discriminator","metadata":{"id":"BmxvGCElV1a2"}},{"cell_type":"code","source":"def Discriminator():\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n    \n    inp = layers.Input(shape=[256, 256, 3], name='input_image')\n    x = inp\n    \n    down1 = down_sample(64, 4, False)(x)       \n    down2 = down_sample(128, 4)(down1)        \n    down3 = down_sample(256, 4)(down2)        \n\n    zero_pad1 = layers.ZeroPadding2D()(down3)\n    conv = layers.Conv2D(512, 4, strides=1, kernel_initializer=initializer, use_bias=False)(zero_pad1)\n\n    norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init)(conv)\n    leaky_relu = layers.LeakyReLU()(norm1)\n    zero_pad2 = layers.ZeroPadding2D()(leaky_relu)\n\n    return tf.keras.Model(inputs=inp, outputs=zero_pad2)","metadata":{"id":"QCm_gjfiJ7pW","execution":{"iopub.status.busy":"2022-03-30T13:16:17.570015Z","iopub.execute_input":"2022-03-30T13:16:17.570397Z","iopub.status.idle":"2022-03-30T13:16:17.583890Z","shell.execute_reply.started":"2022-03-30T13:16:17.570351Z","shell.execute_reply":"2022-03-30T13:16:17.582690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ths function is creating the last layer (the missing one of the previons function) for a discriminator building with the previous function. This structure enable to use the same first layers for the Discriminator 1 and 2 and to specialize the last layer for each binary classification :\n* Is a real peinture\n* Is a generated peinture\n","metadata":{"id":"iJyR9-qBV-iL"}},{"cell_type":"code","source":"def DHead():\n    initializer = tf.random_normal_initializer(0., 0.02)\n    \n    inp = layers.Input(shape=[33, 33, 512], name='input_image')\n    x = inp\n    \n    last = layers.Conv2D(1, 4, strides=1, kernel_initializer=initializer)(x) # (size, 30, 30, 1)\n\n    return tf.keras.Model(inputs=inp, outputs=last)","metadata":{"id":"KxLj16XRJ7pW","execution":{"iopub.status.busy":"2022-03-30T13:16:17.585023Z","iopub.execute_input":"2022-03-30T13:16:17.586639Z","iopub.status.idle":"2022-03-30T13:16:17.602472Z","shell.execute_reply.started":"2022-03-30T13:16:17.586600Z","shell.execute_reply":"2022-03-30T13:16:17.601356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Discriminator building function for real photo","metadata":{"id":"3wbeaW9eX1RB"}},{"cell_type":"code","source":"def DiscriminatorP():\n    initializer = tf.random_normal_initializer(0., 0.02)\n    gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n    \n    inp = layers.Input(shape=[256, 256, 3], name='input_image')\n    x = inp\n    \n    down1 = down_sample(64, 4, False)(x)       \n    down2 = down_sample(128, 4)(down1)        \n    down3 = down_sample(256, 4)(down2)        \n\n    zero_pad1 = layers.ZeroPadding2D()(down3)\n    conv = layers.Conv2D(512, 4, strides=1, kernel_initializer=initializer, use_bias=False)(zero_pad1)\n\n    norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init)(conv)\n    leaky_relu = layers.LeakyReLU()(norm1)\n    zero_pad2 = layers.ZeroPadding2D()(leaky_relu)\n\n    last = layers.Conv2D(1, 4, strides=1, kernel_initializer=initializer)(zero_pad2)\n\n    return tf.keras.Model(inputs=inp, outputs=last)","metadata":{"id":"XNCRjjX0J7pX","execution":{"iopub.status.busy":"2022-03-30T13:16:17.605102Z","iopub.execute_input":"2022-03-30T13:16:17.605683Z","iopub.status.idle":"2022-03-30T13:16:17.617738Z","shell.execute_reply.started":"2022-03-30T13:16:17.605646Z","shell.execute_reply":"2022-03-30T13:16:17.616400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"CycleGAN building class definition","metadata":{"id":"iMKNbobkX25r"}},{"cell_type":"code","source":"class CycleGan(keras.Model):\n    #Executed function at the instanciation time\n    def __init__(\n        self,\n        monet_generator,\n        photo_generator,\n        monet_discriminator,\n        photo_discriminator,\n        dhead1,        \n        dhead2,        \n        lambda_cycle=3,\n        lambda_id=3,\n    ):\n        super(CycleGan, self).__init__()\n        self.m_gen = monet_generator\n        self.p_gen = photo_generator\n        self.m_disc = monet_discriminator\n        self.p_disc = photo_discriminator\n        self.lambda_cycle = lambda_cycle\n        self.lambda_id = lambda_id\n        self.dhead1 = dhead1\n        self.dhead2 = dhead2\n        \n    def compile(\n        self,\n        m_gen_optimizer,\n        p_gen_optimizer,\n        m_disc_optimizer,\n        p_disc_optimizer,\n        gen_loss_fn1,\n        gen_loss_fn2,\n        disc_loss_fn1,\n        disc_loss_fn2,\n        cycle_loss_fn,\n        identity_loss_fn,\n        aug_fn,\n\n    ):\n        super(CycleGan, self).compile()\n        self.m_gen_optimizer = m_gen_optimizer\n        self.p_gen_optimizer = p_gen_optimizer\n        self.m_disc_optimizer = m_disc_optimizer\n        self.p_disc_optimizer = p_disc_optimizer\n        self.gen_loss_fn1 = gen_loss_fn1\n        self.gen_loss_fn2 = gen_loss_fn2\n        self.disc_loss_fn1 = disc_loss_fn1\n        self.disc_loss_fn2 = disc_loss_fn2\n        self.cycle_loss_fn = cycle_loss_fn\n        self.identity_loss_fn = identity_loss_fn\n        self.aug_fn = aug_fn\n\n        self.step_num = 0\n    \n    def train_step(self, batch_data):\n        real_monet, real_photo = batch_data\n        batch_size = tf.shape(real_monet)[0]\n\n        with tf.GradientTape(persistent=True) as tape:\n        \n            # photo to monet back to photo\n            fake_monet = self.m_gen(real_photo, training=True)\n            cycled_photo = self.p_gen(fake_monet, training=True)\n\n            # monet to photo back to monet\n            fake_photo = self.p_gen(real_monet, training=True)\n            cycled_monet = self.m_gen(fake_photo, training=True)\n\n            # generating itself\n            same_monet = self.m_gen(real_monet, training=True)\n            same_photo = self.p_gen(real_photo, training=True)\n\n            # Diffaugment\n            both_monet = tf.concat([real_monet, fake_monet], axis=0)            \n            \n            aug_monet = self.aug_fn(both_monet)\n            \n            aug_real_monet = aug_monet[:batch_size]\n            aug_fake_monet = aug_monet[batch_size:]\n            \n            \n            # two-objective discriminator\n            disc_fake_monet1 = self.dhead1(self.m_disc(aug_fake_monet, training=True), training=True)\n            disc_real_monet1 = self.dhead1(self.m_disc(aug_real_monet, training=True), training=True)\n            disc_fake_monet2 = self.dhead2(self.m_disc(aug_fake_monet, training=True), training=True)\n            disc_real_monet2 = self.dhead2(self.m_disc(aug_real_monet, training=True), training=True)\n            \n            monet_gen_loss1 = self.gen_loss_fn1(disc_fake_monet1) \n            monet_disc_loss1 = self.disc_loss_fn1(disc_real_monet1, disc_fake_monet1)\n            monet_gen_loss2 = self.gen_loss_fn2(disc_fake_monet2)\n            monet_disc_loss2 = self.disc_loss_fn2(disc_real_monet2, disc_fake_monet2)\n\n\n\n            monet_gen_loss = (monet_gen_loss1 + monet_gen_loss2) * 0.4\n            monet_disc_loss = monet_disc_loss1 + monet_disc_loss2\n\n            # discriminator used to check, inputing real images\n            disc_real_photo = self.p_disc(real_photo, training=True)\n            \n            # discriminator used to check, inputing fake images\n            disc_fake_photo = self.p_disc(fake_photo, training=True)\n\n            # evaluates generator loss\n            photo_gen_loss = self.gen_loss_fn1(disc_fake_photo)\n            \n            # evaluates discriminator loss\n            photo_disc_loss = self.disc_loss_fn1(disc_real_photo, disc_fake_photo)\n\n\n            # evaluates total generator loss\n            total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle/ tf.cast(batch_size,tf.float32)) + self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle/ tf.cast(batch_size,tf.float32))\n\n            # evaluates total generator loss\n            total_monet_gen_loss = monet_gen_loss + total_cycle_loss +  self.identity_loss_fn(real_monet, same_monet, self.lambda_id / tf.cast(batch_size,tf.float32))\n            total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_id/ tf.cast(batch_size,tf.float32))\n\n\n\n        # Calculate the gradients for generator and discriminator\n        monet_generator_gradients = tape.gradient(total_monet_gen_loss,self.m_gen.trainable_variables)\n        photo_generator_gradients = tape.gradient(total_photo_gen_loss,self.p_gen.trainable_variables)\n\n        monet_discriminator_gradients = tape.gradient(monet_disc_loss,\n                                                      self.m_disc.trainable_variables)\n        photo_discriminator_gradients = tape.gradient(photo_disc_loss,\n                                                      self.p_disc.trainable_variables)\n        \n\n        # Heads gradients\n        monet_head_gradients = tape.gradient(monet_disc_loss1,\n                                                      self.dhead1.trainable_variables)\n\n        self.m_disc_optimizer.apply_gradients(zip(monet_head_gradients,\n                                                  self.dhead1.trainable_variables))       \n\n        monet_head_gradients = tape.gradient(monet_disc_loss2,\n                                                      self.dhead2.trainable_variables)\n        \n        self.m_disc_optimizer.apply_gradients(zip(monet_head_gradients,\n                                                  self.dhead2.trainable_variables))     \n        \n        \n        \n        # Apply the gradients to the optimizer\n        self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,\n                                                 self.m_gen.trainable_variables))\n\n        self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,\n                                                 self.p_gen.trainable_variables))\n\n        self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,\n                                                  self.m_disc.trainable_variables))\n\n        self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,\n                                                  self.p_disc.trainable_variables))\n        \n    \n        return {\n                'total_monet_gen_loss': total_monet_gen_loss,\n                'total_photo_gen_loss': total_photo_gen_loss,\n                'monet_disc_loss': monet_disc_loss,\n                'monet_disc_loss1':monet_disc_loss1,\n                'monet_disc_loss2':monet_disc_loss2,\n                'photo_disc_loss': photo_disc_loss,\n            }","metadata":{"id":"_rDZbCyYJ7pX","execution":{"iopub.status.busy":"2022-03-30T13:16:17.619543Z","iopub.execute_input":"2022-03-30T13:16:17.620007Z","iopub.status.idle":"2022-03-30T13:16:17.654230Z","shell.execute_reply.started":"2022-03-30T13:16:17.619970Z","shell.execute_reply":"2022-03-30T13:16:17.653211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Loss functions definition","metadata":{"id":"40sVZO95aP81"}},{"cell_type":"markdown","source":"Discriminator Loss function :\n* discriminator_loss1 is used for \"is real photo\" and \"is real peinture\" discriminators\n* discriminator_loss2 is used for \"is generated peinture\" discriminator","metadata":{"id":"0mluohcydS5T"}},{"cell_type":"code","source":"with strategy.scope():\n    def discriminator_loss1(real, generated):\n        real_loss = tf.math.minimum(tf.zeros_like(real), real-tf.ones_like(real))\n\n        generated_loss = tf.math.minimum(tf.zeros_like(generated), -generated-tf.ones_like(generated))\n\n        total_disc_loss = real_loss + generated_loss\n\n        return tf.reduce_mean(-total_disc_loss * 0.5)\n\n    def discriminator_loss2(real, generated):\n        real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.zeros_like(real), real)\n        \n        generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(generated), generated)\n        \n        total_disc_loss = real_loss + generated_loss\n\n        return tf.reduce_mean(total_disc_loss * 0.5)","metadata":{"id":"K8qd4s9zJ7pZ","execution":{"iopub.status.busy":"2022-03-30T13:16:17.656894Z","iopub.execute_input":"2022-03-30T13:16:17.657715Z","iopub.status.idle":"2022-03-30T13:16:17.676276Z","shell.execute_reply.started":"2022-03-30T13:16:17.657663Z","shell.execute_reply":"2022-03-30T13:16:17.675483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Generator Loss function :\n* generator_loss1 is used for \"fake photo\" and \"fake peinture\" generators\n* generator_loss2 is used for \"fake peinture\" generator","metadata":{"id":"0HulpREmaQul"}},{"cell_type":"code","source":"with strategy.scope():\n    def generator_loss1(generated):\n        return  tf.reduce_mean(-generated)\n\n    def generator_loss2(generated):\n        return tf.reduce_mean(tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.zeros_like(generated), generated))\n    ","metadata":{"id":"uztMWV1nJ7pb","execution":{"iopub.status.busy":"2022-03-30T13:16:17.678110Z","iopub.execute_input":"2022-03-30T13:16:17.678668Z","iopub.status.idle":"2022-03-30T13:16:17.691760Z","shell.execute_reply.started":"2022-03-30T13:16:17.678622Z","shell.execute_reply":"2022-03-30T13:16:17.690746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"calc_cycle_loss function is used for calculating the cycle consistency loss between an original image and its cycled image. Cycled image is :\n* for a real peinture, cycled peinture = PeintureGenerator(PhotoGenerator(original peinture))\n* for a real photo, cycled photo = PhotoGenerator(PeintureGenerator(original photo))","metadata":{"id":"ppVTpFgof6-U"}},{"cell_type":"code","source":"with strategy.scope():\n    def calc_cycle_loss(real_image, cycled_image, LAMBDA):\n        loss1 = tf.reduce_sum(tf.abs(real_image - cycled_image))\n\n        return LAMBDA * loss1 * 0.0000152587890625","metadata":{"id":"jypQ6Hw_J7pb","execution":{"iopub.status.busy":"2022-03-30T13:16:17.693461Z","iopub.execute_input":"2022-03-30T13:16:17.693988Z","iopub.status.idle":"2022-03-30T13:16:17.707059Z","shell.execute_reply.started":"2022-03-30T13:16:17.693941Z","shell.execute_reply":"2022-03-30T13:16:17.706289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"identity_loss function is used for forcing the Generator to not change an image if not nedd. For instance:\n* PhotoGenerator should not modify a photo at all (idealy) -> PhotoGenerator(original photo) ~ original photo \n* PeintureGenerator should not modify a photo at all (idealy) -> PeintureGenerator(original peinture) ~ original peinture","metadata":{"id":"yyIk8CHqdItN"}},{"cell_type":"code","source":"with strategy.scope():\n    def identity_loss(real_image, same_image, LAMBDA):\n        loss = tf.reduce_sum(tf.abs(real_image - same_image))\n        \n        return LAMBDA * 0.5 * loss * 0.0000152587890625","metadata":{"id":"wy-sznxfJ7pb","execution":{"iopub.status.busy":"2022-03-30T13:16:17.713569Z","iopub.execute_input":"2022-03-30T13:16:17.714157Z","iopub.status.idle":"2022-03-30T13:16:17.728210Z","shell.execute_reply.started":"2022-03-30T13:16:17.714042Z","shell.execute_reply":"2022-03-30T13:16:17.727177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Data-Augmentation functions","metadata":{"id":"6Ej1euYd5FIK"}},{"cell_type":"code","source":"with strategy.scope():    \n    #General Data-Augmentation calling function\n    def DiffAugment(x, policy='', channels_first=False):\n        if policy:\n            if channels_first:\n                x = tf.transpose(x, [0, 2, 3, 1])\n            for p in policy.split(','):\n                for f in AUGMENT_FNS[p]:\n                    x = f(x)\n            if channels_first:\n                x = tf.transpose(x, [0, 3, 1, 2])\n        return x\n\n    \n    #Random modification of brightness\n    def rand_brightness(x):\n        magnitude = tf.random.uniform([tf.shape(x)[0], 1, 1, 1]) - 0.5\n        x = x + magnitude\n        return x\n\n    #Random modification of saturation\n    def rand_saturation(x):\n        magnitude = tf.random.uniform([tf.shape(x)[0], 1, 1, 1]) * 2\n        x_mean = tf.reduce_sum(x, axis=3, keepdims=True) * 0.3333333333333333333\n        x = (x - x_mean) * magnitude + x_mean\n        return x\n\n    #Random modification of contraste\n    def rand_contrast(x):\n        magnitude = tf.random.uniform([tf.shape(x)[0], 1, 1, 1]) + 0.5\n        x_mean = tf.reduce_sum(x, axis=[1, 2, 3], keepdims=True) * 5.086e-6\n        x = (x - x_mean) * magnitude + x_mean\n        return x\n\n    #Image random translation\n    def rand_translation(x, ratio=0.125):\n        batch_size = tf.shape(x)[0]\n        image_size = tf.shape(x)[1:3]\n        shift = tf.cast(tf.cast(image_size, tf.float32) * ratio + 0.5, tf.int32)\n        translation_x = tf.random.uniform([batch_size, 1], -shift[0], shift[0] + 1, dtype=tf.int32)\n        translation_y = tf.random.uniform([batch_size, 1], -shift[1], shift[1] + 1, dtype=tf.int32)\n        grid_x = tf.clip_by_value(tf.expand_dims(tf.range(image_size[0], dtype=tf.int32), 0) + translation_x + 1, 0, image_size[0] + 1)\n        grid_y = tf.clip_by_value(tf.expand_dims(tf.range(image_size[1], dtype=tf.int32), 0) + translation_y + 1, 0, image_size[1] + 1)\n        x = tf.gather_nd(tf.pad(x, [[0, 0], [1, 1], [0, 0], [0, 0]]), tf.expand_dims(grid_x, -1), batch_dims=1)\n        x = tf.transpose(tf.gather_nd(tf.pad(tf.transpose(x, [0, 2, 1, 3]), [[0, 0], [1, 1], [0, 0], [0, 0]]), tf.expand_dims(grid_y, -1), batch_dims=1), [0, 2, 1, 3])\n        return x\n\n    #Image random cutout\n    def rand_cutout(x, ratio=0.5):\n        batch_size = tf.shape(x)[0]\n        image_size = tf.shape(x)[1:3]\n        cutout_size = tf.cast(tf.cast(image_size, tf.float32) * ratio + 0.5, tf.int32)\n        offset_x = tf.random.uniform([tf.shape(x)[0], 1, 1], maxval=image_size[0] + (1 - cutout_size[0] % 2), dtype=tf.int32)\n        offset_y = tf.random.uniform([tf.shape(x)[0], 1, 1], maxval=image_size[1] + (1 - cutout_size[1] % 2), dtype=tf.int32)\n        grid_batch, grid_x, grid_y = tf.meshgrid(tf.range(batch_size, dtype=tf.int32), tf.range(cutout_size[0], dtype=tf.int32), tf.range(cutout_size[1], dtype=tf.int32), indexing='ij')\n        cutout_grid = tf.stack([grid_batch, grid_x + offset_x - cutout_size[0] // 2, grid_y + offset_y - cutout_size[1] // 2], axis=-1)\n        mask_shape = tf.stack([batch_size, image_size[0], image_size[1]])\n        cutout_grid = tf.maximum(cutout_grid, 0)\n        cutout_grid = tf.minimum(cutout_grid, tf.reshape(mask_shape - 1, [1, 1, 1, 3]))\n        mask = tf.maximum(1 - tf.scatter_nd(cutout_grid, tf.ones([batch_size, cutout_size[0], cutout_size[1]], dtype=tf.float32), mask_shape), 0)\n        x = x * tf.expand_dims(mask, axis=3)\n        return x\n\n    #Dictionnaire des méthodes de data augmentation\n    AUGMENT_FNS = {\n        'color': [rand_brightness, rand_saturation, rand_contrast],\n        'translation': [rand_translation],\n        'cutout': [rand_cutout],\n    }\n    \n    #Fonction intermédiaire d'appel de la fonction de data augmentation\n    def aug_fn(image):\n        return DiffAugment(image,\"color,translation,cutout\")\n","metadata":{"id":"BmChGqN4J7pb","execution":{"iopub.status.busy":"2022-03-30T13:16:17.729896Z","iopub.execute_input":"2022-03-30T13:16:17.730860Z","iopub.status.idle":"2022-03-30T13:16:17.763775Z","shell.execute_reply.started":"2022-03-30T13:16:17.730802Z","shell.execute_reply":"2022-03-30T13:16:17.763014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"function for loop training. This function works with 2 arrays. 1 array with the sequence of learning_rate and 1 array with the sequence of epochs_number used to train the model.\n","metadata":{}},{"cell_type":"code","source":"def Cycle_GAN_training_function(learning_rate_array,number_epoch_array,photo_files_path,earlystop_patience=None):\n    \n    #Creating dataset for test generation\n    photo_ds = load_dataset(photo_files_path).batch(1)\n    \n    #Variable for saving the Loss performance\n    loss_results_df_total = pd.DataFrame()\n\n    #Defining callback function\n    earlystop = keras.callbacks.EarlyStopping(\n        monitor='total_monet_gen_loss',\n        min_delta=0.001,\n        patience=earlystop_patience,\n        verbose=1,\n        mode='min'\n    )\n    \n    \n    with strategy.scope():\n        monet_generator = Generator() # transforms photos to Monet-esque paintings\n        photo_generator = Generator() # transforms Monet paintings to be more like photos\n\n        monet_discriminator = Discriminator() # differentiates real Monet paintings and generated Monet paintings\n        photo_discriminator = DiscriminatorP() # differentiates real photos and generated photos\n        \n        dHead1 = DHead() # Head for BCE\n        dHead2 = DHead() # Head for hinge loss\n\n        #Instanciate the CycleGAN with the generators and the discriminators\n        cycle_gan_model = CycleGan(\n            monet_generator, photo_generator, monet_discriminator, photo_discriminator, dHead1, dHead2\n        )\n\n    #loop training throughout the various learning rate\n    for i in np.arange(len(learning_rate_array)):\n        with strategy.scope():\n            #Update the optimizer with the new learning rate\n            monet_generator_optimizer = tf.keras.optimizers.Adam(learning_rate_array[i], beta_1=0.5)\n            photo_generator_optimizer = tf.keras.optimizers.Adam(learning_rate_array[i], beta_1=0.5)\n            monet_discriminator_optimizer = tf.keras.optimizers.Adam(learning_rate_array[i], beta_1=0.5)\n            photo_discriminator_optimizer = tf.keras.optimizers.Adam(learning_rate_array[i], beta_1=0.5)\n\n            #Compiling the model with the new optimizer\n            cycle_gan_model.compile(\n                m_gen_optimizer = monet_generator_optimizer,\n                p_gen_optimizer = photo_generator_optimizer,\n                m_disc_optimizer = monet_discriminator_optimizer,\n                p_disc_optimizer = photo_discriminator_optimizer,\n                gen_loss_fn1 = generator_loss1,\n                gen_loss_fn2 = generator_loss2,\n                disc_loss_fn1 = discriminator_loss1,\n                disc_loss_fn2 = discriminator_loss2,\n                cycle_loss_fn = calc_cycle_loss,\n                identity_loss_fn = identity_loss,\n                aug_fn = aug_fn\n            )\n            \n        #Add earlystop callback whether an earlystop_patience has been given or not\n        if earlystop_patience==None:\n            history = cycle_gan_model.fit(final_dataset,steps_per_epoch=STEPS_PER_EPOCH, epochs=number_epoch_array[i])\n        else:\n            history = cycle_gan_model.fit(final_dataset,steps_per_epoch=STEPS_PER_EPOCH, epochs=number_epoch_array[i], callbacks=[earlystop])\n        \n        #Plot some samples to see how the training performed\n        _, ax = plt.subplots(5, 2, figsize=(32, 32))\n        for i, img in enumerate(photo_ds.take(5)):\n            prediction = monet_generator(img, training=False)[0].numpy()\n            prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n            img = (img[0] * 127.5 + 127.5).numpy().astype(np.uint8)\n\n            ax[i, 0].imshow(img)\n            ax[i, 1].imshow(prediction)\n            ax[i, 0].set_title(\"Input Photo\")\n            ax[i, 1].set_title(\"Monet-esque\")\n            ax[i, 0].axis(\"off\")\n            ax[i, 1].axis(\"off\")\n        plt.show()\n\n        #Compute and save Loss\n        loss_results_df = pd.DataFrame(history.history)\n        loss_results_df = loss_results_df.applymap(np.mean)\n        loss_results_df_total = pd.concat([loss_results_df_total,loss_results_df], ignore_index=True)\n        \n        #Increment indice i for looping through the learning_rate_array and number_epoch_array\n        i+=1\n    \n    \n    # Plot the various Loss\n    plt.plot(loss_results_df_total.index, loss_results_df_total['total_monet_gen_loss'], color='g', label='Loss Monet Generator')\n    plt.plot(loss_results_df_total.index, loss_results_df_total['total_photo_gen_loss'], color='r', label='Loss Photo Generator')\n    plt.plot(loss_results_df_total.index, loss_results_df_total['monet_disc_loss'], color='b', label='Loss Monet Discriminator')\n    plt.plot(loss_results_df_total.index, loss_results_df_total['photo_disc_loss'], color='m', label='Loss Photo Discriminator')\n    \n    for x_axis in number_epoch_array[:len(number_epoch_array)-2]:\n        plt.axvline(x=x_axis, color='k', label='Epoch 22')\n\n        \n    plt.legend()\n    plt.show()\n    \n    #Saving the weights of the :\n    # * Photo Generator and Discriminator\n    # * Peinture Generator and Discriminator\n    cycle_gan_model.p_gen.save_weights(\"p_gen_weights.h5\")\n    cycle_gan_model.p_disc.save_weights(\"p_disc_weights.h5\")\n    cycle_gan_model.m_gen.save_weights(\"m_gen_weights.h5\")\n    cycle_gan_model.m_disc.save_weights(\"m_disc_weights.h5\")\n    \n    #When looping is finished return the total saved Loss\n    return monet_generator","metadata":{"execution":{"iopub.status.busy":"2022-03-30T13:16:17.765463Z","iopub.execute_input":"2022-03-30T13:16:17.766101Z","iopub.status.idle":"2022-03-30T13:16:17.792910Z","shell.execute_reply.started":"2022-03-30T13:16:17.766052Z","shell.execute_reply":"2022-03-30T13:16:17.792199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define variables for training and start Cycle_GAN_training_function","metadata":{}},{"cell_type":"code","source":"%%time\nlearning_rate_array = [2e-4,1e-4,5e-5,2.5e-5,1e-5]\nnumber_epoch_array = [400,300,200,200,200]\n\nloss_results_df_total = pd.DataFrame()\n        \nfinal_monet_generator = Cycle_GAN_training_function(learning_rate_array,number_epoch_array,PHOTO_FILENAMES,earlystop_patience=None)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-28T20:30:41.366627Z","iopub.execute_input":"2022-03-28T20:30:41.367495Z","iopub.status.idle":"2022-03-28T21:41:38.747818Z","shell.execute_reply.started":"2022-03-28T20:30:41.367435Z","shell.execute_reply":"2022-03-28T21:41:38.746178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating the fake peintures and saving them into the \"../images/\" folder","metadata":{}},{"cell_type":"code","source":"%%time\n! mkdir images\n! mkdir ../images\n#Création d'un batch de photos dédié à la génération du fichier à soumettre\nfast_photo_ds = load_dataset(PHOTO_FILENAMES).batch(32*strategy.num_replicas_in_sync).prefetch(32*strategy.num_replicas_in_sync)\ni = 1\nfor img in fast_photo_ds:\n    prediction = final_monet_generator(img, training=False).numpy()\n    prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n    for pred in prediction:\n        im = PIL.Image.fromarray(pred)\n        im.save(\"../images/\" + str(i) + \".jpg\")\n        i += 1","metadata":{"execution":{"iopub.status.busy":"2022-03-28T21:41:38.749583Z","iopub.execute_input":"2022-03-28T21:41:38.750846Z","iopub.status.idle":"2022-03-28T21:45:48.466482Z","shell.execute_reply.started":"2022-03-28T21:41:38.750706Z","shell.execute_reply":"2022-03-28T21:45:48.465002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating the zip archive and saving it into the \"/kaggle/working/images\" folder","metadata":{}},{"cell_type":"code","source":"shutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{"execution":{"iopub.status.busy":"2022-03-28T21:45:48.469201Z","iopub.execute_input":"2022-03-28T21:45:48.469503Z","iopub.status.idle":"2022-03-28T21:45:52.885215Z","shell.execute_reply.started":"2022-03-28T21:45:48.46946Z","shell.execute_reply":"2022-03-28T21:45:52.884006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}