{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-30T04:17:20.565351Z","iopub.execute_input":"2021-11-30T04:17:20.566059Z","iopub.status.idle":"2021-11-30T04:17:20.593511Z","shell.execute_reply.started":"2021-11-30T04:17:20.565945Z","shell.execute_reply":"2021-11-30T04:17:20.592574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import (Dense, \n                                     BatchNormalization, \n                                     LeakyReLU, \n                                     Reshape, \n                                     Conv2DTranspose,\n                                     Conv2D,\n                                     Dropout,\n                                     Flatten)\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n    \nprint(tf.__version__)\nGCS_PATH = KaggleDatasets().get_gcs_path()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:20.595486Z","iopub.execute_input":"2021-11-30T04:17:20.59582Z","iopub.status.idle":"2021-11-30T04:17:32.748212Z","shell.execute_reply.started":"2021-11-30T04:17:20.595779Z","shell.execute_reply":"2021-11-30T04:17:32.747405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))\nPHOTOS = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:32.749366Z","iopub.execute_input":"2021-11-30T04:17:32.749616Z","iopub.status.idle":"2021-11-30T04:17:32.905768Z","shell.execute_reply.started":"2021-11-30T04:17:32.749589Z","shell.execute_reply":"2021-11-30T04:17:32.904834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [256, 256]\n#scale to [-1,1]\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    return image\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:32.907737Z","iopub.execute_input":"2021-11-30T04:17:32.907961Z","iopub.status.idle":"2021-11-30T04:17:32.917083Z","shell.execute_reply.started":"2021-11-30T04:17:32.907936Z","shell.execute_reply":"2021-11-30T04:17:32.916144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"monet_ds = load_dataset(MONET_FILENAMES, labeled=True).batch(1)\nphoto_ds = load_dataset(PHOTOS, labeled=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:32.918292Z","iopub.execute_input":"2021-11-30T04:17:32.918523Z","iopub.status.idle":"2021-11-30T04:17:33.110988Z","shell.execute_reply.started":"2021-11-30T04:17:32.918498Z","shell.execute_reply":"2021-11-30T04:17:33.110356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for element in photo_ds:\n    x=element\n    break","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:33.112319Z","iopub.execute_input":"2021-11-30T04:17:33.112731Z","iopub.status.idle":"2021-11-30T04:17:33.713929Z","shell.execute_reply.started":"2021-11-30T04:17:33.112703Z","shell.execute_reply":"2021-11-30T04:17:33.713066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=tf.image.resize(x, [256, 256]) # WAS [28,28]\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:33.715343Z","iopub.execute_input":"2021-11-30T04:17:33.716247Z","iopub.status.idle":"2021-11-30T04:17:33.725525Z","shell.execute_reply.started":"2021-11-30T04:17:33.7162Z","shell.execute_reply":"2021-11-30T04:17:33.724487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_generator_model():\n    model = tf.keras.Sequential()\n    model.add(Dense(7*7*512, use_bias=False, input_shape=(256,))) #originally 7*7*7, 256*256\n    model.add(BatchNormalization())\n    model.add(LeakyReLU())\n\n    model.add(Reshape((7,7,512)))#was 7,7,512\n#     assert model.output_shape == (None, 7, 7, 512) # Note: None is the batch size\n    model.add(Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False)) #originally 128\n#     assert model.output_shape == (None, 7, 7, 128)\n    model.add(BatchNormalization())\n    model.add(LeakyReLU())\n    \n    model.add(Conv2DTranspose(64, (5, 5), strides=(2, 2), padding='same', use_bias=False)) #originally 64\n#     assert model.output_shape == (None, 14, 14, 64)\n    model.add(BatchNormalization())\n    model.add(LeakyReLU())\n\n    model.add(Conv2DTranspose(3, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))#originally 3\n#     print(model.output_shape)\n#     assert model.output_shape == (None, 28, 28, 3)\n    return model\n    #model = tf.keras.Sequential()\n    #model.add(Dense(7*7*512, use_bias=False, input_shape=(256*256,)))\n    #model.add(BatchNormalization())\n    #model.add(LeakyReLU())\n\n    #model.add(Reshape((7,7,512)))\n    #assert model.output_shape == (None, 7, 7, 512) # Note: None is the batch size\n\n    #model.add(Conv2DTranspose(32, kernel_size=(3,3), input_shape=(28,28,3)))\n    #assert model.output_shape == (None, 7, 7, 128)\n    #model.add(Conv2DTranspose(64, (3,3)))\n    #model.add(MaxPooling2D(pool_size=(2,2)))\n    #model.add(Dropout(0.25))\n              \n    #model.add(Conv2DTranspose(3, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:33.727051Z","iopub.execute_input":"2021-11-30T04:17:33.727536Z","iopub.status.idle":"2021-11-30T04:17:33.738519Z","shell.execute_reply.started":"2021-11-30T04:17:33.727491Z","shell.execute_reply":"2021-11-30T04:17:33.73769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"generator = make_generator_model()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:33.739654Z","iopub.execute_input":"2021-11-30T04:17:33.740002Z","iopub.status.idle":"2021-11-30T04:17:33.91771Z","shell.execute_reply.started":"2021-11-30T04:17:33.739974Z","shell.execute_reply":"2021-11-30T04:17:33.916877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noise = tf.random.normal([3, 256])*127.5+100\nstarting_image = tf.reshape(x, (3, 256*256))#y[:,:,0:3]\ngenerated_image = generator(noise, training=False)\nplt.imshow(generated_image[0, :, :, 0:3]) #formerly cmap='gray'","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:33.920786Z","iopub.execute_input":"2021-11-30T04:17:33.921104Z","iopub.status.idle":"2021-11-30T04:17:34.209474Z","shell.execute_reply.started":"2021-11-30T04:17:33.921064Z","shell.execute_reply":"2021-11-30T04:17:34.208391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First descrimnator model\ndef make_discriminator_model():\n    model = tf.keras.Sequential()\n    \n    model.add(Conv2D(64, (5, 5), strides=(2, 2), padding='same', input_shape=[28, 28, 3]))\n    model.add(LeakyReLU())\n    model.add(Dropout(0.3))\n\n    model.add(Conv2D(128, (5, 5), strides=(2, 2), padding='same'))\n    model.add(LeakyReLU())\n    model.add(Dropout(0.3))\n\n    #model.add(Flatten())\n    model.add(Dense(3))\n    print(model.output_shape)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:34.210805Z","iopub.execute_input":"2021-11-30T04:17:34.211046Z","iopub.status.idle":"2021-11-30T04:17:34.219034Z","shell.execute_reply.started":"2021-11-30T04:17:34.211018Z","shell.execute_reply":"2021-11-30T04:17:34.218107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discriminator = make_discriminator_model()\ndecision = discriminator(generated_image)\ncross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n\ndef discriminator_loss(real_output, fake_output):\n    real_loss = cross_entropy(tf.ones_like(real_output), real_output)\n    fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)\n    total_loss = real_loss + fake_loss\n    return total_loss\n\ndef generator_loss(fake_output):\n    return cross_entropy(tf.ones_like(fake_output), fake_output)\n\ngenerator_optimizer = tf.keras.optimizers.Adam(1e-4)\ndiscriminator_optimizer = tf.keras.optimizers.Adam(1e-4)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:34.220108Z","iopub.execute_input":"2021-11-30T04:17:34.220889Z","iopub.status.idle":"2021-11-30T04:17:34.330677Z","shell.execute_reply.started":"2021-11-30T04:17:34.220849Z","shell.execute_reply":"2021-11-30T04:17:34.329753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n\ndef discriminator_loss(real_output, fake_output):\n    real_loss = cross_entropy(tf.ones_like(real_output), real_output)\n    fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)\n    total_loss = real_loss + fake_loss\n    return total_loss\n\ndef generator_loss(fake_output):\n    return cross_entropy(tf.ones_like(fake_output), fake_output)\n\ngenerator_optimizer = tf.keras.optimizers.Adam(1e-4)\ndiscriminator_optimizer = tf.keras.optimizers.Adam(1e-4)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:34.333187Z","iopub.execute_input":"2021-11-30T04:17:34.333551Z","iopub.status.idle":"2021-11-30T04:17:34.340944Z","shell.execute_reply.started":"2021-11-30T04:17:34.333522Z","shell.execute_reply":"2021-11-30T04:17:34.33974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_examples_to_generate = 16\nnoise_dim = 256  #WAS 100\nseed = tf.random.normal([num_examples_to_generate, noise_dim])","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:17:34.342676Z","iopub.execute_input":"2021-11-30T04:17:34.342932Z","iopub.status.idle":"2021-11-30T04:17:34.356093Z","shell.execute_reply.started":"2021-11-30T04:17:34.342903Z","shell.execute_reply":"2021-11-30T04:17:34.355051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef train_step(images):\n  \n    # 1 - Create a random noise to feed it into the model\n    # for the image generation\n    #noise = tf.random.normal([BATCH_SIZE, noise_dim])\n    #noise = tf.random.normal([3, 256])*127.5+100\n\n    # 2 - Generate images and calculate loss values\n    # GradientTape method records operations for automatic differentiation.\n    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n      generated_images = generator(noise, training=True)\n\n      real_output = discriminator(images, training=True)\n      fake_output = discriminator(generated_images, training=True)\n\n      gen_loss = generator_loss(fake_output)\n      disc_loss = discriminator_loss(real_output, fake_output)\n\n    # 3 - Calculate gradients using loss values and model variables\n    # \"gradient\" method computes the gradient using \n    # operations recorded in context of this tape (gen_tape and disc_tape).\n    \n    # It accepts a target (e.g., gen_loss) variable and \n    # a source variable (e.g.,generator.trainable_variables)\n    # target --> a list or nested structure of Tensors or Variables to be differentiated.\n    # source --> a list or nested structure of Tensors or Variables.\n    # target will be differentiated against elements in sources.\n\n    # \"gradient\" method returns a list or nested structure of Tensors  \n    # (or IndexedSlices, or None), one for each element in sources. \n    # Returned structure is the same as the structure of sources.\n    gradients_of_generator = gen_tape.gradient(gen_loss, \n                                               generator.trainable_variables)\n    gradients_of_discriminator = disc_tape.gradient(disc_loss, \n                                                discriminator.trainable_variables)\n    \n    # 4 - Process  Gradients and Run the Optimizer\n    # \"apply_gradients\" method processes aggregated gradients. \n    # ex: optimizer.apply_gradients(zip(grads, vars))\n    \"\"\"\n    Example use of apply_gradients:\n    grads = tape.gradient(loss, vars)\n    grads = tf.distribute.get_replica_context().all_reduce('sum', grads)\n    # Processing aggregated gradients.\n    optimizer.apply_gradients(zip(grads, vars), experimental_aggregate_gradients=False)\n    \"\"\"\n    generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:18:34.882123Z","iopub.execute_input":"2021-11-30T04:18:34.882528Z","iopub.status.idle":"2021-11-30T04:18:34.915529Z","shell.execute_reply.started":"2021-11-30T04:18:34.882489Z","shell.execute_reply":"2021-11-30T04:18:34.914708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom IPython import display # A command shell for interactive computing in Python.\n\nsamples=[]\ndef train(dataset, epochs):\n  # A. For each epoch, do the following:\n  for epoch in range(epochs):\n    print(epoch)\n    start = time.time()\n    # 1 - For each batch of the epoch, \n    for image_batch in dataset:\n      # 1.a - run the custom \"train_step\" function\n      # we just declared above\n        train_step(image_batch)\n\n    # 2 - Produce images for the GIF as we go\n    display.clear_output(wait=True)\n    generate_and_save_images(generator,\n                             epoch + 1,\n                             seed)\n\n    # 3 - Save the model every 5 epochs as \n    # a checkpoint, which we will use later\n    #if (epoch + 1) % 5 == 0:\n     # checkpoint.save(file_prefix = checkpoint_prefix)\n\n    # 4 - Print out the completed epoch no. and the time spent\n    print ('Time for epoch {} is {} sec'.format(epoch + 1, time.time()-start))\n    # B. Generate a final image after the training is completed\n    display.clear_output(wait=True)\n    generate_and_save_images(generator,\n                           epochs,\n                           seed)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:24:54.405188Z","iopub.execute_input":"2021-11-30T04:24:54.4067Z","iopub.status.idle":"2021-11-30T04:24:54.417245Z","shell.execute_reply.started":"2021-11-30T04:24:54.406641Z","shell.execute_reply":"2021-11-30T04:24:54.415994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_and_save_images(model, epoch, test_input):\n    predictions = model(test_input, training=False)\n    fig = plt.figure(figsize=(3,3))\n    samples.append(predictions[0][0])\n    for i in range(predictions.shape[0]):\n        plt.subplot(4, 4, i+1)\n        plt.imshow(predictions[i,:,:,0:3]*127.5)\n        plt.axis('off')\n        plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n        plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:25:52.257308Z","iopub.execute_input":"2021-11-30T04:25:52.257603Z","iopub.status.idle":"2021-11-30T04:25:52.265174Z","shell.execute_reply.started":"2021-11-30T04:25:52.257573Z","shell.execute_reply":"2021-11-30T04:25:52.264116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32\nEPOCHS = 1000\n\ntrain(monet_ds, EPOCHS)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:25:53.328441Z","iopub.execute_input":"2021-11-30T04:25:53.328734Z","iopub.status.idle":"2021-11-30T04:41:29.624607Z","shell.execute_reply.started":"2021-11-30T04:25:53.328699Z","shell.execute_reply":"2021-11-30T04:41:29.623705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(samples[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-30T04:45:24.481486Z","iopub.execute_input":"2021-11-30T04:45:24.481802Z","iopub.status.idle":"2021-11-30T04:45:24.676048Z","shell.execute_reply.started":"2021-11-30T04:45:24.48177Z","shell.execute_reply":"2021-11-30T04:45:24.675242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}