{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install tensorflow_addons","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-21T18:16:07.792637Z","iopub.execute_input":"2023-03-21T18:16:07.794319Z","iopub.status.idle":"2023-03-21T18:16:17.655176Z","shell.execute_reply.started":"2023-03-21T18:16:07.794268Z","shell.execute_reply":"2023-03-21T18:16:17.653879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow_datasets as tfds\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tensorflow import keras\nfrom tensorflow.keras.applications import vgg19\n! mkdir ../images\nimport shutil\nimport time\nimport random\n\n\n!gdown --id 18WzMkgwOCZ7nzMhGM4j7fw9q6sJg4Hm7\n!unzip /content/advNeuralstyleTransfer\n\n# This chunk works.\n!pip uninstall --yes gdown # After running this line, restart Colab runtime.\n\n!pip install gdown -U --no-cache-dir\n","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:17.657651Z","iopub.execute_input":"2023-03-21T18:16:17.658265Z","iopub.status.idle":"2023-03-21T18:16:33.564955Z","shell.execute_reply.started":"2023-03-21T18:16:17.658227Z","shell.execute_reply":"2023-03-21T18:16:33.563694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load in the data","metadata":{}},{"cell_type":"code","source":"data,data_info = tfds.load('cycle_gan/monet2photo', download=True, with_info=True, shuffle_files=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:33.569021Z","iopub.execute_input":"2023-03-21T18:16:33.569458Z","iopub.status.idle":"2023-03-21T18:16:33.733112Z","shell.execute_reply.started":"2023-03-21T18:16:33.569396Z","shell.execute_reply":"2023-03-21T18:16:33.732019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**resize the size of image from 320 to 256**","metadata":{}},{"cell_type":"code","source":"def imresize(image):\n    image = tf.image.resize(image['image'], (256,256))\n    return image\n","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:33.736044Z","iopub.execute_input":"2023-03-21T18:16:33.736669Z","iopub.status.idle":"2023-03-21T18:16:33.743064Z","shell.execute_reply.started":"2023-03-21T18:16:33.736629Z","shell.execute_reply":"2023-03-21T18:16:33.741895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_monet, train_photos, test_monet, test_photos= data['trainA'], data['trainB'], data['testA'], data['testB']\ntrain_monet=train_monet.map(imresize).cache().shuffle(64).batch(1)\ntrain_photos=train_photos.map(imresize).cache().shuffle(64).batch(1)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:33.744945Z","iopub.execute_input":"2023-03-21T18:16:33.746347Z","iopub.status.idle":"2023-03-21T18:16:33.833959Z","shell.execute_reply.started":"2023-03-21T18:16:33.746306Z","shell.execute_reply":"2023-03-21T18:16:33.832963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualize images and labels from an image classification dataset**","metadata":{}},{"cell_type":"code","source":"tfds.visualization.show_examples(data['trainB'],data_info)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:33.835761Z","iopub.execute_input":"2023-03-21T18:16:33.836177Z","iopub.status.idle":"2023-03-21T18:16:35.214597Z","shell.execute_reply.started":"2023-03-21T18:16:33.836134Z","shell.execute_reply":"2023-03-21T18:16:35.213490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfds.visualization.show_examples(data['trainA'],data_info)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:35.216560Z","iopub.execute_input":"2023-03-21T18:16:35.217954Z","iopub.status.idle":"2023-03-21T18:16:36.669951Z","shell.execute_reply.started":"2023-03-21T18:16:35.217909Z","shell.execute_reply":"2023-03-21T18:16:36.668988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg_model = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\nvgg_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:36.671976Z","iopub.execute_input":"2023-03-21T18:16:36.673198Z","iopub.status.idle":"2023-03-21T18:16:37.241890Z","shell.execute_reply.started":"2023-03-21T18:16:36.673158Z","shell.execute_reply":"2023-03-21T18:16:37.241055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content_layers = ['block5_conv2']\nstyle_layers = ['block1_conv1', 'block2_conv1', 'block3_conv1', 'block4_conv1', 'block5_conv1']\n","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.242952Z","iopub.execute_input":"2023-03-21T18:16:37.243314Z","iopub.status.idle":"2023-03-21T18:16:37.250346Z","shell.execute_reply.started":"2023-03-21T18:16:37.243275Z","shell.execute_reply":"2023-03-21T18:16:37.249568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_model(layer_names):\n    # Retrieve the output layers corresponding to the content and style layers\n    vgg_model = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\n    vgg_model.trainable = False\n    outputs = [vgg_model.get_layer(name).output for name in layer_names]\n    model=tf.keras.Model([vgg_model.input], outputs)\n    # model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.255162Z","iopub.execute_input":"2023-03-21T18:16:37.255560Z","iopub.status.idle":"2023-03-21T18:16:37.271125Z","shell.execute_reply.started":"2023-03-21T18:16:37.255515Z","shell.execute_reply":"2023-03-21T18:16:37.270216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The goal is:Takes an input tensor,\nflattens it, computes the gram matrix by matrix multiplication and transpose,\nand normalizes it by the total number of spatial locations in the input tensor.","metadata":{}},{"cell_type":"code","source":"def gram_matrix(input_tensor):\n    channels = tf.shape(input_tensor)[-1]\n    flattened = tf.reshape(input_tensor, [-1, channels])\n    result = tf.matmul(flattened, flattened, transpose_a=True)\n    num_locations = tf.cast(tf.shape(input_tensor)[1]*tf.shape(input_tensor)[2], tf.float32)\n    return result / (num_locations)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.272589Z","iopub.execute_input":"2023-03-21T18:16:37.272960Z","iopub.status.idle":"2023-03-21T18:16:37.287331Z","shell.execute_reply.started":"2023-03-21T18:16:37.272920Z","shell.execute_reply":"2023-03-21T18:16:37.286491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class entire_model(tf.keras.models.Model):\n    def __init__(self, style_layers, content_layers):\n        super(entire_model, self).__init__()\n        self.vgg=my_model(style_layers + content_layers)\n        self.style_layers=style_layers\n        self.content_layers=content_layers\n        self.num_style_layers=len(style_layers)\n        self.vgg.trainable=False\n\n    def call(self, inputs):\n        inputs=inputs\n        preprocessed_input=tf.keras.applications.vgg19.preprocess_input(inputs)\n        outputs=self.vgg(preprocessed_input)# Pass the preprocessed input to my_model\n\n        # Separate the representations of style and content\n        style_outputs, content_outputs=(outputs[:self.num_style_layers], outputs[self.num_style_layers:])\n        # Calculate the gram matrix for each layer in the style output. This will be the final style representation\n        style_outputs=[gram_matrix(layer) for layer in style_outputs]\n\n        # Store the content and style representation in dictionaries in a layer by layer manner\n        content_dict = {content_name:value\n                    for content_name, value\n                    in zip(self.content_layers, content_outputs)}\n\n        style_dict = {style_name:value\n                  for style_name, value\n                  in zip(self.style_layers, style_outputs)}\n\n        return {'content': content_dict, 'style': style_dict}\n        ","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.288347Z","iopub.execute_input":"2023-03-21T18:16:37.288858Z","iopub.status.idle":"2023-03-21T18:16:37.305319Z","shell.execute_reply.started":"2023-03-21T18:16:37.288821Z","shell.execute_reply":"2023-03-21T18:16:37.303944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Returns a dict of dicts with content and style representations, i.e., gram matrix of the style_layers and the content of the content_layers","metadata":{}},{"cell_type":"markdown","source":"The goal is:These are the weights assigned to style and content loss for neural style transfer, \nalong with the specific weights assigned to each layer of the VGG19 network for style loss.","metadata":{}},{"cell_type":"code","source":"style_weight=1\ncontent_weight=100\nstyle_weights = {'block1_conv1': 0.3,\n                 'block2_conv1': 0.45,\n                 'block3_conv1': 0.15,\n                 'block4_conv1': 0.05,\n                 'block5_conv1': 0.05}","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.308032Z","iopub.execute_input":"2023-03-21T18:16:37.309481Z","iopub.status.idle":"2023-03-21T18:16:37.317704Z","shell.execute_reply.started":"2023-03-21T18:16:37.309442Z","shell.execute_reply":"2023-03-21T18:16:37.316359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The function computes the total loss for neural style transfer by combining the style and content losses with appropriate weights. \nIt calculates the mean squared error between the output and target representations of each style and content layer. \nThe resulting loss value is returned.","metadata":{}},{"cell_type":"code","source":"def total_cost(outputs):\n    style_outputs=outputs['style']\n    content_outputs=outputs['content']\n    style_loss=tf.add_n([style_weights[name]*tf.reduce_mean((style_outputs[name]-style_targets[name])**2)\n                        for name in style_outputs.keys()])\n    style_loss*=style_weight/len(style_layers)# Normalize\n\n    content_loss = tf.add_n([tf.reduce_mean((content_outputs[name]-content_targets[name])**2)\n                             for name in content_outputs.keys()])\n    content_loss*=content_weight/len(content_layers)\n    loss=style_loss+content_loss\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.319709Z","iopub.execute_input":"2023-03-21T18:16:37.320673Z","iopub.status.idle":"2023-03-21T18:16:37.329094Z","shell.execute_reply.started":"2023-03-21T18:16:37.320634Z","shell.execute_reply":"2023-03-21T18:16:37.327970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a TensorFlow function that performs one training step in the neural style transfer process. \nIt computes the loss, calculates gradients and applies them to the input image using an optimizer.\nThe function is optimized for performance using the @tf.function decorator.","metadata":{}},{"cell_type":"code","source":"@tf.function()\ndef train_step(image):\n    with tf.GradientTape() as tape:\n        outputs = extractor(image)\n        loss = total_cost(outputs)\n\n    grad = tape.gradient(loss, image)\n    opt.apply_gradients([(grad, image)])\n    image.assign(clip_0_1(image))\n    # print(outputs)\n    return loss.numpy()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.331052Z","iopub.execute_input":"2023-03-21T18:16:37.332125Z","iopub.status.idle":"2023-03-21T18:16:37.345468Z","shell.execute_reply.started":"2023-03-21T18:16:37.332088Z","shell.execute_reply":"2023-03-21T18:16:37.344294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.config.run_functions_eagerly(True)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.347390Z","iopub.execute_input":"2023-03-21T18:16:37.348513Z","iopub.status.idle":"2023-03-21T18:16:37.358147Z","shell.execute_reply.started":"2023-03-21T18:16:37.348473Z","shell.execute_reply":"2023-03-21T18:16:37.356830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import vstack\nplt.rcdefaults()\ndef show_plot(imagesX, imagesY1, imagesY2,iter):\n    images = vstack((imagesX, imagesY1, imagesY2))\n    titles = ['Cotent', 'Style', 'Generated ']\n    # scale from [-1,1] to [0,1]\n    # plot images row by row\n    for i in range(3):\n        # define subplot\n        plt.subplot(1, len(images), 1 + i)\n        # turn off axis\n        plt.axis('off')\n        # plot raw pixel data\n        plt.imshow(images[i])\n        # title\n        plt.title(titles[i])\n    # plt.savefig(\"gif/\"+str(iter)+\".jpg\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.360191Z","iopub.execute_input":"2023-03-21T18:16:37.361387Z","iopub.status.idle":"2023-03-21T18:16:37.372500Z","shell.execute_reply.started":"2023-03-21T18:16:37.361349Z","shell.execute_reply":"2023-03-21T18:16:37.371214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Similarity between two images.**","metadata":{}},{"cell_type":"code","source":"def LossSimilarity2Imges(y_true, y_pred):\n  return tf.reduce_mean(tf.image.ssim(y_true, y_pred, 1))","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.374616Z","iopub.execute_input":"2023-03-21T18:16:37.375845Z","iopub.status.idle":"2023-03-21T18:16:37.383504Z","shell.execute_reply.started":"2023-03-21T18:16:37.375679Z","shell.execute_reply":"2023-03-21T18:16:37.382315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The model","metadata":{}},{"cell_type":"code","source":"total_loss=[]\ntotal_ssim_content=[]\ntotal_ssim_style=[]\nnum_of_images=300\nnum_iterations=15\nfor j in range(num_of_images):\n    for im in train_photos.skip(random.randint(0,500)).take(1):\n        content_img = (im.numpy()/255)\n    for im in train_monet.skip(random.randint(0,500)).take(1):\n        style_img = (im.numpy()/255)\n\n    losses=[]\n    content_ssim=[]\n    style_ssim=[]\n    extractor=entire_model(style_layers, content_layers)\n    style_targets = extractor(style_img)['style']\n    content_targets = extractor(content_img)['content']\n    generate_image = tf.Variable(content_img)\n    opt =tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)\n    # Since this is a float image, define a function to keep the pixel values between 0 and 1\n    def clip_0_1(image):\n        return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)\n\n    start_time = time.time()\n    print(\"[Photo Number:\",j+1,\"]\")\n    for i in range(num_iterations+1):\n        loss=train_step(generate_image)\n        c_ssim=LossSimilarity2Imges(content_img,generate_image.read_value()).numpy()\n        s_ssim=LossSimilarity2Imges(style_img,generate_image.read_value()).numpy()\n        content_ssim.append(c_ssim)\n        style_ssim.append(s_ssim)\n        losses.append(loss)\n        g_img=generate_image.read_value().numpy()\n        # print(\"Iteration:\",i,\", Loss:\",loss)\n        # print(\"Content Loss:\",LossSimilarity2Imges(content_img,g_img).numpy())\n        # print(\"Style Loss:\",LossSimilarity2Imges(style_img,g_img).numpy())\n\n    print(\"Time: {:.2f} seconds\".format( time.time() - start_time))        \n    show_plot(content_img,style_img,g_img,i)\n    total_loss.append(losses)\n    total_ssim_content.append(content_ssim)\n    total_ssim_style.append(style_ssim)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:16:37.385630Z","iopub.execute_input":"2023-03-21T18:16:37.386839Z","iopub.status.idle":"2023-03-21T18:31:10.151817Z","shell.execute_reply.started":"2023-03-21T18:16:37.386801Z","shell.execute_reply":"2023-03-21T18:31:10.150299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(total_loss[0])","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:31:10.153381Z","iopub.execute_input":"2023-03-21T18:31:10.154062Z","iopub.status.idle":"2023-03-21T18:31:10.161727Z","shell.execute_reply.started":"2023-03-21T18:31:10.154013Z","shell.execute_reply":"2023-03-21T18:31:10.160386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extractor=entire_model(style_layers, content_layers)\nextractor.built = True\nextractor.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:31:10.163231Z","iopub.execute_input":"2023-03-21T18:31:10.163886Z","iopub.status.idle":"2023-03-21T18:31:10.611118Z","shell.execute_reply.started":"2023-03-21T18:31:10.163851Z","shell.execute_reply":"2023-03-21T18:31:10.610194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nplt.plot(total_loss[0])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:31:10.612339Z","iopub.execute_input":"2023-03-21T18:31:10.612713Z","iopub.status.idle":"2023-03-21T18:31:10.881128Z","shell.execute_reply.started":"2023-03-21T18:31:10.612675Z","shell.execute_reply":"2023-03-21T18:31:10.880074Z"},"trusted":true},"execution_count":null,"outputs":[]}]}