{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow_datasets as tfds\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tensorflow import keras\nfrom tensorflow.keras.applications import vgg19","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-27T13:17:22.304873Z","iopub.execute_input":"2023-02-27T13:17:22.305654Z","iopub.status.idle":"2023-02-27T13:17:22.311872Z","shell.execute_reply.started":"2023-02-27T13:17:22.305588Z","shell.execute_reply":"2023-02-27T13:17:22.310491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Upload The DataSet","metadata":{}},{"cell_type":"code","source":"dataset,info = tfds.load('cycle_gan/monet2photo',\n                    download=True,\n                    with_info=True,\n                    shuffle_files=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:22.314130Z","iopub.execute_input":"2023-02-27T13:17:22.315183Z","iopub.status.idle":"2023-02-27T13:17:22.469393Z","shell.execute_reply.started":"2023-02-27T13:17:22.315146Z","shell.execute_reply":"2023-02-27T13:17:22.468229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Resize images to 256 on 256","metadata":{}},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.resize(image['image'], (256,256))\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:22.471159Z","iopub.execute_input":"2023-02-27T13:17:22.471526Z","iopub.status.idle":"2023-02-27T13:17:22.477403Z","shell.execute_reply.started":"2023-02-27T13:17:22.471487Z","shell.execute_reply":"2023-02-27T13:17:22.476215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_monet, train_photo= dataset['trainA'], dataset['trainB']\ntest_monet, test_photo=dataset['testA'], dataset['testB']\n\ntrain_monet=train_monet.map(decode_image).cache().shuffle(32).batch(1)\ntrain_photo=train_photo.map(decode_image).cache().shuffle(32).batch(1)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:22.480366Z","iopub.execute_input":"2023-02-27T13:17:22.481398Z","iopub.status.idle":"2023-02-27T13:17:22.539326Z","shell.execute_reply.started":"2023-02-27T13:17:22.481360Z","shell.execute_reply":"2023-02-27T13:17:22.538424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfds.visualization.show_examples(dataset['trainA'],info)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:22.540856Z","iopub.execute_input":"2023-02-27T13:17:22.541211Z","iopub.status.idle":"2023-02-27T13:17:23.967012Z","shell.execute_reply.started":"2023-02-27T13:17:22.541176Z","shell.execute_reply":"2023-02-27T13:17:23.965919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfds.visualization.show_examples(dataset['trainB'],info)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:23.968560Z","iopub.execute_input":"2023-02-27T13:17:23.969006Z","iopub.status.idle":"2023-02-27T13:17:25.116070Z","shell.execute_reply.started":"2023-02-27T13:17:23.968971Z","shell.execute_reply":"2023-02-27T13:17:25.114948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The View of VGG Model","metadata":{}},{"cell_type":"code","source":"vgg_model = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\nvgg_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.117711Z","iopub.execute_input":"2023-02-27T13:17:25.118153Z","iopub.status.idle":"2023-02-27T13:17:25.564191Z","shell.execute_reply.started":"2023-02-27T13:17:25.118116Z","shell.execute_reply":"2023-02-27T13:17:25.563376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content_layers=['block4_conv2']\nstyle_layers = ['block1_conv1',\n                'block2_conv1',\n                'block3_conv1', \n                'block4_conv1', \n                'block5_conv1']","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.565264Z","iopub.execute_input":"2023-02-27T13:17:25.565679Z","iopub.status.idle":"2023-02-27T13:17:25.571421Z","shell.execute_reply.started":"2023-02-27T13:17:25.565621Z","shell.execute_reply":"2023-02-27T13:17:25.570606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_model(layer_names):\n    # Retrieve the output layers corresponding to the content and style layers\n    vgg_model = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\n    vgg_model.trainable = False\n    outputs = [vgg_model.get_layer(name).output for name in layer_names]\n    model=tf.keras.Model([vgg_model.input], outputs)\n    # model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.578524Z","iopub.execute_input":"2023-02-27T13:17:25.578889Z","iopub.status.idle":"2023-02-27T13:17:25.589569Z","shell.execute_reply.started":"2023-02-27T13:17:25.578853Z","shell.execute_reply":"2023-02-27T13:17:25.588800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gram_matrix(input_tensor): # input_tensor is of shape ch, n_H, n_W\n    result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)\n    input_shape = tf.shape(input_tensor)\n    num_locations = tf.cast(input_shape[1]*input_shape[2], tf.float32) # Unrolls n_H and n_W\n    return result/(num_locations)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.591012Z","iopub.execute_input":"2023-02-27T13:17:25.591320Z","iopub.status.idle":"2023-02-27T13:17:25.600534Z","shell.execute_reply.started":"2023-02-27T13:17:25.591286Z","shell.execute_reply":"2023-02-27T13:17:25.599779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining the entire model","metadata":{}},{"cell_type":"code","source":"class entire_model(tf.keras.models.Model):\n    def __init__(self, style_layers, content_layers):\n        super(entire_model, self).__init__()\n        self.vgg=my_model(style_layers + content_layers)\n        self.style_layers=style_layers\n        self.content_layers=content_layers\n        self.num_style_layers=len(style_layers)\n        self.vgg.trainable=False\n\n    def call(self, inputs):\n        inputs=inputs*255.0 # Scale back the pixel values\n        preprocessed_input=tf.keras.applications.vgg19.preprocess_input(inputs)\n        outputs=self.vgg(preprocessed_input)# Pass the preprocessed input to my_model\n\n        # Separate the representations of style and content\n        style_outputs, content_outputs=(outputs[:self.num_style_layers], outputs[self.num_style_layers:])\n        # Calculate the gram matrix for each layer in the style output. This will be the final style representation\n        style_outputs=[gram_matrix(layer) for layer in style_outputs]\n\n        # Store the content and style representation in dictionaries in a layer by layer manner\n        content_dict = {content_name:value\n                    for content_name, value\n                    in zip(self.content_layers, content_outputs)}\n\n        style_dict = {style_name:value\n                  for style_name, value\n                  in zip(self.style_layers, style_outputs)}\n\n        return {'content': content_dict, 'style': style_dict}\n        # Returns a dict of dicts with content and style representations, i.e., gram matrix of the style_layers and\n        # the content of the content_layers","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.601894Z","iopub.execute_input":"2023-02-27T13:17:25.602250Z","iopub.status.idle":"2023-02-27T13:17:25.616144Z","shell.execute_reply.started":"2023-02-27T13:17:25.602216Z","shell.execute_reply":"2023-02-27T13:17:25.614870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining wieghts for each layer","metadata":{}},{"cell_type":"code","source":"style_weight=1\ncontent_weight=100\nstyle_weights = {'block1_conv1': 0.1,\n                 'block2_conv1': 0.45,\n                 'block3_conv1': 0.15,\n                 'block4_conv1': 0.05,\n                 'block5_conv1': 0.05}","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.617855Z","iopub.execute_input":"2023-02-27T13:17:25.618694Z","iopub.status.idle":"2023-02-27T13:17:25.629114Z","shell.execute_reply.started":"2023-02-27T13:17:25.618656Z","shell.execute_reply":"2023-02-27T13:17:25.627989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def total_cost(outputs):\n    style_outputs=outputs['style']\n    content_outputs=outputs['content']\n    style_loss=tf.add_n([style_weights[name]*tf.reduce_mean((style_outputs[name]-style_targets[name])**2)\n                        for name in style_outputs.keys()])\n    style_loss*=style_weight/len(style_layers)# Normalize\n\n    content_loss = tf.add_n([tf.reduce_mean((content_outputs[name]-content_targets[name])**2)\n                             for name in content_outputs.keys()])\n    content_loss*=content_weight/len(content_layers)\n    loss=style_loss+content_loss\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.630705Z","iopub.execute_input":"2023-02-27T13:17:25.631542Z","iopub.status.idle":"2023-02-27T13:17:25.639751Z","shell.execute_reply.started":"2023-02-27T13:17:25.631504Z","shell.execute_reply":"2023-02-27T13:17:25.638690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Traning","metadata":{}},{"cell_type":"code","source":"@tf.function()\ndef train_step(image):\n    with tf.GradientTape() as tape:\n        outputs = extractor(image)\n        loss = total_cost(outputs)\n\n    grad = tape.gradient(loss, image)\n    opt.apply_gradients([(grad, image)])\n    image.assign(clip_0_1(image))\n    return loss.numpy()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.641349Z","iopub.execute_input":"2023-02-27T13:17:25.642045Z","iopub.status.idle":"2023-02-27T13:17:25.653399Z","shell.execute_reply.started":"2023-02-27T13:17:25.642007Z","shell.execute_reply":"2023-02-27T13:17:25.652242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntf.config.run_functions_eagerly(True)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.655102Z","iopub.execute_input":"2023-02-27T13:17:25.655888Z","iopub.status.idle":"2023-02-27T13:17:25.663289Z","shell.execute_reply.started":"2023-02-27T13:17:25.655851Z","shell.execute_reply":"2023-02-27T13:17:25.662285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import vstack\n# plt.rcParams[\"figure.figsize\"] = (10,10)\nplt.rcdefaults()\ndef show_plot(imagesX, imagesY1, imagesY2,iter):\n    images = vstack((imagesX, imagesY1, imagesY2))\n    titles = ['Cotent', 'Style', 'Generated ' + str(iter) + \" step\"]\n    # scale from [-1,1] to [0,1]\n    # plot images row by row\n    for i in range(3):\n        # define subplot\n        plt.subplot(1, len(images), 1 + i)\n        # turn off axis\n        plt.axis('off')\n        # plot raw pixel data\n        plt.imshow(images[i])\n        # title\n        plt.title(titles[i])\n    # plt.savefig(\"gif/\"+str(iter)+\".jpg\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.664984Z","iopub.execute_input":"2023-02-27T13:17:25.665396Z","iopub.status.idle":"2023-02-27T13:17:25.674727Z","shell.execute_reply.started":"2023-02-27T13:17:25.665361Z","shell.execute_reply":"2023-02-27T13:17:25.673683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ssim_loss(y_true, y_pred):\n  return tf.reduce_mean(tf.image.ssim(y_true, y_pred, 1))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.676382Z","iopub.execute_input":"2023-02-27T13:17:25.676843Z","iopub.status.idle":"2023-02-27T13:17:25.685670Z","shell.execute_reply.started":"2023-02-27T13:17:25.676704Z","shell.execute_reply":"2023-02-27T13:17:25.684494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc_ssim_content=[]\nacc_ssim_style=[]","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.687317Z","iopub.execute_input":"2023-02-27T13:17:25.687810Z","iopub.status.idle":"2023-02-27T13:17:25.695473Z","shell.execute_reply.started":"2023-02-27T13:17:25.687677Z","shell.execute_reply":"2023-02-27T13:17:25.694549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir ../images\nimport shutil\nimport time\nimport random","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:25.696938Z","iopub.execute_input":"2023-02-27T13:17:25.697575Z","iopub.status.idle":"2023-02-27T13:17:26.733031Z","shell.execute_reply.started":"2023-02-27T13:17:25.697540Z","shell.execute_reply":"2023-02-27T13:17:26.731775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_loss=[]\ntotal_ssim_content=[]\ntotal_ssim_style=[]\nnum_of_images=7001\nfor j in range(num_of_images):\n    for im in train_photo.skip(random.randint(0,500)).take(1):\n        content_img = (im.numpy()/255)\n    for im in train_monet.skip(random.randint(0,500)).take(1):\n        style_img = (im.numpy()/255)\n#     style_img=style_train[j]\n#     content_img=content_train[j]\n    losses=[]\n    content_ssim=[]\n    style_ssim=[]\n    extractor=entire_model(style_layers, content_layers)\n    style_targets = extractor(style_img)['style']\n    content_targets = extractor(content_img)['content']\n    generate_image = tf.Variable(content_img)\n    opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)\n    # Since this is a float image, define a function to keep the pixel values between 0 and 1\n    def clip_0_1(image):\n        return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)\n    \n    num_iterations=5\n    start_time = time.time()\n    # opt.build([generate_image])\n    print(\"[Photo Number:\",j+1,\"]\")\n    for i in range(num_iterations+1):\n        loss=train_step(generate_image)\n        if(i%25==0):\n            c_ssim=ssim_loss(content_img,generate_image.read_value()).numpy()\n            s_ssim=ssim_loss(style_img,generate_image.read_value()).numpy()\n            content_ssim.append(c_ssim)\n            style_ssim.append(s_ssim)\n            losses.append(loss)\n            g_img=generate_image.read_value().numpy()\n            print(\"Iteration:\",i,\", Loss:\",loss)\n            print(\"Content Loss:\",ssim_loss(content_img,g_img).numpy())\n            print(\"Style Loss:\",ssim_loss(style_img,g_img).numpy())\n            \n    print(\"Time: {:.2f} seconds\".format( time.time() - start_time))\n    show_plot(content_img,style_img,g_img,i)\n    print(\"\\n\")\n    total_loss.append(losses)\n    total_ssim_content.append(content_ssim)\n    total_ssim_style.append(style_ssim)\n    plt.imshow(g_img[0])\n    plt.axis('off')\n    plt.savefig(\"../images/\" + str(j)+\".jpg\") ","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:26.735348Z","iopub.execute_input":"2023-02-27T13:17:26.736071Z","iopub.status.idle":"2023-02-27T13:17:41.321173Z","shell.execute_reply.started":"2023-02-27T13:17:26.736024Z","shell.execute_reply":"2023-02-27T13:17:41.320100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:41.322760Z","iopub.execute_input":"2023-02-27T13:17:41.323373Z","iopub.status.idle":"2023-02-27T13:17:41.347716Z","shell.execute_reply.started":"2023-02-27T13:17:41.323332Z","shell.execute_reply":"2023-02-27T13:17:41.344859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losses_avg=[0]*len(total_loss[0])\nfor l in total_loss:\n    for i in range(len(l)):\n      losses_avg[i]+=l[i]/num_of_images     ","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:41.349291Z","iopub.execute_input":"2023-02-27T13:17:41.349970Z","iopub.status.idle":"2023-02-27T13:17:41.355858Z","shell.execute_reply.started":"2023-02-27T13:17:41.349933Z","shell.execute_reply":"2023-02-27T13:17:41.354653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssim_content_avg=[0]*len(total_ssim_content[0])\nfor l in total_ssim_content:\n    for i in range(len(l)):\n      ssim_content_avg[i]+=l[i]/num_of_images\nacc_ssim_content.append(ssim_content_avg)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:41.358005Z","iopub.execute_input":"2023-02-27T13:17:41.358862Z","iopub.status.idle":"2023-02-27T13:17:41.365360Z","shell.execute_reply.started":"2023-02-27T13:17:41.358824Z","shell.execute_reply":"2023-02-27T13:17:41.364407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssim_style_avg=[0]*len(total_ssim_style[0])\nfor l in total_ssim_style:\n    for i in range(len(l)):\n      ssim_style_avg[i]+=l[i]/num_of_images    \nacc_ssim_style.append(ssim_style_avg)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:41.373368Z","iopub.execute_input":"2023-02-27T13:17:41.373949Z","iopub.status.idle":"2023-02-27T13:17:41.381347Z","shell.execute_reply.started":"2023-02-27T13:17:41.373913Z","shell.execute_reply":"2023-02-27T13:17:41.379001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nplt.plot(losses)\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\n# plt.legend(['train', 'val'], loc='upper right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:41.382674Z","iopub.execute_input":"2023-02-27T13:17:41.387185Z","iopub.status.idle":"2023-02-27T13:17:41.645152Z","shell.execute_reply.started":"2023-02-27T13:17:41.387149Z","shell.execute_reply":"2023-02-27T13:17:41.644123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nplt.plot(ssim_content_avg)\nplt.plot(ssim_style_avg)\nplt.title('model ssim')\nplt.ylabel('value')\nplt.xlabel('iteration')\nplt.legend(['content', 'style'], loc='upper right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T13:17:41.646585Z","iopub.execute_input":"2023-02-27T13:17:41.647697Z","iopub.status.idle":"2023-02-27T13:17:41.883549Z","shell.execute_reply.started":"2023-02-27T13:17:41.647618Z","shell.execute_reply":"2023-02-27T13:17:41.882596Z"},"trusted":true},"execution_count":null,"outputs":[]}]}