{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Utils","metadata":{"id":"buJeccWRZX26","papermill":{"duration":0.009995,"end_time":"2023-03-19T22:36:17.861569","exception":false,"start_time":"2023-03-19T22:36:17.851574","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Import Libraries","metadata":{"id":"52FrgSEzQYWb","papermill":{"duration":0.008852,"end_time":"2023-03-19T22:36:17.879872","exception":false,"start_time":"2023-03-19T22:36:17.871020","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%capture\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport shutil\nimport tensorflow as tf\nimport tensorflow_datasets as tfds\nimport os\nfrom tensorflow import keras\nimport random\nimport time\nfrom IPython.display import clear_output","metadata":{"id":"DVQDuMr52due","papermill":{"duration":8.360062,"end_time":"2023-03-19T22:36:26.249025","exception":false,"start_time":"2023-03-19T22:36:17.888963","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-21T11:17:32.104396Z","iopub.execute_input":"2023-03-21T11:17:32.105014Z","iopub.status.idle":"2023-03-21T11:17:45.800265Z","shell.execute_reply.started":"2023-03-21T11:17:32.104957Z","shell.execute_reply":"2023-03-21T11:17:45.798699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset\nmonet_jpg - 300 Monet paintings sized 256x256 in JPEG format
\nmonet_tfrec - 300 Monet paintings sized 256x256 in TFRecord format
\nphoto_jpg - 7028 photos sized 256x256 in JPEG format
\nphoto_tfrec - 7028 photos sized 256x256 in TFRecord format","metadata":{"id":"1hxROtsyjWPW","papermill":{"duration":0.010129,"end_time":"2023-03-19T22:36:26.268754","exception":false,"start_time":"2023-03-19T22:36:26.258625","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### File names","metadata":{"id":"IlSkIXYtYQi_","papermill":{"duration":0.009116,"end_time":"2023-03-19T22:36:26.287577","exception":false,"start_time":"2023-03-19T22:36:26.278461","status":"completed"},"tags":[]}},{"cell_type":"code","source":"MONET_FILENAMES = tf.io.gfile.glob('/kaggle/input/gan-getting-started/monet_tfrec/*.tfrec')\nPHOTO_FILENAMES = tf.io.gfile.glob('/kaggle/input/gan-getting-started/photo_tfrec/*.tfrec')\n\nprint(len(MONET_FILENAMES))\nprint(len(PHOTO_FILENAMES))","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:26.307838Z","iopub.status.busy":"2023-03-19T22:36:26.307256Z","iopub.status.idle":"2023-03-19T22:36:26.326242Z","shell.execute_reply":"2023-03-19T22:36:26.325086Z"},"id":"pOr9XYNO4QS1","papermill":{"duration":0.03166,"end_time":"2023-03-19T22:36:26.328422","exception":false,"start_time":"2023-03-19T22:36:26.296762","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load dataset using tensorflow","metadata":{"papermill":{"duration":0.009612,"end_time":"2023-03-19T22:36:26.347534","exception":false,"start_time":"2023-03-19T22:36:26.337922","status":"completed"},"tags":[]}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ndef read_tfrecord(example):\n tfrecord_format = {\n \"image_name\": tf.io.FixedLenFeature([], tf.string),\n \"image\": tf.io.FixedLenFeature([], tf.string),\n \"target\": tf.io.FixedLenFeature([], tf.string)\n }\n example = tf.io.parse_single_example(example, tfrecord_format)\n image = tf.image.decode_jpeg(example['image'], channels=3)\n return image\n\ndef load_dataset(filenames):\n dataset = tf.data.TFRecordDataset(filenames)\n dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n return dataset","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:26.368235Z","iopub.status.busy":"2023-03-19T22:36:26.367260Z","iopub.status.idle":"2023-03-19T22:36:26.374410Z","shell.execute_reply":"2023-03-19T22:36:26.373385Z"},"papermill":{"duration":0.019468,"end_time":"2023-03-19T22:36:26.376216","exception":false,"start_time":"2023-03-19T22:36:26.356748","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"monet_train_images = load_dataset(MONET_FILENAMES)\nphoto_train_images = load_dataset(PHOTO_FILENAMES)","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:26.396450Z","iopub.status.busy":"2023-03-19T22:36:26.396067Z","iopub.status.idle":"2023-03-19T22:36:26.584825Z","shell.execute_reply":"2023-03-19T22:36:26.583879Z"},"papermill":{"duration":0.201727,"end_time":"2023-03-19T22:36:26.587162","exception":false,"start_time":"2023-03-19T22:36:26.385435","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Function - Plot Single Example\nThis function receives an image and a label and it will display it on a plot","metadata":{"id":"UxFQGdYzYffQ","papermill":{"duration":0.009266,"end_time":"2023-03-19T22:36:26.606252","exception":false,"start_time":"2023-03-19T22:36:26.596986","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_single_example(monet, photo):\n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 7))\n\n ax1.imshow(monet)\n ax1.set_title(\"Style image\")\n ax1.axis('off')\n\n ax2.imshow(photo)\n ax2.set_title(\"Content image\")\n ax2.axis('off')\n plt.show()","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:26.626681Z","iopub.status.busy":"2023-03-19T22:36:26.626294Z","iopub.status.idle":"2023-03-19T22:36:26.632334Z","shell.execute_reply":"2023-03-19T22:36:26.631192Z"},"id":"zHABcIRJZOTC","papermill":{"duration":0.018879,"end_time":"2023-03-19T22:36:26.634365","exception":false,"start_time":"2023-03-19T22:36:26.615486","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{"id":"5trNlNNHawd6","papermill":{"duration":0.009125,"end_time":"2023-03-19T22:36:26.652911","exception":false,"start_time":"2023-03-19T22:36:26.643786","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Show number of examples in monet and photo\n","metadata":{"id":"t6hlzURV498F","papermill":{"duration":0.009037,"end_time":"2023-03-19T22:36:26.671233","exception":false,"start_time":"2023-03-19T22:36:26.662196","status":"completed"},"tags":[]}},{"cell_type":"code","source":"images_number_monet = 0\nimages_number_photo = 0\n\nfor image in monet_train_images:\n images_number_monet += 1\n\nfor image in photo_train_images:\n images_number_photo += 1\n\nprint(f\"Number of monet images: {images_number_monet}\")\nprint(f\"Number of photo images: {images_number_photo}\")","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:26.691958Z","iopub.status.busy":"2023-03-19T22:36:26.690997Z","iopub.status.idle":"2023-03-19T22:36:37.665315Z","shell.execute_reply":"2023-03-19T22:36:37.664186Z"},"id":"HYsfRMHu5Ew8","outputId":"6396d32b-6092-4e86-ad5e-a3c1cea0ffdd","papermill":{"duration":10.986849,"end_time":"2023-03-19T22:36:37.667259","exception":false,"start_time":"2023-03-19T22:36:26.680410","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Example\nRandom example from the data set","metadata":{"id":"AY8o0dPQ-_vp","papermill":{"duration":0.009347,"end_time":"2023-03-19T22:36:37.686466","exception":false,"start_time":"2023-03-19T22:36:37.677119","status":"completed"},"tags":[]}},{"cell_type":"code","source":"monet_image = monet_train_images.as_numpy_iterator().next()\nphoto_image = photo_train_images.as_numpy_iterator().next()\n\nplot_single_example(monet_image, photo_image)","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:37.707578Z","iopub.status.busy":"2023-03-19T22:36:37.707048Z","iopub.status.idle":"2023-03-19T22:36:38.125597Z","shell.execute_reply":"2023-03-19T22:36:38.124014Z"},"id":"Q3carekO_A2w","outputId":"88455eac-4493-43d4-8a0f-f614e22436c2","papermill":{"duration":0.435255,"end_time":"2023-03-19T22:36:38.131466","exception":false,"start_time":"2023-03-19T22:36:37.696211","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Resize","metadata":{"id":"I2Sa6XcJhLKy","papermill":{"duration":0.01762,"end_time":"2023-03-19T22:36:38.167010","exception":false,"start_time":"2023-03-19T22:36:38.149390","status":"completed"},"tags":[]}},{"cell_type":"code","source":"height = 320\nwidth = 320\n\ndef resize_dataset(x, img_height: int=height, img_width: int=width):\n x = tf.image.resize(x, (img_height, img_width))\n return x","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.204699Z","iopub.status.busy":"2023-03-19T22:36:38.204369Z","iopub.status.idle":"2023-03-19T22:36:38.209053Z","shell.execute_reply":"2023-03-19T22:36:38.208079Z"},"id":"hKaw2PYmjMRw","papermill":{"duration":0.025564,"end_time":"2023-03-19T22:36:38.210688","exception":false,"start_time":"2023-03-19T22:36:38.185124","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalize","metadata":{"id":"hqtx8JbRmYxj","papermill":{"duration":0.017885,"end_time":"2023-03-19T22:36:38.246940","exception":false,"start_time":"2023-03-19T22:36:38.229055","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def normalize_dataset(x):\n x = tf.cast(x, tf.float32)\n x = x / 255.0\n return x","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.284800Z","iopub.status.busy":"2023-03-19T22:36:38.284460Z","iopub.status.idle":"2023-03-19T22:36:38.289307Z","shell.execute_reply":"2023-03-19T22:36:38.288333Z"},"id":"BU1YpX-hs9H5","papermill":{"duration":0.026347,"end_time":"2023-03-19T22:36:38.291099","exception":false,"start_time":"2023-03-19T22:36:38.264752","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing function","metadata":{"papermill":{"duration":0.017645,"end_time":"2023-03-19T22:36:38.326519","exception":false,"start_time":"2023-03-19T22:36:38.308874","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def preprocessing(x):\n x = resize_dataset(x)\n x = normalize_dataset(x)\n return x","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.363886Z","iopub.status.busy":"2023-03-19T22:36:38.363515Z","iopub.status.idle":"2023-03-19T22:36:38.368192Z","shell.execute_reply":"2023-03-19T22:36:38.366896Z"},"papermill":{"duration":0.025795,"end_time":"2023-03-19T22:36:38.370055","exception":false,"start_time":"2023-03-19T22:36:38.344260","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Apply preprocessing on datasets","metadata":{"papermill":{"duration":0.01769,"end_time":"2023-03-19T22:36:38.405398","exception":false,"start_time":"2023-03-19T22:36:38.387708","status":"completed"},"tags":[]}},{"cell_type":"code","source":"batch_size = 1\n\ntrain_monet_after_preprocessing=monet_train_images.map(preprocessing, AUTO).cache().batch(batch_size).prefetch(AUTO)\ntrain_photo_after_preprocessing=photo_train_images.map(preprocessing, AUTO).cache().batch(batch_size).prefetch(AUTO)","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.442676Z","iopub.status.busy":"2023-03-19T22:36:38.442076Z","iopub.status.idle":"2023-03-19T22:36:38.571323Z","shell.execute_reply":"2023-03-19T22:36:38.570292Z"},"id":"pAh_BJ7hZbR8","papermill":{"duration":0.1509,"end_time":"2023-03-19T22:36:38.573945","exception":false,"start_time":"2023-03-19T22:36:38.423045","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Style Transfer Model","metadata":{"id":"e9PF7Mo6fM-s","papermill":{"duration":0.017314,"end_time":"2023-03-19T22:36:38.608856","exception":false,"start_time":"2023-03-19T22:36:38.591542","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Create new model with relevant layers","metadata":{"id":"3FHZ6HdFNzHb","papermill":{"duration":0.017026,"end_time":"2023-03-19T22:36:38.643472","exception":false,"start_time":"2023-03-19T22:36:38.626446","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def create_new_model_from_model_relevant_layers(relevant_layers_list, model):\n outputs = [model.get_layer(layer_name).output for layer_name in relevant_layers_list]\n new_model = tf.keras.Model([model.input], outputs)\n return new_model","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.680023Z","iopub.status.busy":"2023-03-19T22:36:38.679665Z","iopub.status.idle":"2023-03-19T22:36:38.684162Z","shell.execute_reply":"2023-03-19T22:36:38.683487Z"},"id":"ZLl70uFj_7L-","papermill":{"duration":0.024955,"end_time":"2023-03-19T22:36:38.685873","exception":false,"start_time":"2023-03-19T22:36:38.660918","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Gram matrix","metadata":{"id":"HkqxMd9bN5Aa","papermill":{"duration":0.017328,"end_time":"2023-03-19T22:36:38.720968","exception":false,"start_time":"2023-03-19T22:36:38.703640","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Gram matrix\ndef gram_matrix(some_tensor):\n temp_tensor = some_tensor\n temp_tensor = tf.squeeze(temp_tensor)\n result = tf.matmul(temp_tensor, temp_tensor, transpose_b=True)\n gram = tf.expand_dims(result, axis=0)\n return gram","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.757912Z","iopub.status.busy":"2023-03-19T22:36:38.757157Z","iopub.status.idle":"2023-03-19T22:36:38.762189Z","shell.execute_reply":"2023-03-19T22:36:38.761553Z"},"id":"au8QSDmhBrLg","papermill":{"duration":0.025285,"end_time":"2023-03-19T22:36:38.763855","exception":false,"start_time":"2023-03-19T22:36:38.738570","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Style model class","metadata":{"id":"q_4BKUuvOBCI","papermill":{"duration":0.017398,"end_time":"2023-03-19T22:36:38.798723","exception":false,"start_time":"2023-03-19T22:36:38.781325","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class style_model(tf.keras.models.Model):\n def __init__(self, style_layers, content_layers, model):\n super(style_model, self).__init__()\n self.model = model\n self.vgg = create_new_model_from_model_relevant_layers(style_and_content_layers, self.model)\n self.style_layers = style_layers\n self.content_layers = content_layers\n self.num_style_layers = len(style_layers)\n self.vgg.trainable = False\n\n def call(self, inputs):\n # Scale back the pixel values\n inputs = inputs * 255.0\n # Preprocess them with respect to VGG19 stats\n preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs)\n # Pass through the mini network\n outputs = self.vgg(preprocessed_input)\n # Segregate the style and content representations\n style_outputs, content_outputs = (outputs[:self.num_style_layers],\n outputs[self.num_style_layers:])\n\n # Calculate the gram matrix for each layer\n style_outputs = [gram_matrix(style_output)\n for style_output in style_outputs]\n\n # Assign the content representation and gram matrix in\n # a layer by layer fashion in dicts\n content_dict = {content_name:value\n for content_name, value\n in zip(self.content_layers, content_outputs)}\n\n style_dict = {style_name:value\n for style_name, value\n in zip(self.style_layers, style_outputs)}\n\n return {'content':content_dict, 'style':style_dict}","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.836153Z","iopub.status.busy":"2023-03-19T22:36:38.835405Z","iopub.status.idle":"2023-03-19T22:36:38.843380Z","shell.execute_reply":"2023-03-19T22:36:38.842742Z"},"id":"EApYPlbQDCtg","papermill":{"duration":0.028698,"end_time":"2023-03-19T22:36:38.845051","exception":false,"start_time":"2023-03-19T22:36:38.816353","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Total Loss function","metadata":{"papermill":{"duration":0.017182,"end_time":"2023-03-19T22:36:38.879796","exception":false,"start_time":"2023-03-19T22:36:38.862614","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# The loss function to optimize\ndef style_content_and_total_loss(outputs):\n style_outputs=outputs['style']\n content_outputs=outputs['content']\n style_loss=tf.add_n([style_weights[name] * tf.reduce_mean((style_outputs[name]-style_targets[name])**2)\n for name in style_outputs.keys()])\n style_loss*=style_weight/len(style_layers)\n\n content_loss = tf.add_n([tf.reduce_mean((content_outputs[name]-content_targets[name])**2)\n for name in content_outputs.keys()])\n content_loss*=content_weight/len(content_layers)\n loss=style_loss+content_loss\n return loss, style_loss, content_loss","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.916312Z","iopub.status.busy":"2023-03-19T22:36:38.915747Z","iopub.status.idle":"2023-03-19T22:36:38.921576Z","shell.execute_reply":"2023-03-19T22:36:38.920965Z"},"papermill":{"duration":0.025952,"end_time":"2023-03-19T22:36:38.923132","exception":false,"start_time":"2023-03-19T22:36:38.897180","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model train step function","metadata":{"papermill":{"duration":0.016903,"end_time":"2023-03-19T22:36:38.957377","exception":false,"start_time":"2023-03-19T22:36:38.940474","status":"completed"},"tags":[]}},{"cell_type":"code","source":"@tf.function()\ndef train_step(image):\n with tf.GradientTape() as tape:\n # Extract the features\n outputs = extractor(image)\n # Calculate the loss\n loss, style_loss, content_loss = style_content_and_total_loss(outputs)\n\n # Determine the gradients of the loss function w.r.t the image pixels\n grad = tape.gradient(loss, image)\n\n # Update the pixels\n opt.apply_gradients([(grad, image)])\n \n # Clip the pixel values that fall outside the range of [0,1]\n image.assign(tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0))\n\n return loss, style_loss, content_loss","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:38.994319Z","iopub.status.busy":"2023-03-19T22:36:38.993739Z","iopub.status.idle":"2023-03-19T22:36:38.999226Z","shell.execute_reply":"2023-03-19T22:36:38.998589Z"},"papermill":{"duration":0.025946,"end_time":"2023-03-19T22:36:39.000869","exception":false,"start_time":"2023-03-19T22:36:38.974923","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save image function","metadata":{"id":"SfmLv3QpaRjp","papermill":{"duration":0.017584,"end_time":"2023-03-19T22:36:39.036127","exception":false,"start_time":"2023-03-19T22:36:39.018543","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def save_image(image_number, image, image_save_location, resize: tuple=()):\n image_number_str = str(image_number)\n image_name = f\"{image_number_str}.jpg\"\n squeeze_image = np.squeeze(image, axis=0)\n \n if resize:\n squeeze_image = resize_dataset(squeeze_image, resize[0], resize[1]).numpy()\n\n plt.imsave(image_save_location + image_name, squeeze_image)\n\n print(f\"Image {image_number_str}, with shape of {squeeze_image.shape}, saved in location:\")\n print(image_save_location + image_name)","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:39.073639Z","iopub.status.busy":"2023-03-19T22:36:39.073126Z","iopub.status.idle":"2023-03-19T22:36:39.078105Z","shell.execute_reply":"2023-03-19T22:36:39.077443Z"},"id":"wrIR6MyyaQjv","papermill":{"duration":0.025706,"end_time":"2023-03-19T22:36:39.079713","exception":false,"start_time":"2023-03-19T22:36:39.054007","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot content, style and generated images","metadata":{"id":"mj4fDJT09wfm","papermill":{"duration":0.017502,"end_time":"2023-03-19T22:36:39.114774","exception":false,"start_time":"2023-03-19T22:36:39.097272","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_images_with_generated(iteration: int, content_image, style_image, generated_image) -> None:\n images_list = [content_image, style_image, generated_image]\n images_titles = [\"Content Image\",\n \"Style Image\",\n \"Generated Image - {0} step\".format(iteration)]\n fig, axs = plt.subplots(1, len(images_list), figsize=(14, 10))\n i = 0\n\n for (image, title) in zip(images_list, images_titles):\n axs[i].imshow(np.squeeze(image, axis=0))\n axs[i].set_title(title)\n axs[i].axis('off')\n i += 1\n\n plt.show()","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:39.151900Z","iopub.status.busy":"2023-03-19T22:36:39.151571Z","iopub.status.idle":"2023-03-19T22:36:39.158164Z","shell.execute_reply":"2023-03-19T22:36:39.157312Z"},"id":"qwQFrlwvJXUS","papermill":{"duration":0.027365,"end_time":"2023-03-19T22:36:39.159922","exception":false,"start_time":"2023-03-19T22:36:39.132557","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create and train style transfer model","metadata":{"papermill":{"duration":0.01791,"end_time":"2023-03-19T22:36:39.195763","exception":false,"start_time":"2023-03-19T22:36:39.177853","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Load pre-trained model","metadata":{"id":"xalP3shHogvT","papermill":{"duration":0.017909,"end_time":"2023-03-19T22:36:39.231824","exception":false,"start_time":"2023-03-19T22:36:39.213915","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Import VGG19 model\nVGG19_model = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\n\n# Freeze VGG19 model from being trained\nVGG19_model.trainable = False","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:39.270777Z","iopub.status.busy":"2023-03-19T22:36:39.269736Z","iopub.status.idle":"2023-03-19T22:36:42.551542Z","shell.execute_reply":"2023-03-19T22:36:42.550565Z"},"id":"r09nsD8xohs-","papermill":{"duration":3.304018,"end_time":"2023-03-19T22:36:42.554007","exception":false,"start_time":"2023-03-19T22:36:39.249989","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the layer names for convenience\nfor layer in VGG19_model.layers:\n print(layer.name)","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:42.597773Z","iopub.status.busy":"2023-03-19T22:36:42.597420Z","iopub.status.idle":"2023-03-19T22:36:42.601996Z","shell.execute_reply":"2023-03-19T22:36:42.601335Z"},"id":"N-986lYfopka","papermill":{"duration":0.028477,"end_time":"2023-03-19T22:36:42.603746","exception":false,"start_time":"2023-03-19T22:36:42.575269","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save relevant layers for content and style","metadata":{"id":"vgk9y9AuouAu","papermill":{"duration":0.020316,"end_time":"2023-03-19T22:36:42.644460","exception":false,"start_time":"2023-03-19T22:36:42.624144","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Style layers - low level features(like lines, edges, dots, and curves)\nstyle_layers = ['block1_conv1',\n 'block2_conv1',\n 'block3_conv1',\n 'block4_conv1',\n 'block5_conv1']\n\n# Content layers - high level features(like humans, buildings, and items)\ncontent_layers = ['block4_conv2']\n\nstyle_and_content_layers = style_layers + content_layers\n\nnum_content_layers = len(content_layers)\nnum_style_layers = len(style_layers)","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:42.688251Z","iopub.status.busy":"2023-03-19T22:36:42.687614Z","iopub.status.idle":"2023-03-19T22:36:42.692708Z","shell.execute_reply":"2023-03-19T22:36:42.692056Z"},"id":"zM9lsZBjou_P","papermill":{"duration":0.029242,"end_time":"2023-03-19T22:36:42.694457","exception":false,"start_time":"2023-03-19T22:36:42.665215","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define custom weights","metadata":{"papermill":{"duration":0.025215,"end_time":"2023-03-19T22:36:42.739988","exception":false,"start_time":"2023-03-19T22:36:42.714773","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Custom weights for style and content updates\nstyle_weight=150\ncontent_weight=10\n\n# Custom weights for different style layers\nstyle_weights = {'block1_conv1': 1.,\n 'block2_conv1': 0.8,\n 'block3_conv1': 0.6,\n 'block4_conv1': 0.5,\n 'block5_conv1': 0.3} ","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:42.784379Z","iopub.status.busy":"2023-03-19T22:36:42.783814Z","iopub.status.idle":"2023-03-19T22:36:42.787910Z","shell.execute_reply":"2023-03-19T22:36:42.787258Z"},"papermill":{"duration":0.027826,"end_time":"2023-03-19T22:36:42.789552","exception":false,"start_time":"2023-03-19T22:36:42.761726","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters","metadata":{"id":"caJOGfKLKnaY","papermill":{"duration":0.021526,"end_time":"2023-03-19T22:36:42.833203","exception":false,"start_time":"2023-03-19T22:36:42.811677","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Hyperparameters\nnumber_of_images = 1\nepochs = 2000\nlearning_rate=0.001\nbeta_1=0.92\nepsilon=1e-07\n\nearly_stop_threshold = 4343185000000\nearly_stop_patience = 10","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:42.877231Z","iopub.status.busy":"2023-03-19T22:36:42.876639Z","iopub.status.idle":"2023-03-19T22:36:42.880722Z","shell.execute_reply":"2023-03-19T22:36:42.880065Z"},"id":"evhHnTFdKmRS","papermill":{"duration":0.028244,"end_time":"2023-03-19T22:36:42.882421","exception":false,"start_time":"2023-03-19T22:36:42.854177","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train and create generated images","metadata":{"id":"-vXDXbzT96My","papermill":{"duration":0.020005,"end_time":"2023-03-19T22:36:42.923267","exception":false,"start_time":"2023-03-19T22:36:42.903262","status":"completed"},"tags":[]}},{"cell_type":"code","source":"image_save_location = \"../images/\"\n\n# Create folder for generated images\n!mkdir ../images\n\nloss_all = []\nbest_loss_all = []\nbest_step_loss_all = []\n\nstyle_loss_list_all = []\ncontent_loss_list_all = []\n\nstyle_accuracy_list_all = []\ncontent_accuracy_list_all = []","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:42.967995Z","iopub.status.busy":"2023-03-19T22:36:42.967408Z","iopub.status.idle":"2023-03-19T22:36:43.236645Z","shell.execute_reply":"2023-03-19T22:36:43.235584Z"},"papermill":{"duration":0.294774,"end_time":"2023-03-19T22:36:43.239058","exception":false,"start_time":"2023-03-19T22:36:42.944284","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate images\nfor image_number in range(1, number_of_images + 1):\n random_style_image = train_monet_after_preprocessing.skip(random.randint(0,images_number_monet - 1)).take(1)\n random_content_image = train_photo_after_preprocessing.skip(random.randint(0,images_number_photo - 1)).take(1)\n\n random_style_image = np.squeeze(np.array(list(random_style_image.as_numpy_iterator())), axis=0)\n random_content_image = np.squeeze(np.array(list(random_content_image.as_numpy_iterator())), axis=0)\n\n generate_image = tf.Variable(random_content_image)\n\n losses_current = []\n style_loss_list_current = []\n content_loss_list_current = []\n\n extractor = style_model(style_layers, content_layers, VGG19_model)\n style_targets = extractor(random_style_image)['style']\n content_targets = extractor(random_content_image)['content']\n\n opt = tf.optimizers.Adam(learning_rate=learning_rate, beta_1=beta_1, epsilon=epsilon)\n\n early_stop_counter = 0\n best_loss = None\n best_image = None\n best_step = None\n\n start_time = time.time()\n print(\"[Epoch Number:\",image_number,\"]\")\n\n # Train image number of epochs\n for i in range(epochs):\n loss, style_loss, content_loss = train_step(generate_image)\n\n if best_loss:\n if best_loss - loss >= early_stop_threshold:\n best_loss = loss\n best_image = generate_image.read_value().numpy()\n best_step = i\n early_stop_counter = 0\n else:\n early_stop_counter += 1\n else:\n best_loss = loss\n best_image = generate_image.read_value().numpy()\n best_step = i\n\n if(i%20==0):\n generate_image_current = generate_image.read_value().numpy()\n\n losses_current.append(loss)\n style_loss_list_current.append(style_loss)\n content_loss_list_current.append(content_loss)\n\n generate_targets = extractor(generate_image_current)\n\n print(f\"Step: {i}\")\n print(f\"Loss: {loss.numpy()}\")\n print(f\"Style Loss: {style_loss.numpy()}\")\n print(f\"Content Loss: {content_loss.numpy()}\")\n\n if (i%100 == 0):\n plot_images_with_generated(str(i), random_content_image, random_style_image, generate_image_current)\n\n if early_stop_counter == early_stop_patience and best_loss:\n generate_image_current = best_image\n print(f\"Stopped at iteration {i} after {early_stop_counter} iteration best_loss didnt updated more than {early_stop_threshold} loss\")\n break\n\n generate_image_last = generate_image.read_value().numpy()\n\n clear_output(wait=True)\n\n print(\"Image generate total time: {:.2f} seconds\".format(time.time() - start_time))\n plot_images_with_generated(f\"BEST IMAGE {best_step}\", random_content_image, random_style_image, best_image)\n\n loss_all.append(losses_current)\n best_loss_all.append(best_loss)\n best_step_loss_all.append(best_step)\n style_loss_list_all.append(style_loss_list_current)\n content_loss_list_all.append(content_loss_list_current)\n\n save_image(image_number, generate_image_last, image_save_location, (256,256))","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:36:43.284351Z","iopub.status.busy":"2023-03-19T22:36:43.283097Z","iopub.status.idle":"2023-03-19T22:38:18.182625Z","shell.execute_reply":"2023-03-19T22:38:18.181371Z"},"id":"fH35WNG5GsxO","outputId":"6e4d59c4-6ad6-481a-c6c1-e82b4fa48c20","papermill":{"duration":94.925498,"end_time":"2023-03-19T22:38:18.185306","exception":false,"start_time":"2023-03-19T22:36:43.259808","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save images in zip","metadata":{"papermill":{"duration":0.027629,"end_time":"2023-03-19T22:38:18.240967","exception":false,"start_time":"2023-03-19T22:38:18.213338","status":"completed"},"tags":[]}},{"cell_type":"code","source":"shutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/input/images\")","metadata":{"id":"xuKwSZSKACXu","papermill":{"duration":0.03778,"end_time":"2023-03-19T22:38:18.306078","exception":false,"start_time":"2023-03-19T22:38:18.268298","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-21T11:19:19.696392Z","iopub.execute_input":"2023-03-21T11:19:19.696830Z","iopub.status.idle":"2023-03-21T11:19:50.974477Z","shell.execute_reply.started":"2023-03-21T11:19:19.696795Z","shell.execute_reply":"2023-03-21T11:19:50.973009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot model loss","metadata":{"papermill":{"duration":0.026619,"end_time":"2023-03-19T22:38:18.359776","exception":false,"start_time":"2023-03-19T22:38:18.333157","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# First image loss\nfirstTotalLossList = [x.numpy() for x in loss_all[0]]\nlen_of_loss = [i*20 for i in range(len(firstTotalLossList))]\n\n\nplt.figure(figsize=(7, 5))\nplt.plot(len_of_loss, firstTotalLossList)\nplt.title(\"Model loss for first image\")\nplt.xlabel(\"epoch\")\nplt.ylabel(\"loss\")\n\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-03-19T22:38:18.415864Z","iopub.status.busy":"2023-03-19T22:38:18.415350Z","iopub.status.idle":"2023-03-19T22:38:18.564166Z","shell.execute_reply":"2023-03-19T22:38:18.563465Z"},"papermill":{"duration":0.179084,"end_time":"2023-03-19T22:38:18.565900","exception":false,"start_time":"2023-03-19T22:38:18.386816","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}