{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Neural Style Transfer","metadata":{"id":"6bZSpdC52NNm"}},{"cell_type":"markdown","source":"##Utils","metadata":{"id":"EK4XSsnuGqnV"}},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom keras.applications import vgg19\nfrom keras import preprocessing\nimport random\nimport os\nfrom os import listdir\nimport shutil\nimport zipfile\nimport cv2","metadata":{"id":"_zvHsHQLsbsE","outputId":"dabedf70-296e-45bf-ccca-d9f1379ac829","execution":{"iopub.status.busy":"2023-03-21T22:56:39.985753Z","iopub.execute_input":"2023-03-21T22:56:39.98647Z","iopub.status.idle":"2023-03-21T22:56:47.453961Z","shell.execute_reply.started":"2023-03-21T22:56:39.986432Z","shell.execute_reply":"2023-03-21T22:56:47.452779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{"id":"s2A9seiA3f7S"}},{"cell_type":"markdown","source":"a function to open, resize and format pictures into appropriate tensors","metadata":{"id":"I6SJZUEVqpq3"}},{"cell_type":"code","source":"def preprocess_image(image_path):\n  img = tf.keras.utils.load_img(image_path, target_size=(image_height, image_width))\n  img =  tf.keras.utils.img_to_array(img)\n  img = np.expand_dims(img, axis=0)\n  img = vgg19.preprocess_input(img)\n  return tf.convert_to_tensor(img)","metadata":{"id":"Tiboh0kmqtFZ","execution":{"iopub.status.busy":"2023-03-21T22:56:47.455956Z","iopub.execute_input":"2023-03-21T22:56:47.456609Z","iopub.status.idle":"2023-03-21T22:56:47.46354Z","shell.execute_reply.started":"2023-03-21T22:56:47.456578Z","shell.execute_reply":"2023-03-21T22:56:47.46245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"a functions to convert a tensor into a valid image, deprocessing the vgg19 preprocessing","metadata":{"id":"Jv67FyVkqtfs"}},{"cell_type":"code","source":"def adding_mean_values(image):\n  vgg19_imagenet_mean_normalization_values = {\n      'red': 123.68,\n      'green': 116.779,\n      'blue': 103.939\n  }\n  image[:, :, 0] += vgg19_imagenet_mean_normalization_values['blue']\n  image[:, :, 1] += vgg19_imagenet_mean_normalization_values['green']\n  image[:, :, 2] += vgg19_imagenet_mean_normalization_values['red']\n  return image\n\ndef BGR_to_RGB(image):\n  return image[:, :, ::-1]\n\ndef deprocess_image(image):\n  image = np.reshape(image, (image_height, image_width, 3))\n  image = adding_mean_values(image)\n  image = BGR_to_RGB(image)\n  image = np.clip(image, 0, 255).astype(\"uint8\")\n  image = cv2.resize(image, dsize=(256,256), interpolation=cv2.INTER_CUBIC)\n\n  return image","metadata":{"id":"i4voeFXeqjOY","execution":{"iopub.status.busy":"2023-03-21T22:56:47.465235Z","iopub.execute_input":"2023-03-21T22:56:47.465994Z","iopub.status.idle":"2023-03-21T22:56:47.475261Z","shell.execute_reply.started":"2023-03-21T22:56:47.465946Z","shell.execute_reply":"2023-03-21T22:56:47.474241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss Functions","metadata":{"id":"gV1Qxt-ilJmf"}},{"cell_type":"markdown","source":"The style loss function allows for the generation of stylized images that match the style of a reference image. By utilizing the gram matrices of feature maps from both the style reference image and the generated image, the style loss function ensures that the generated image maintains the desired style.","metadata":{"id":"bpZbYH452qJn"}},{"cell_type":"code","source":"def gram_matrix(x):\n    x = tf.transpose(x, (2, 0, 1))\n    features = tf.reshape(x, (tf.shape(x)[0], -1))\n    gram = tf.matmul(features, tf.transpose(features))\n    return gram\n\ndef style_loss(style, combination):\n    S = gram_matrix(style)\n    C = gram_matrix(combination)\n    channels = 3\n    size = image_height * image_width\n    return tf.reduce_sum(tf.square(S - C)) / (4.0 * (channels**2) * (size**2))\n","metadata":{"id":"8zxADr6k2iTP","execution":{"iopub.status.busy":"2023-03-21T22:56:47.478489Z","iopub.execute_input":"2023-03-21T22:56:47.478925Z","iopub.status.idle":"2023-03-21T22:56:47.486991Z","shell.execute_reply.started":"2023-03-21T22:56:47.47889Z","shell.execute_reply":"2023-03-21T22:56:47.485044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The content loss function is an auxiliary component in neural style transfer that helps maintain the content of the base image in the generated image by keeping the high-level representation of the generated image close to that of the base image.","metadata":{"id":"rtc51szt71xg"}},{"cell_type":"code","source":"def content_loss(base, combination):\n    return tf.reduce_sum(tf.square(combination - base))","metadata":{"id":"Wt9gRkz271EO","execution":{"iopub.status.busy":"2023-03-21T22:56:47.488828Z","iopub.execute_input":"2023-03-21T22:56:47.489289Z","iopub.status.idle":"2023-03-21T22:56:47.495315Z","shell.execute_reply.started":"2023-03-21T22:56:47.489251Z","shell.execute_reply":"2023-03-21T22:56:47.494333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The total variation loss in neural style transfer is a regularization loss that maintains local coherence of the generated image.","metadata":{"id":"Rf5RBQU58CpK"}},{"cell_type":"code","source":"def total_variation_loss(x):\n    a = tf.square(\n        x[:, : image_height - 1, : image_width - 1, :] - x[:, 1:, : image_width - 1, :]\n    )\n    b = tf.square(\n        x[:, : image_height - 1, : image_width - 1, :] - x[:, : image_height - 1, 1:, :]\n    )\n    return tf.reduce_sum(tf.pow(a + b, 1.25))","metadata":{"id":"9eRkPS3E8CAv","execution":{"iopub.status.busy":"2023-03-21T22:56:47.496722Z","iopub.execute_input":"2023-03-21T22:56:47.497669Z","iopub.status.idle":"2023-03-21T22:56:47.507113Z","shell.execute_reply.started":"2023-03-21T22:56:47.497634Z","shell.execute_reply":"2023-03-21T22:56:47.506157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The function that computes total loss is a combination of the content loss, style loss, and total variation loss, which balances the importance of preserving the content and style of the input images while maintaining local coherence in the generated image.","metadata":{"id":"dfwTTmdsF7dp"}},{"cell_type":"code","source":"def compute_loss(combination_image, base_image, style_reference_image):\n    input_tensor = tf.concat(\n        [base_image, style_reference_image, combination_image], axis=0\n    )\n    features = feature_extractor(input_tensor)\n\n    loss = tf.zeros(shape=())\n\n    # Add content loss\n    layer_features = features[content_layer_name]\n    base_image_features = layer_features[0, :, :, :]\n    combination_features = layer_features[2, :, :, :]\n    loss = loss + content_weight * content_loss(\n        base_image_features, combination_features\n    )\n\n    # Add style loss\n    total_style_loss = tf.zeros(shape=())\n    for style_layer in style_layers:\n        layer_features = features[style_layer['name']]\n        style_reference_features = layer_features[1, :, :, :]\n        combination_features = layer_features[2, :, :, :]\n        sl = style_loss(style_reference_features, combination_features)\n        total_style_loss += style_layer['weight'] * sl\n\n    loss += style_weight * total_style_loss\n\n    # Add total variation loss\n    loss += total_variation_weight * total_variation_loss(combination_image)\n    return loss","metadata":{"id":"N-VentKX8VSF","execution":{"iopub.status.busy":"2023-03-21T22:56:47.509455Z","iopub.execute_input":"2023-03-21T22:56:47.51052Z","iopub.status.idle":"2023-03-21T22:56:47.519984Z","shell.execute_reply.started":"2023-03-21T22:56:47.510481Z","shell.execute_reply":"2023-03-21T22:56:47.518924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build feature extractor model","metadata":{"id":"xnuQTkga1QDi"}},{"cell_type":"markdown","source":" Build a VGG19 model loaded with pre-trained ImageNet weights\n","metadata":{"id":"p9vFCCoP24nL"}},{"cell_type":"code","source":"model = vgg19.VGG19(weights=\"imagenet\", include_top=False)\n\noutputs_dict = dict([(layer.name, layer.output) for layer in model.layers])\nfeature_extractor = tf.keras.Model(inputs=model.inputs, outputs=outputs_dict)","metadata":{"id":"yoPAJ4bm22YM","outputId":"97ddef28-962c-44b1-bcfa-1da6f22398bd","execution":{"iopub.status.busy":"2023-03-21T22:56:47.523127Z","iopub.execute_input":"2023-03-21T22:56:47.523472Z","iopub.status.idle":"2023-03-21T22:56:53.530001Z","shell.execute_reply.started":"2023-03-21T22:56:47.523429Z","shell.execute_reply":"2023-03-21T22:56:53.528942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{"id":"CeTV6_Q8yqs2"}},{"cell_type":"markdown","source":"Loss and gradient computation","metadata":{"id":"C6DpNg8qNcNj"}},{"cell_type":"code","source":"@tf.function\ndef compute_loss_and_grads(combination_image, base_image, style_reference_image):\n    with tf.GradientTape() as tape:\n        loss = compute_loss(combination_image, base_image, style_reference_image)\n    grads = tape.gradient(loss, combination_image)\n    return loss, grads","metadata":{"id":"lKHUp47sNb02","execution":{"iopub.status.busy":"2023-03-21T22:56:53.531499Z","iopub.execute_input":"2023-03-21T22:56:53.53186Z","iopub.status.idle":"2023-03-21T22:56:53.538643Z","shell.execute_reply.started":"2023-03-21T22:56:53.531823Z","shell.execute_reply":"2023-03-21T22:56:53.537502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(content_image, style_image, combination_image, dir_path):\n  for i in range(1, num_iterations + 1):\n    loss, grads = compute_loss_and_grads(\n        combination_image,\n        content_image,\n        style_image)\n    optimizer.apply_gradients([(grads, combination_image)])\n    if i % 100 == 0:\n        print(\"Iteration %d: loss=%.2f\" % (i, loss))\n        img = deprocess_image(combination_image.numpy())\n        fname = dir_path + \"/generated_at_%d.png\" % i\n        tf.keras.utils.save_img(fname, img)\n  \n  return fname","metadata":{"id":"Z6FX3R9c0SpC","execution":{"iopub.status.busy":"2023-03-21T22:56:53.543215Z","iopub.execute_input":"2023-03-21T22:56:53.543913Z","iopub.status.idle":"2023-03-21T22:56:53.551382Z","shell.execute_reply.started":"2023-03-21T22:56:53.543872Z","shell.execute_reply":"2023-03-21T22:56:53.550405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_styled_image(content_image_path, style_image_path, dir_path):\n  content_image = preprocess_image(content_image_path)\n  style_image = preprocess_image(style_image_path)\n  combination_image = tf.Variable(preprocess_image(content_image_path))\n  styled_image_path = train(content_image, style_image, combination_image, dir_path)\n  return styled_image_path","metadata":{"id":"Fv0EgBnvNjDD","execution":{"iopub.status.busy":"2023-03-21T22:56:53.552994Z","iopub.execute_input":"2023-03-21T22:56:53.553529Z","iopub.status.idle":"2023-03-21T22:56:53.563335Z","shell.execute_reply.started":"2023-03-21T22:56:53.553491Z","shell.execute_reply":"2023-03-21T22:56:53.562344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_files_full_path(parent_directory):\n    return [f'{parent_directory}/{f}' for f in listdir(parent_directory)]\n\ndef random_file_full_path(parent_directory):\n    return f'{parent_directory}/{random.choice(listdir(parent_directory))}'","metadata":{"execution":{"iopub.status.busy":"2023-03-21T22:56:53.564885Z","iopub.execute_input":"2023-03-21T22:56:53.565303Z","iopub.status.idle":"2023-03-21T22:56:53.573412Z","shell.execute_reply.started":"2023-03-21T22:56:53.565266Z","shell.execute_reply":"2023-03-21T22:56:53.571849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flat = lambda list: [item for sublist in list for item in sublist]\nsub_files = lambda parent: flat([[f'{parent}/{folder}/{f}' for f in listdir(f'{parent}/{folder}')] for folder in listdir(parent)])","metadata":{"execution":{"iopub.status.busy":"2023-03-21T22:56:53.575073Z","iopub.execute_input":"2023-03-21T22:56:53.575476Z","iopub.status.idle":"2023-03-21T22:56:53.583314Z","shell.execute_reply.started":"2023-03-21T22:56:53.575437Z","shell.execute_reply":"2023-03-21T22:56:53.582311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_to_outputs(filepath, outputs_path, name):\n    with zipfile.ZipFile(outputs_path, 'a') as zipf:\n        # Add a file located at the source_path to the destination within the zip\n        # file. It will overwrite existing files if the names collide, but it\n        # will give a warning\n        source_path = filepath\n        destination = name if name else filepath.split('/')[-1]\n        zipf.write(source_path, destination)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T22:56:53.584817Z","iopub.execute_input":"2023-03-21T22:56:53.58535Z","iopub.status.idle":"2023-03-21T22:56:53.593555Z","shell.execute_reply.started":"2023-03-21T22:56:53.585306Z","shell.execute_reply":"2023-03-21T22:56:53.59275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Running","metadata":{"id":"BoYgFWcIV8vr"}},{"cell_type":"markdown","source":"Define the hyperparameters","metadata":{"id":"dKXirnbBV8vs"}},{"cell_type":"code","source":"image_height, image_width = (320, 320)\n\ntotal_variation_weight = 1e-6\nstyle_weight = 1e-6\ncontent_weight = 2.5e-8\n\nnum_iterations = 800","metadata":{"id":"bg2fzrWGV8vs","execution":{"iopub.status.busy":"2023-03-21T22:56:53.595466Z","iopub.execute_input":"2023-03-21T22:56:53.596376Z","iopub.status.idle":"2023-03-21T22:56:53.602873Z","shell.execute_reply.started":"2023-03-21T22:56:53.596335Z","shell.execute_reply":"2023-03-21T22:56:53.602029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"selecting layers for extracting the content and style of an image.","metadata":{"id":"HOmbrimXV8vs"}},{"cell_type":"code","source":"style_layers = [\n    {\n      'name': 'block1_conv2',\n      'weight': 0.15\n    },\n    {\n      'name': 'block2_conv2',\n      'weight': 0.2\n    },\n    {\n      'name': 'block3_conv1',\n      'weight': 0.2\n    },\n    {\n      'name': 'block4_conv1',\n      'weight': 0.2\n    },\n    {\n      'name': 'block5_conv1',\n      'weight': 0.2\n    }\n]\nnum_style_layers = len(style_layers)\n\ncontent_layer_name  = 'block5_conv2'","metadata":{"id":"MUsLGNk4V8vs","execution":{"iopub.status.busy":"2023-03-21T22:56:53.604383Z","iopub.execute_input":"2023-03-21T22:56:53.605113Z","iopub.status.idle":"2023-03-21T22:56:53.612272Z","shell.execute_reply.started":"2023-03-21T22:56:53.605074Z","shell.execute_reply":"2023-03-21T22:56:53.611365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Repeatedly run vanilla gradient descent steps to minimize the loss, and save the resulting image every 100 iterations.\n\nWe decay the learning rate by 0.96 every 100 steps.","metadata":{"id":"23YSFMaTJgCQ"}},{"cell_type":"code","source":"outputs = '/kaggle/working/images.zip'\n\ntry:\n    shutil.rmtree('/tmp/outputs')\nexcept:\n    pass\n    \ntry:\n    os.remove(outputs)\nexcept:\n    pass\n    \nfor index, file in enumerate(get_files_full_path('/kaggle/input/gan-getting-started/photo_jpg')[:2]):\n    optimizer = tf.keras.optimizers.SGD(\n        tf.keras.optimizers.schedules.ExponentialDecay(\n            initial_learning_rate=100.0, decay_steps=100, decay_rate=0.96\n        )\n    )\n    \n    style_image_path = random_file_full_path('/kaggle/input/gan-getting-started/monet_jpg')\n                             \n    output_directory = f'/tmp/outputs/output{(index % 450) + 1}'\n                             \n    if not os.path.exists(output_directory):\n        os.makedirs(output_directory)\n                             \n    styled_image_path = get_styled_image(file, style_image_path, output_directory)\n    add_to_outputs(styled_image_path, outputs, f'{index+1}.png')","metadata":{"id":"sGR3GQXRV8vt","outputId":"4074a0d3-1a88-424b-c718-23839ba011d0","execution":{"iopub.status.busy":"2023-03-21T22:56:53.613796Z","iopub.execute_input":"2023-03-21T22:56:53.61447Z","iopub.status.idle":"2023-03-21T22:58:22.757738Z","shell.execute_reply.started":"2023-03-21T22:56:53.614434Z","shell.execute_reply":"2023-03-21T22:58:22.7566Z"},"trusted":true},"execution_count":null,"outputs":[]}]}