{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Deep Learning Inference Notebook**\n### Authors: Simon Raviv, Adam Gavriely\n##### Deep Learning Course BIU, 2021.\n---\n<br/>\n\nIn this project we were asked to train a Cycle GAN that can transfer style from images painted by Monet to 'regular' images using only 30 monet images.\n\nThe training notebook was submitted to the [Kaggle competition](https://www.kaggle.com/c/gan-getting-started) which use MiFID metric to evaluate the model performance.  \nOur best MiFID score was : \n\nThis is the best results we achieved in this project timeframe, but there's a lot more that can be done to improve the time and performance of the model.\n","metadata":{"id":"8vs_s613kymw"}},{"cell_type":"markdown","source":"# Load required libraries","metadata":{"id":"GLeFyk9ElBQn"}},{"cell_type":"code","source":"!pip install -q tensorflow-addons\n!pip install -q gdown","metadata":{"id":"CzCPMwMbW_PG","execution":{"iopub.status.busy":"2021-08-12T16:59:13.638002Z","iopub.execute_input":"2021-08-12T16:59:13.638365Z","iopub.status.idle":"2021-08-12T16:59:44.021408Z","shell.execute_reply.started":"2021-08-12T16:59:13.638278Z","shell.execute_reply":"2021-08-12T16:59:44.020037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gdown\nimport os\nimport shutil\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport numpy as np\nfrom glob import glob\n\nfrom tensorflow_addons.layers import InstanceNormalization","metadata":{"id":"xULApCVXdglG","execution":{"iopub.status.busy":"2021-08-12T16:59:44.023782Z","iopub.execute_input":"2021-08-12T16:59:44.024305Z","iopub.status.idle":"2021-08-12T16:59:50.033453Z","shell.execute_reply.started":"2021-08-12T16:59:44.024258Z","shell.execute_reply":"2021-08-12T16:59:50.032272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global Parameters & Paths","metadata":{"id":"TuqKyGcGeke1"}},{"cell_type":"code","source":"input_image_size = (256, 256, 3)\n\nchosen_monet_file_id = \"1diLgk2a-snDTHI7_nbqjJjKEZR8kXjPQ\"\nphotos_zip_file_id = \"1HCYFybMEle_KQaL2KBKoOPjEVq2eA0dz\"\nmonet_generator_file_id = '1-HLVUS-OHe8qqM4pbymkdtlBUcULYxB_' #TODO CHANGE TO GOOD MODEL\n\nchosen_monet_filename = 'chosen_monet.tfrec'\nphotos_tfrec_folder_name = 'photo_tfrec'\nmonet_generator_filename = 'monet_generator_model.h5'","metadata":{"id":"Ec6f-Q-ieTPN","execution":{"iopub.status.busy":"2021-08-12T17:11:34.292162Z","iopub.execute_input":"2021-08-12T17:11:34.292521Z","iopub.status.idle":"2021-08-12T17:11:34.300011Z","shell.execute_reply.started":"2021-08-12T17:11:34.292473Z","shell.execute_reply":"2021-08-12T17:11:34.297960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read the data\n","metadata":{"id":"yFHaUZ4Mec7s"}},{"cell_type":"markdown","source":"## Download remote data","metadata":{"id":"nV8UFsuNWf6V"}},{"cell_type":"code","source":"def local_download_files_from_drive(id, filename, isZip=False):\n    file_url = f\"https://drive.google.com/uc?id={id}\"\n    ext = '.zip' if isZip else ''\n    gdown.download(file_url, filename+ext, quiet=True)\n    if isZip is True:\n        shutil.unpack_archive(filename+ext, extract_dir=filename)\n        os.remove(filename+ext)","metadata":{"id":"B3Iri4I-WlF1","execution":{"iopub.status.busy":"2021-08-12T17:11:35.347174Z","iopub.execute_input":"2021-08-12T17:11:35.347545Z","iopub.status.idle":"2021-08-12T17:11:35.353189Z","shell.execute_reply.started":"2021-08-12T17:11:35.347499Z","shell.execute_reply":"2021-08-12T17:11:35.352052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download all remote files to local memory:\nlocal_download_files_from_drive(monet_generator_file_id, monet_generator_filename)\nlocal_download_files_from_drive(chosen_monet_file_id, chosen_monet_filename)\nlocal_download_files_from_drive(photos_zip_file_id, photos_tfrec_folder_name, isZip=True)","metadata":{"id":"JO1n7I8sWfaF","execution":{"iopub.status.busy":"2021-08-12T17:11:35.584324Z","iopub.execute_input":"2021-08-12T17:11:35.584675Z","iopub.status.idle":"2021-08-12T17:11:44.602915Z","shell.execute_reply.started":"2021-08-12T17:11:35.584648Z","shell.execute_reply":"2021-08-12T17:11:44.601657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility functions for downloading, reading, parsing and visualizing the images","metadata":{"id":"cajrgj0leodu"}},{"cell_type":"code","source":"# Map values in the range [-1, 1]\ndef normalize_img(img):\n    img = tf.cast(img, dtype=tf.float32)\n    return (img / 127.5) - 1.0\n\n# Map values in the range [0, 255]\ndef denormalize_img(img):\n    img = tf.cast(img, dtype=tf.float32)\n    return tf.cast((img + 1.0) * 127.5, tf.uint8)\n    \ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = normalize_img(image)\n    image = tf.reshape(image, [*input_image_size])\n    return image\n\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    return image\n\ndef load_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord)\n    return dataset\n\ndef visualize_dataset(dataset, rows, cols, images_normalized=True, imsize=3):\n    plt.figure(figsize=(cols*imsize,rows*imsize))\n    for i, im in enumerate(dataset.take(rows*cols)):\n        if images_normalized is True:\n            im = denormalize_img(im)\n        plt.subplot(rows, cols, i+1)\n        plt.imshow(im)\n        plt.axis(False)\n    plt.tight_layout()\n    plt.show()","metadata":{"id":"SrkN6IrseTde","execution":{"iopub.status.busy":"2021-08-12T16:59:51.027910Z","iopub.status.idle":"2021-08-12T16:59:51.028355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read the NON-Monet photos","metadata":{"id":"hKVwICGyjFxv"}},{"cell_type":"code","source":"filenames = glob(f\"./{photos_tfrec_folder_name}/*.tfrec\")\nphoto_dataset = load_dataset(filenames)\nvisualize_dataset(photo_dataset, rows=2, cols=6, images_normalized=True)","metadata":{"id":"jxpVYpD1IF2L","outputId":"ec1fb029-8e3d-4f0e-b9d8-d3dc5b8f350b","execution":{"iopub.status.busy":"2021-08-12T16:59:51.029628Z","iopub.status.idle":"2021-08-12T16:59:51.030373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Chosen Monet Files","metadata":{"id":"b5Q4x09EfCF2"}},{"cell_type":"code","source":"chosen_monet_dataset = load_dataset(chosen_monet_filename)\nvisualize_dataset(chosen_monet_dataset, rows=5, cols=6, images_normalized=True)","metadata":{"id":"szHLR6BDfBVG","outputId":"03ad95e1-d846-44bf-cd75-d059e22f703a","execution":{"iopub.status.busy":"2021-08-12T16:59:51.031532Z","iopub.status.idle":"2021-08-12T16:59:51.032345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the model and visualize predictions","metadata":{"id":"99nCLOYBjWtW"}},{"cell_type":"markdown","source":"We must define the special reflection padding layer in order to load the model","metadata":{"id":"yB8Z1UohsMB-"}},{"cell_type":"code","source":"class ReflectionPadding2D(tf.keras.layers.Layer):\n\n    def __init__(self, padding=(1, 1), **kwargs):\n        self.padding = tuple(padding)\n        super(ReflectionPadding2D, self).__init__(**kwargs)\n\n    def call(self, input_tensor, mask=None):\n        padding_width, padding_height = self.padding\n        padding_tensor = [\n            [0, 0],\n            [padding_height, padding_height],\n            [padding_width, padding_width],\n            [0, 0],\n        ]\n        return tf.pad(input_tensor, padding_tensor, mode=\"REFLECT\")\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'padding': self.padding\n        })\n        return config","metadata":{"id":"I0p_f1-GsK7u","execution":{"iopub.status.busy":"2021-08-12T16:59:51.033728Z","iopub.status.idle":"2021-08-12T16:59:51.034548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"monet_generator = tf.keras.models.load_model('./' + monet_generator_filename, compile=False, custom_objects={'ReflectionPadding2D': ReflectionPadding2D})","metadata":{"id":"aqt7ToA8WWYu","execution":{"iopub.status.busy":"2021-08-12T16:59:51.036302Z","iopub.status.idle":"2021-08-12T16:59:51.037061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize predictions","metadata":{"id":"3JWxx93MjVR3"}},{"cell_type":"code","source":"def visualize_predictions(model, images_dateset, rows, cols, imsize=3):\n    plt.figure(figsize=(cols*imsize, rows*imsize))\n    amount_to_show = rows*cols\n    for i, img in zip(range(1, amount_to_show, 2), images_dateset.take(amount_to_show).batch(1)):\n        input_img = denormalize_img(img[0]).numpy()\n        plt.subplot(rows, cols, i)\n        plt.title(\"Input image\")\n        plt.imshow(input_img)\n\n        prediction = model(img, training=False).numpy()\n        prediction = denormalize_img(prediction[0])\n        plt.subplot(rows, cols, i+1)\n        plt.title(\"Translated image\")\n        plt.imshow(prediction)\n\n    plt.tight_layout()\n    plt.show()\n\nvisualize_predictions(monet_generator, photo_dataset, rows=3, cols=6, imsize=4)","metadata":{"id":"lZmrc6hOjUt-","outputId":"a7c03338-67c8-4960-f2cd-4ad6c9d3227f","execution":{"iopub.status.busy":"2021-08-12T16:59:51.038348Z","iopub.status.idle":"2021-08-12T16:59:51.039153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As you can see we perform pretty well on nature images such as water, skies, mountains etc. and perform not quite as well on other stuff images e.g geometric objects, animals or people.\n\nWe believe this is due to the fact that Monet chose to focus the nature as his inpiration the vast majority of his paintings and you can see from our manually selected 30 monet images that we aimed to focus on that part of his paintings to make the learning of the model more focused.\n","metadata":{"id":"NVur1pBBpcmA"}},{"cell_type":"markdown","source":"# Generate all the photos to monet images\n\nUsed for kaggle submissions","metadata":{"id":"mBbYzUPM8NXm"}},{"cell_type":"code","source":"from tqdm import tqdm\nfrom PIL import Image\n!mkdir ../images\ni = 1\nfor img in tqdm(photo_dataset.batch(1)):\n    prediction = monet_generator.predict(img)\n    prediction = denormalize_img(prediction)\n    im = Image.fromarray(prediction[0].numpy())\n    im.save(\"../images/\" + str(i) + \".jpg\")\n    i += 1\n\nshutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{"id":"sJwcszCBtk5B","execution":{"iopub.status.busy":"2021-08-12T16:59:51.040777Z","iopub.status.idle":"2021-08-12T16:59:51.041656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}