{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a id='imports'></a>\n## Imports","metadata":{}},{"cell_type":"markdown","source":"Code based on https://www.kaggle.com/code/annatmp/monet-cyclegan-mlip-2021 was primarily interested in the EDA aspect of the notebook for our own datasets.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\n\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport os, random, json, PIL, shutil, re, imageio, glob\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n    \nprint(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:41:36.619843Z","iopub.execute_input":"2022-04-25T16:41:36.620127Z","iopub.status.idle":"2022-04-25T16:41:36.631639Z","shell.execute_reply.started":"2022-04-25T16:41:36.620098Z","shell.execute_reply":"2022-04-25T16:41:36.630612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='eda'></a>\n## Exploratory Data Analysis\n","metadata":{"trusted":true}},{"cell_type":"markdown","source":"## Visualization of a few images\n\nLet's have a first look at the data we got. All the images for the challenge are of the size 256x256 and sorted by type already","metadata":{}},{"cell_type":"code","source":"BASE_PATH = '../input/gan-getting-started'\nMONET_PATH = os.path.join(BASE_PATH, 'monet_jpg')\nPHOTO_PATH = os.path.join(BASE_PATH, 'photo_jpg')\nGENERATED_PATH='../input/generatedimagesphase2'\n\n\nimport cv2\nimport math\nimport random\n\n\ndef load_images(paths):\n    images = []\n    for img in paths:\n        try:\n            img = cv2.imread(img)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        except:\n            print(\"Could not load {}\".format(img))\n        images.append(img)\n    return images\n        \n        \ndef show_folder_info(path):\n    d_image_sizes = {}\n    for image_name in os.listdir(path):\n        image = cv2.imread(os.path.join(path, image_name))\n        d_image_sizes[image.shape] = d_image_sizes.get(image.shape, 0) + 1\n        \n    for size, count in d_image_sizes.items():\n        print(f'shape: {size}\\tcount: {count}')\n        \ndef visualize_images(images, title=None):\n    plt.figure(figsize=(16,16))\n    \n    w = int(len(images) ** .5)\n    h = math.ceil(len(images) / w)\n    \n    if title:\n        plt.suptitle(title)\n        \n    for idx, image in enumerate(images):\n        \n        plt.subplot(h, w, idx + 1)\n        plt.imshow(image)\n        plt.axis('off')\n    \n    \n    \n    plt.show()\n        ","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:41:36.633734Z","iopub.execute_input":"2022-04-25T16:41:36.634099Z","iopub.status.idle":"2022-04-25T16:41:36.647641Z","shell.execute_reply.started":"2022-04-25T16:41:36.634062Z","shell.execute_reply":"2022-04-25T16:41:36.646818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MONET_IMAGES = [os.path.join(MONET_PATH, file) for file in os.listdir(MONET_PATH)]\nmonet_images = load_images(MONET_IMAGES)\n\nPHOTO_IMAGES = [os.path.join(PHOTO_PATH, file) for file in os.listdir(PHOTO_PATH)]\nphoto_images = load_images(PHOTO_IMAGES)\n\nGENERATED_PATH='../input/generatedimagesphase2'\nGENERATED_IMAGES=[os.path.join(GENERATED_PATH, file) for file in os.listdir(GENERATED_PATH)]\ngenerated_images=load_images(GENERATED_IMAGES)\n\nBERKELEY_PATH='../input/berkeley-monet'\nBERKELEY_IMAGES=[os.path.join(BERKELEY_PATH, file) for file in os.listdir(BERKELEY_PATH)]\nberkeley_images=load_images(BERKELEY_IMAGES)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:41:36.681706Z","iopub.execute_input":"2022-04-25T16:41:36.682296Z","iopub.status.idle":"2022-04-25T16:42:02.483585Z","shell.execute_reply.started":"2022-04-25T16:41:36.682258Z","shell.execute_reply":"2022-04-25T16:42:02.482582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **BERKELEY DATASET**","metadata":{}},{"cell_type":"code","source":"visualize_images(random.sample(berkeley_images,15), \"Samples of the Berkeley Dataset\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:02.485456Z","iopub.execute_input":"2022-04-25T16:42:02.485715Z","iopub.status.idle":"2022-04-25T16:42:03.653102Z","shell.execute_reply.started":"2022-04-25T16:42:02.485688Z","shell.execute_reply":"2022-04-25T16:42:03.651666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Monet Style Images","metadata":{}},{"cell_type":"code","source":"#visualize_images(random.sample(monet_images,15), \"Samples of the Monet Dataset\")\nvisualize_images(random.sample(monet_images,15),'One Image')","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:03.654547Z","iopub.execute_input":"2022-04-25T16:42:03.654871Z","iopub.status.idle":"2022-04-25T16:42:05.084655Z","shell.execute_reply.started":"2022-04-25T16:42:03.654833Z","shell.execute_reply":"2022-04-25T16:42:05.083532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Photos","metadata":{}},{"cell_type":"code","source":"visualize_images(random.sample(photo_images,15), \"Samples of the Photo Dataset\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:05.086927Z","iopub.execute_input":"2022-04-25T16:42:05.087234Z","iopub.status.idle":"2022-04-25T16:42:06.261092Z","shell.execute_reply.started":"2022-04-25T16:42:05.087199Z","shell.execute_reply":"2022-04-25T16:42:06.260292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GENERATED IMAGES","metadata":{}},{"cell_type":"code","source":"visualize_images(random.sample(generated_images,15), \"Samples of the Generated Dataset\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:06.26237Z","iopub.execute_input":"2022-04-25T16:42:06.262708Z","iopub.status.idle":"2022-04-25T16:42:07.673229Z","shell.execute_reply.started":"2022-04-25T16:42:06.262674Z","shell.execute_reply":"2022-04-25T16:42:07.672297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Visual Inspection*\n\nComparing the Monet images and the photos we can obviously see which are paintings and which are photos. The photos have way smoother transitions, the monet images \"jump\" in colours. Monet was a impressionist painter, an art style which is characterized by a small but still visible brush strokes. \nSo our final images should also have this characteristic but showing the content of the content images.  \n\nFurther, the colours of photos are more natural than the colours of the mone style images. This is not too surprising, but we want to keep this in mind when judging the final output of our GAN","metadata":{}},{"cell_type":"markdown","source":"### Data Stats:","metadata":{}},{"cell_type":"code","source":"print(\"Monet Picture Overview\")\nshow_folder_info(MONET_PATH)\n\nprint(\"Content Picture Overview\")\nshow_folder_info(PHOTO_PATH)\n\nprint(\"Generated Picture Overview\")\nshow_folder_info(GENERATED_PATH)\n\nprint(\"Berkeley Picture Overview\")\nshow_folder_info(BERKELEY_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:07.674804Z","iopub.execute_input":"2022-04-25T16:42:07.675044Z","iopub.status.idle":"2022-04-25T16:42:31.551877Z","shell.execute_reply.started":"2022-04-25T16:42:07.675017Z","shell.execute_reply":"2022-04-25T16:42:31.550709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='data_loading'></a>\n## Data Loading","metadata":{}},{"cell_type":"code","source":"GCS_PATH_FULL_DATA = KaggleDatasets().get_gcs_path(\"gan-getting-started\")\nGCS_PATH_CLEAN_DATA = KaggleDatasets().get_gcs_path(\"cleaned-monet-data\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:31.555131Z","iopub.execute_input":"2022-04-25T16:42:31.555959Z","iopub.status.idle":"2022-04-25T16:42:32.367699Z","shell.execute_reply.started":"2022-04-25T16:42:31.555913Z","shell.execute_reply":"2022-04-25T16:42:32.366779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We load a cleaned version of the monet dataset. The original had two round images and some really dark and not quite \"Monet-like\" images. The following images were excluded:\n","metadata":{}},{"cell_type":"markdown","source":"The images for this challenges are already cleaned. All are of size 256x256 and have three channels (RGB)","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = [256, 256]\nHEIGHT_IMG = 256\nWIDTH_IMG = 256\nN_CHANNELS = 3 ","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:32.369033Z","iopub.execute_input":"2022-04-25T16:42:32.369308Z","iopub.status.idle":"2022-04-25T16:42:32.374049Z","shell.execute_reply.started":"2022-04-25T16:42:32.369278Z","shell.execute_reply":"2022-04-25T16:42:32.373029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below we define two functions. `read_tfrecord(example)` retrieves the images data from the tf record, `decode_image` transforms it into a `tf.image`","metadata":{}},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example):\n    tfrecord_format = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:32.375437Z","iopub.execute_input":"2022-04-25T16:42:32.375699Z","iopub.status.idle":"2022-04-25T16:42:32.38759Z","shell.execute_reply.started":"2022-04-25T16:42:32.37567Z","shell.execute_reply":"2022-04-25T16:42:32.386832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lastly, we create a function that loads the dataset and combines the upper functions","metadata":{}},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:32.390831Z","iopub.execute_input":"2022-04-25T16:42:32.391074Z","iopub.status.idle":"2022-04-25T16:42:32.402489Z","shell.execute_reply.started":"2022-04-25T16:42:32.391047Z","shell.execute_reply":"2022-04-25T16:42:32.40163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lastly, we need a function that combines all our data loading functions","metadata":{}},{"cell_type":"markdown","source":"Compute the average activation for the monet images. The inception model is just used for feature extraction which means we have to compute the activation map for the original model only once","metadata":{}},{"cell_type":"markdown","source":"<a id='eda2'></a>\n## Exhaustive EDA","metadata":{}},{"cell_type":"markdown","source":"### Histogram analysis\n\nOpenCV uses the BGR channel order, thus the first histogram of an image is its blue value histogram, the second gree and the third red","metadata":{}},{"cell_type":"code","source":"monet_histograms = [[cv2.calcHist([image],[i],None,[256],[0,256]) for i in range(3)] for image in monet_images]\nphoto_histograms = [[cv2.calcHist([image],[i],None,[256],[0,256]) for i in range(3)] for image in photo_images]\ngenerated_histograms=[[cv2.calcHist([image],[i],None,[256],[0,256]) for i in range(3)] for image in generated_images]\nberkeley_histograms=[[cv2.calcHist([image],[i],None,[256],[0,256]) for i in range(3)] for image in berkeley_images]","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:32.403599Z","iopub.execute_input":"2022-04-25T16:42:32.403907Z","iopub.status.idle":"2022-04-25T16:42:34.417354Z","shell.execute_reply.started":"2022-04-25T16:42:32.403869Z","shell.execute_reply":"2022-04-25T16:42:34.4164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_histograms(img, title):\n    plt.figure(figsize=(16,16))\n    \n    plt.title(title)\n    color = ('b','g','r')\n    \n    \n    w = 4\n    h = int(len(img)/2)\n    \n    idxs = [i for i in range(len(img)*2)[::2]]\n    \n    for idx, image in zip(idxs,img):\n        \n        plt.subplot(h, w, idx + 1)\n        plt.imshow(image)\n        plt.axis('off')\n        \n        plt.subplot(h, w, idx + 2)\n        for i,col in enumerate(color):   \n            histr = cv2.calcHist([image],[i],None,[256],[0,256])\n            plt.plot(histr,color = col)\n            plt.xlim([0,256])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:34.418902Z","iopub.execute_input":"2022-04-25T16:42:34.41925Z","iopub.status.idle":"2022-04-25T16:42:34.430432Z","shell.execute_reply.started":"2022-04-25T16:42:34.419208Z","shell.execute_reply":"2022-04-25T16:42:34.429432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms(random.sample(monet_images,8), \"Histogram for Monet Image\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:34.43162Z","iopub.execute_input":"2022-04-25T16:42:34.431843Z","iopub.status.idle":"2022-04-25T16:42:36.258253Z","shell.execute_reply.started":"2022-04-25T16:42:34.431817Z","shell.execute_reply":"2022-04-25T16:42:36.257166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms(random.sample(photo_images,8), \"Histogram for Photo\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:36.25962Z","iopub.execute_input":"2022-04-25T16:42:36.25984Z","iopub.status.idle":"2022-04-25T16:42:38.360844Z","shell.execute_reply.started":"2022-04-25T16:42:36.259814Z","shell.execute_reply":"2022-04-25T16:42:38.359863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms(random.sample(generated_images,8),\"Histogram for Generated Images\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:38.36233Z","iopub.execute_input":"2022-04-25T16:42:38.362591Z","iopub.status.idle":"2022-04-25T16:42:40.161277Z","shell.execute_reply.started":"2022-04-25T16:42:38.362562Z","shell.execute_reply":"2022-04-25T16:42:40.160328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"berkeley","metadata":{}},{"cell_type":"code","source":"plot_histograms(random.sample(berkeley_images,8),\"Histogram for Berkeley Images\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:40.162956Z","iopub.execute_input":"2022-04-25T16:42:40.163254Z","iopub.status.idle":"2022-04-25T16:42:41.964271Z","shell.execute_reply.started":"2022-04-25T16:42:40.163219Z","shell.execute_reply":"2022-04-25T16:42:41.963469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The histograms indicate that \n\n(1) The monet images seem to use on average more colours for every channel. We come to this conclusion, since the mean pixel intensity for the Monet images is higher than for the photos. However, given the amount of photo files, there are several outliers in both directions. \n\n(2) The range of intensity values used for the Monet images is smaller than for the photos. I.e. the Monet images have more \"unused\" intensity values. This may be explained by the fact that photo transitions are way more smooth than the transitions of colours in expressionistic paintings. \n\nHowever, these observations are not too surprising considering that we have paintings and images.","metadata":{}},{"cell_type":"markdown","source":"### Analysis of the image intensity value distributions","metadata":{}},{"cell_type":"code","source":"color = ('b','g','r')\nmonet_means = {c: [np.mean(x[:,:,i]) for x in monet_images] for i,c in enumerate(color)}\nphoto_means = {c: [np.mean(x[:,:,i]) for x in photo_images] for i,c in enumerate(color)}\ngenerated_means={c: [np.mean(x[:,:,i]) for x in generated_images] for i,c in enumerate(color)}\nberkeley_means={c: [np.mean(x[:,:,i]) for x in berkeley_images] for i,c in enumerate(color)}\n\n\nmonet_std = {c: [np.std(x[:,:,i]) for x in monet_images] for i,c in enumerate(color)}\nphoto_std = {c: [np.std(x[:,:,i]) for x in photo_images] for i,c in enumerate(color)}\ngenerated_std={c: [np.std(x[:,:,i]) for x in generated_images] for i,c in enumerate(color)}\nberkeley_std={c: [np.std(x[:,:,i]) for x in berkeley_images] for i,c in enumerate(color)}","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:41.965595Z","iopub.execute_input":"2022-04-25T16:42:41.966374Z","iopub.status.idle":"2022-04-25T16:42:54.891246Z","shell.execute_reply.started":"2022-04-25T16:42:41.96633Z","shell.execute_reply":"2022-04-25T16:42:54.890376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def boxplots_for_comparison(monet, photo, generated,berkeley, title):\n    for i,c in enumerate(color):\n        fig, ax = plt.subplots()\n        ax.set_title('{} Channel: {}'.format(c.upper(), title))\n        ax.boxplot([monet[c], photo[c], generated[c],berkeley[c]])\n        ax.set_xticklabels([\"Monet\",\"Photo\",\"Generated\",\"Berkeley\"])\n        fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:54.892562Z","iopub.execute_input":"2022-04-25T16:42:54.892873Z","iopub.status.idle":"2022-04-25T16:42:54.900577Z","shell.execute_reply.started":"2022-04-25T16:42:54.892832Z","shell.execute_reply":"2022-04-25T16:42:54.899618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxplots_for_comparison(monet_means, photo_means, generated_means,berkeley_means, \"Pixel Value Mean comparison - Monet and Photo and Generated and Berkeley\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:54.901726Z","iopub.execute_input":"2022-04-25T16:42:54.902028Z","iopub.status.idle":"2022-04-25T16:42:55.761017Z","shell.execute_reply.started":"2022-04-25T16:42:54.901987Z","shell.execute_reply":"2022-04-25T16:42:55.759534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxplots_for_comparison(monet_std, photo_std,generated_std,berkeley_std, \"Pixel Value standard deviation comparison - Monet and Photo and Generated and Berkeley\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:55.762374Z","iopub.execute_input":"2022-04-25T16:42:55.762876Z","iopub.status.idle":"2022-04-25T16:42:56.342676Z","shell.execute_reply.started":"2022-04-25T16:42:55.762843Z","shell.execute_reply":"2022-04-25T16:42:56.341768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### \"Frequency\" of intensity values\n\nThe boxplots only tell us the mean and the standard deviation of the intensity values. Now it would be interesting to see if both Monet and the photos make use of the full range of intensity values. That is, are there any empty bins in images? Our assumption is that Monet images might not make use of the full range of intensity values (based on the histograms)","metadata":{}},{"cell_type":"code","source":"zero_values_monet = {c: [np.count_nonzero(hist[i].ravel()==0) for hist in monet_histograms] for i,c in enumerate(color)}\nzero_values_photo = {c: [np.count_nonzero(hist[i].ravel()==0) for hist in photo_histograms] for i,c in enumerate(color)}","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:56.343998Z","iopub.execute_input":"2022-04-25T16:42:56.34422Z","iopub.status.idle":"2022-04-25T16:42:56.442689Z","shell.execute_reply.started":"2022-04-25T16:42:56.344191Z","shell.execute_reply":"2022-04-25T16:42:56.441752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxplots_for_comparison(zero_values_monet,zero_values_photo, \"Number of not used intensity values Monet and Photo\" )","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:42:56.444129Z","iopub.execute_input":"2022-04-25T16:42:56.444445Z","iopub.status.idle":"2022-04-25T16:42:56.47301Z","shell.execute_reply.started":"2022-04-25T16:42:56.444405Z","shell.execute_reply":"2022-04-25T16:42:56.471885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}