{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Importing libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt, zipfile\nimport os\nimport glob\nimport math\nimport random\nimport time\nimport datetime\nimport shutil\nimport imageio\nfrom tqdm import tqdm, tqdm_notebook\n\nfrom dataclasses import dataclass\nfrom pathlib import Path\nimport warnings\nfrom scipy import linalg\n\nimport xml.etree.ElementTree as ET \n\nimport cv2\nfrom PIL import Image\n\nimport tensorflow as tf\nfrom tensorflow.keras.initializers import RandomNormal\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense, BatchNormalization, LeakyReLU, Reshape,\\\nConv2DTranspose, Conv2D, Flatten, Dropout, Embedding, ReLU\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.contrib.eager.python import tfe\n\n#from IPython import display\n\n# libraries for SpectralNorm\nfrom tensorflow.keras import backend as K\nfrom keras.engine import *\nfrom keras.legacy import interfaces\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras import constraints\nfrom keras.utils.generic_utils import func_dump\nfrom keras.utils.generic_utils import func_load\nfrom keras.utils.generic_utils import deserialize_keras_object\nfrom keras.utils.generic_utils import has_arg\nfrom keras.utils import conv_utils\n\nprint(os.listdir(\"../input\"))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-09T22:59:33.937115Z","iopub.execute_input":"2022-02-09T22:59:33.93741Z","iopub.status.idle":"2022-02-09T22:59:33.950279Z","shell.execute_reply.started":"2022-02-09T22:59:33.937361Z","shell.execute_reply":"2022-02-09T22:59:33.949194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.enable_eager_execution()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T23:08:04.257511Z","iopub.execute_input":"2022-02-09T23:08:04.257791Z","iopub.status.idle":"2022-02-09T23:08:04.261746Z","shell.execute_reply.started":"2022-02-09T23:08:04.257743Z","shell.execute_reply":"2022-02-09T23:08:04.260675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Setting input variables","metadata":{}},{"cell_type":"code","source":"image_width = 64\nimage_height = 64\nimage_channels = 3\nimage_sample_size = 10000\nimage_output_dir = '../output_images/'\nimage_input_dir = '../input/generative-dog-images/all-dogs/all-dogs/'\nimage_ann_dir = \"../input/generative-dog-images/annotation/Annotation/\"\nOUT_DIR = Path('../output_images/')\nMODEL_PATH = '../input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'\nTRAIN_DIR = Path('../input/generative-dog-images/all-dogs/all-dogs')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T23:08:54.159652Z","iopub.execute_input":"2022-02-09T23:08:54.159978Z","iopub.status.idle":"2022-02-09T23:08:54.165616Z","shell.execute_reply.started":"2022-02-09T23:08:54.159921Z","shell.execute_reply":"2022-02-09T23:08:54.164179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image preprocessing and EDA","metadata":{}},{"cell_type":"markdown","source":"### Creating the image features","metadata":{}},{"cell_type":"code","source":"dog_breed_dict = {}\nfor annotation in os.listdir(image_ann_dir):\n    annotations = annotation.split('-')\n    dog_breed_dict[annotations[0]] = annotations[1]","metadata":{"execution":{"iopub.status.busy":"2022-02-09T23:09:00.571234Z","iopub.execute_input":"2022-02-09T23:09:00.571541Z","iopub.status.idle":"2022-02-09T23:09:00.59777Z","shell.execute_reply.started":"2022-02-09T23:09:00.571484Z","shell.execute_reply":"2022-02-09T23:09:00.596851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I'm going to use <b>OpenCV</b> for a quick function to read and transform the images to <b>RGB</b>.","metadata":{}},{"cell_type":"code","source":"def read_image(src):\n    img = cv2.imread(src)\n    if img is None:\n        raise FileNotFoundError\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:31.350776Z","iopub.status.idle":"2022-02-09T22:58:31.351709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Crop the images and apply scaling","metadata":{}},{"cell_type":"code","source":"def load_cropped_images(dog_breed_dict=dog_breed_dict, image_ann_dir=image_ann_dir, sample_size=25000, \n                        image_width=image_width, image_height=image_height, image_channels=image_channels):\n    curIdx = 0\n    breeds = []\n    dog_images_np = np.zeros((sample_size,image_width,image_height,image_channels))\n    for breed_folder in os.listdir(image_ann_dir):\n        for dog_ann in tqdm(os.listdir(image_ann_dir + breed_folder)):\n            try:\n                img = read_image(os.path.join(image_input_dir, dog_ann + '.jpg'))\n            except FileNotFoundError:\n                continue\n                \n            tree = ET.parse(os.path.join(image_ann_dir + breed_folder, dog_ann))\n            root = tree.getroot()\n            \n            size = root.find('size')\n            width = int(size.find('width').text)\n            height = int(size.find('height').text)\n            objects = root.findall('object')\n            for o in objects:\n                bndbox = o.find('bndbox') \n                \n                xmin = int(bndbox.find('xmin').text)\n                ymin = int(bndbox.find('ymin').text)\n                xmax = int(bndbox.find('xmax').text)\n                ymax = int(bndbox.find('ymax').text)\n                \n                xmin = max(0, xmin - 4)        # 4 : margin\n                xmax = min(width, xmax + 4)\n                ymin = max(0, ymin - 4)\n                ymax = min(height, ymax + 4)\n\n                w = np.min((xmax - xmin, ymax - ymin))\n                w = min(w, width, height)                     # available w\n\n                if w > xmax - xmin:\n                    xmin = min(max(0, xmin - int((w - (xmax - xmin))/2)), width - w)\n                    xmax = xmin + w\n                if w > ymax - ymin:\n                    ymin = min(max(0, ymin - int((w - (ymax - ymin))/2)), height - w)\n                    ymax = ymin + w\n                \n                img_cropped = img[ymin:ymin+w, xmin:xmin+w, :]      # [h,w,c]\n                # Interpolation method\n                if xmax - xmin > image_width:\n                    interpolation = cv2.INTER_AREA          # shrink\n                else:\n                    interpolation = cv2.INTER_CUBIC         # expansion\n                    \n                img_cropped = cv2.resize(img_cropped, (image_width, image_height), \n                                         interpolation=interpolation)  # resize\n                    \n                dog_images_np[curIdx,:,:,:] = np.asarray(img_cropped)\n                dog_breed_name = dog_breed_dict[dog_ann.split('_')[0]]\n                breeds.append(dog_breed_name)\n                curIdx += 1\n                \n    return dog_images_np, breeds","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:31.353206Z","iopub.status.idle":"2022-02-09T22:58:31.35401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*The features take about 2-3 mins to load. *\n\n&nbsp;&nbsp;&nbsp;By trial, we can figure out that the result images with single dogs only are <b>22125</b> and thus we can specify the exact size of our numpy array. There are <b>120</b> different dog breeds.","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\ndog_images_np, breeds = load_cropped_images(sample_size=22125)\nest_time = round(time.time() - start_time)\nprint(\"Feature loading time: {}.\".format(str(datetime.timedelta(seconds=est_time))))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:31.355271Z","iopub.status.idle":"2022-02-09T22:58:31.356077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Loaded features shape: ', dog_images_np.shape)\nprint('Loaded labels: ', len(breeds))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:31.357409Z","iopub.status.idle":"2022-02-09T22:58:31.358218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;Now that we have the features and labels, we can plot them in a square grid to see what the crops look like and to make sure that they are labeled correctly.","metadata":{}},{"cell_type":"code","source":"def plot_features(features, labels, image_width=image_width, image_height=image_height, \n                image_channels=image_channels,\n                examples=25, disp_labels=True): \n  \n    if not math.sqrt(examples).is_integer():\n        print('Please select a valid number of examples.')\n        return\n    \n    imgs = []\n    classes = []\n    for i in range(examples):\n        rnd_idx = np.random.randint(0, len(labels))\n        imgs.append(features[rnd_idx, :, :, :])\n        classes.append(labels[rnd_idx])\n    \n    \n    fig, axes = plt.subplots(round(math.sqrt(examples)), round(math.sqrt(examples)),figsize=(15,15),\n    subplot_kw = {'xticks':[], 'yticks':[]},\n    gridspec_kw = dict(hspace=0.3, wspace=0.01))\n    \n    for i, ax in enumerate(axes.flat):\n        if disp_labels == True:\n            ax.title.set_text(classes[i])\n        ax.imshow(imgs[i])","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:31.857708Z","iopub.execute_input":"2022-02-09T22:58:31.858189Z","iopub.status.idle":"2022-02-09T22:58:31.868405Z","shell.execute_reply.started":"2022-02-09T22:58:31.858103Z","shell.execute_reply":"2022-02-09T22:58:31.867288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that we need to normalize the pixel values to make sure that the dogs are plotted correctly.","metadata":{}},{"cell_type":"code","source":"print('Plotting cropped images by their specified coordinates..')\nplot_features(dog_images_np / 255., breeds, examples=25, disp_labels=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:33.172559Z","iopub.execute_input":"2022-02-09T22:58:33.173029Z","iopub.status.idle":"2022-02-09T22:58:33.198577Z","shell.execute_reply.started":"2022-02-09T22:58:33.172811Z","shell.execute_reply":"2022-02-09T22:58:33.197434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The crops definitely are not perfect, but they do seem to capture only one dog in each image.","metadata":{}},{"cell_type":"markdown","source":"### Normalize the pixel values of the images","metadata":{}},{"cell_type":"code","source":"dog_images_np = (dog_images_np - 127.5) / 127.5  # normalize the pixel range to [-1, 1] ((image - 127.5) / 127.5) or [0, 1] (image / 255.) alternatively","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:34.667943Z","iopub.execute_input":"2022-02-09T22:58:34.668235Z","iopub.status.idle":"2022-02-09T22:58:34.805007Z","shell.execute_reply.started":"2022-02-09T22:58:34.668189Z","shell.execute_reply":"2022-02-09T22:58:34.80386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is what the images would look like in the input layer of <b>D</b>.","metadata":{}},{"cell_type":"code","source":"print('Plotting cropped images by their specified coordinates..')\nplot_features(dog_images_np, breeds, examples=25, disp_labels=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:34.806321Z","iopub.status.idle":"2022-02-09T22:58:34.806944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If we select a random image and print out the <b>max and min</b> we can see that the values are correct.","metadata":{}},{"cell_type":"code","source":"print(np.max(dog_images_np[3,:,:,:]), np.min(dog_images_np[3,:,:,:]))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:35.264902Z","iopub.execute_input":"2022-02-09T22:58:35.265222Z","iopub.status.idle":"2022-02-09T22:58:35.290963Z","shell.execute_reply.started":"2022-02-09T22:58:35.265169Z","shell.execute_reply":"2022-02-09T22:58:35.290024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deprocessing back to the original values","metadata":{}},{"cell_type":"markdown","source":"What if we want to return them back to normal? Easy, we just apply the reverse operations and plot them again.","metadata":{}},{"cell_type":"code","source":"plot_features((dog_images_np * 127.5 + 127.5) / 255., breeds, examples=25, disp_labels=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:36.050192Z","iopub.execute_input":"2022-02-09T22:58:36.05053Z","iopub.status.idle":"2022-02-09T22:58:36.089055Z","shell.execute_reply.started":"2022-02-09T22:58:36.050487Z","shell.execute_reply":"2022-02-09T22:58:36.086808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating the image dataset","metadata":{}},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;Now let's use our numpy array of features to construct a <b>Tensorflow</b> dataset object. First, we can convert the data types to <b>float32</b>, which always helps to preserve some memory.","metadata":{}},{"cell_type":"code","source":"print(\"Dog features shape:\", dog_images_np.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:36.501851Z","iopub.execute_input":"2022-02-09T22:58:36.502147Z","iopub.status.idle":"2022-02-09T22:58:36.527044Z","shell.execute_reply.started":"2022-02-09T22:58:36.502094Z","shell.execute_reply":"2022-02-09T22:58:36.52589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dog_features_tf = tf.cast(dog_images_np, 'float32')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:36.62942Z","iopub.execute_input":"2022-02-09T22:58:36.629649Z","iopub.status.idle":"2022-02-09T22:58:36.649532Z","shell.execute_reply.started":"2022-02-09T22:58:36.629605Z","shell.execute_reply":"2022-02-09T22:58:36.648496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Augmentation","metadata":{}},{"cell_type":"code","source":"def flip(x: tf.Tensor) -> (tf.Tensor):\n    x = tf.image.random_flip_left_right(x)\n    return x\n\ndef zoom(x: tf.Tensor) -> (tf.Tensor):\n    # Generate 20 crop settings, ranging from a 1% to 20% crop.\n    scales = list(np.arange(0.7, 1.0, 0.01))\n    boxes = np.zeros((len(scales), 4))\n\n    for i, scale in enumerate(scales):\n        x1 = y1 = 0.5 - (0.5 * scale)\n        x2 = y2 = 0.5 + (0.5 * scale)\n        boxes[i] = [x1, y1, x2, y2]\n\n    def random_crop(img):\n        # Create different crops for an image\n        crops = tf.image.crop_and_resize([img], boxes=boxes, box_ind=np.zeros(len(scales)), crop_size=(64, 64))\n        # Return a random crop\n        return crops[tf.random_uniform(shape=[], minval=0, maxval=len(scales), dtype=tf.int32)]\n\n\n    choice = tf.random_uniform(shape=[], minval=0., maxval=1., dtype=tf.float32)\n\n    # Only apply cropping 50% of the time\n    return (tf.cond(choice < 0.5, lambda: x, lambda: random_crop(x)))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:36.976974Z","iopub.execute_input":"2022-02-09T22:58:36.977273Z","iopub.status.idle":"2022-02-09T22:58:36.990236Z","shell.execute_reply.started":"2022-02-09T22:58:36.977225Z","shell.execute_reply":"2022-02-09T22:58:36.989287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Set model hyperparameters","metadata":{}},{"cell_type":"code","source":"sample_size = 22125\nbatch_size = 128\nweight_init_std = 0.02\nweight_init_mean = 0.0\nleaky_relu_slope = 0.2\ndownsize_factor = 2\ndropout_rate = 0.5\nscale_factor = 4 ** downsize_factor\nlr_initial_d = tfe.Variable(0.0002)\nlr_initial_g = tfe.Variable(0.0002)\nlr_decay_steps = 1000\nnoise_dim = 128","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:37.364866Z","iopub.execute_input":"2022-02-09T22:58:37.36516Z","iopub.status.idle":"2022-02-09T22:58:39.542283Z","shell.execute_reply.started":"2022-02-09T22:58:37.365112Z","shell.execute_reply":"2022-02-09T22:58:39.541542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create tensorflow-type dataset","metadata":{}},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;Now we can use <b>Tensorflow</b> to create the dataset by shuffling it, applying some augmentation, and finally separating it into batches of the specified <b>Batch size</b>.","metadata":{}},{"cell_type":"code","source":"dog_features_data = tf.data.Dataset.from_tensor_slices(dog_features_tf).shuffle(sample_size).map(flip).batch(batch_size, drop_remainder=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:39.543986Z","iopub.execute_input":"2022-02-09T22:58:39.54427Z","iopub.status.idle":"2022-02-09T22:58:39.569925Z","shell.execute_reply.started":"2022-02-09T22:58:39.544225Z","shell.execute_reply":"2022-02-09T22:58:39.568764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dog_features_data)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:39.57095Z","iopub.status.idle":"2022-02-09T22:58:39.571527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalization techniques","metadata":{}},{"cell_type":"markdown","source":"### Weight Initialization","metadata":{}},{"cell_type":"code","source":"weight_initializer = tf.keras.initializers.TruncatedNormal(stddev=weight_init_std, mean=weight_init_mean,\n                                                          seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:39.572627Z","iopub.status.idle":"2022-02-09T22:58:39.573213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Spectral Normalization","metadata":{}},{"cell_type":"code","source":"class DenseSN(Dense):\n    def build(self, input_shape):\n        assert len(input_shape) >= 2\n        input_dim = input_shape[-1]\n        self.kernel = self.add_weight(shape=(input_dim, self.units),\n                                      initializer=self.kernel_initializer,\n                                      name='kernel',\n                                      regularizer=self.kernel_regularizer,\n                                      constraint=self.kernel_constraint)\n        if self.use_bias:\n            self.bias = self.add_weight(shape=(self.units,),\n                                        initializer=self.bias_initializer,\n                                        name='bias',\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n        else:\n            self.bias = None\n        self.u = self.add_weight(shape=tuple([1, self.kernel.shape.as_list()[-1]]),\n                                 initializer=initializers.RandomNormal(0, 1),\n                                 name='sn',\n                                 trainable=False)\n        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})\n        self.built = True\n        \n    def call(self, inputs, training=None):\n        def _l2normalize(v, eps=1e-12):\n            return v / (K.sum(v ** 2) ** 0.5 + eps)\n        def power_iteration(W, u):\n            _u = u\n            _v = _l2normalize(K.dot(_u, K.transpose(W)))\n            _u = _l2normalize(K.dot(_v, W))\n            return _u, _v\n        W_shape = self.kernel.shape.as_list()\n        #Flatten the Tensor\n        W_reshaped = K.reshape(self.kernel, [-1, W_shape[-1]])\n        _u, _v = power_iteration(W_reshaped, self.u)\n        #Calculate Sigma\n        sigma=K.dot(_v, W_reshaped)\n        sigma=K.dot(sigma, K.transpose(_u))\n        #normalize it\n        W_bar = W_reshaped / sigma\n        #reshape weight tensor\n        if training in {0, False}:\n            W_bar = K.reshape(W_bar, W_shape)\n        else:\n            with tf.control_dependencies([self.u.assign(_u)]):\n                 W_bar = K.reshape(W_bar, W_shape)  \n        output = K.dot(inputs, W_bar)\n        if self.use_bias:\n            output = K.bias_add(output, self.bias, data_format='channels_last')\n        if self.activation is not None:\n            output = self.activation(output)\n        return output \n    \nclass ConvSN2D(Conv2D):\n\n    def build(self, input_shape):\n        if self.data_format == 'channels_first':\n            channel_axis = 1\n        else:\n            channel_axis = -1\n        if input_shape[channel_axis] is None:\n            raise ValueError('The channel dimension of the inputs '\n                             'should be defined. Found `None`.')\n        input_dim = input_shape[channel_axis]\n        kernel_shape = self.kernel_size + (input_dim, self.filters)\n\n        self.kernel = self.add_weight(shape=kernel_shape,\n                                      initializer=self.kernel_initializer,\n                                      name='kernel',\n                                      regularizer=self.kernel_regularizer,\n                                      constraint=self.kernel_constraint)\n\n        if self.use_bias:\n            self.bias = self.add_weight(shape=(self.filters,),\n                                        initializer=self.bias_initializer,\n                                        name='bias',\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n        else:\n            self.bias = None\n            \n        self.u = self.add_weight(shape=tuple([1, self.kernel.shape.as_list()[-1]]),\n                         initializer=initializers.RandomNormal(0, 1),\n                         name='sn',\n                         trainable=False)\n        \n        # Set input spec.\n        self.input_spec = InputSpec(ndim=self.rank + 2,\n                                    axes={channel_axis: input_dim})\n        self.built = True\n    def call(self, inputs, training=None):\n        def _l2normalize(v, eps=1e-12):\n            return v / (K.sum(v ** 2) ** 0.5 + eps)\n        def power_iteration(W, u):\n            #Accroding the paper, we only need to do power iteration one time.\n            _u = u\n            _v = _l2normalize(K.dot(_u, K.transpose(W)))\n            _u = _l2normalize(K.dot(_v, W))\n            return _u, _v\n        #Spectral Normalization\n        W_shape = self.kernel.shape.as_list()\n        #Flatten the Tensor\n        W_reshaped = K.reshape(self.kernel, [-1, W_shape[-1]])\n        _u, _v = power_iteration(W_reshaped, self.u)\n        #Calculate Sigma\n        sigma=K.dot(_v, W_reshaped)\n        sigma=K.dot(sigma, K.transpose(_u))\n        #normalize it\n        W_bar = W_reshaped / sigma\n        #reshape weight tensor\n        if training in {0, False}:\n            W_bar = K.reshape(W_bar, W_shape)\n        else:\n            with tf.control_dependencies([self.u.assign(_u)]):\n                W_bar = K.reshape(W_bar, W_shape)\n                \n        outputs = K.conv2d(\n                inputs,\n                W_bar,\n                strides=self.strides,\n                padding=self.padding,\n                data_format=self.data_format,\n                dilation_rate=self.dilation_rate)\n        if self.use_bias:\n            outputs = K.bias_add(\n                outputs,\n                self.bias,\n                data_format=self.data_format)\n        if self.activation is not None:\n            return self.activation(outputs)\n        return outputs\n\nclass ConvSN2DTranspose(Conv2DTranspose):\n\n    def build(self, input_shape):\n        if len(input_shape) != 4:\n            raise ValueError('Inputs should have rank ' +\n                             str(4) +\n                             '; Received input shape:', str(input_shape))\n        if self.data_format == 'channels_first':\n            channel_axis = 1\n        else:\n            channel_axis = -1\n        if input_shape[channel_axis] is None:\n            raise ValueError('The channel dimension of the inputs '\n                             'should be defined. Found `None`.')\n        input_dim = input_shape[channel_axis]\n        kernel_shape = self.kernel_size + (self.filters, input_dim)\n\n        self.kernel = self.add_weight(shape=kernel_shape,\n                                      initializer=self.kernel_initializer,\n                                      name='kernel',\n                                      regularizer=self.kernel_regularizer,\n                                      constraint=self.kernel_constraint)\n        if self.use_bias:\n            self.bias = self.add_weight(shape=(self.filters,),\n                                        initializer=self.bias_initializer,\n                                        name='bias',\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n        else:\n            self.bias = None\n            \n        self.u = self.add_weight(shape=tuple([1, self.kernel.shape.as_list()[-1]]),\n                         initializer=initializers.RandomNormal(0, 1),\n                         name='sn',\n                         trainable=False)\n        \n        # Set input spec.\n        self.input_spec = InputSpec(ndim=4, axes={channel_axis: input_dim})\n        self.built = True  \n    \n    def call(self, inputs):\n        input_shape = K.shape(inputs)\n        batch_size = input_shape[0]\n        if self.data_format == 'channels_first':\n            h_axis, w_axis = 2, 3\n        else:\n            h_axis, w_axis = 1, 2\n\n        height, width = input_shape[h_axis], input_shape[w_axis]\n        kernel_h, kernel_w = self.kernel_size\n        stride_h, stride_w = self.strides\n        if self.output_padding is None:\n            out_pad_h = out_pad_w = None\n        else:\n            out_pad_h, out_pad_w = self.output_padding\n\n        # Infer the dynamic output shape:\n        out_height = conv_utils.deconv_length(height,\n                                              stride_h, kernel_h,\n                                              self.padding,\n                                              out_pad_h)\n        out_width = conv_utils.deconv_length(width,\n                                             stride_w, kernel_w,\n                                             self.padding,\n                                             out_pad_w)\n        if self.data_format == 'channels_first':\n            output_shape = (batch_size, self.filters, out_height, out_width)\n        else:\n            output_shape = (batch_size, out_height, out_width, self.filters)\n            \n        #Spectral Normalization    \n        def _l2normalize(v, eps=1e-12):\n            return v / (K.sum(v ** 2) ** 0.5 + eps)\n        def power_iteration(W, u):\n            #Accroding the paper, we only need to do power iteration one time.\n            _u = u\n            _v = _l2normalize(K.dot(_u, K.transpose(W)))\n            _u = _l2normalize(K.dot(_v, W))\n            return _u, _v\n        W_shape = self.kernel.shape.as_list()\n        #Flatten the Tensor\n        W_reshaped = K.reshape(self.kernel, [-1, W_shape[-1]])\n        _u, _v = power_iteration(W_reshaped, self.u)\n        #Calculate Sigma\n        sigma=K.dot(_v, W_reshaped)\n        sigma=K.dot(sigma, K.transpose(_u))\n        #normalize it\n        W_bar = W_reshaped / sigma\n        #reshape weight tensor\n        if training in {0, False}:\n            W_bar = K.reshape(W_bar, W_shape)\n        else:\n            with tf.control_dependencies([self.u.assign(_u)]):\n                W_bar = K.reshape(W_bar, W_shape)\n        self.kernel = W_bar\n        \n        outputs = K.conv2d_transpose(\n            inputs,\n            self.kernel,\n            output_shape,\n            self.strides,\n            padding=self.padding,\n            data_format=self.data_format)\n\n        if self.use_bias:\n            outputs = K.bias_add(\n                outputs,\n                self.bias,\n                data_format=self.data_format)\n\n        if self.activation is not None:\n            return self.activation(outputs)\n        return outputs","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:39.574418Z","iopub.status.idle":"2022-02-09T22:58:39.575015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transposed_conv(model, out_channels, ksize, stride_size, ptype='same'):\n    model.add(Conv2DTranspose(out_channels, (ksize, ksize),\n                              strides=(stride_size, stride_size), padding=ptype, \n                              kernel_initializer=weight_initializer, use_bias=False))\n    model.add(BatchNormalization())\n    model.add(ReLU())\n    return model\n\ndef transposed_convSN(model, out_channels, ksize, stride_size, ptype='same'):\n    model.add(ConvSN2DTranspose(out_channels, (ksize, ksize), \n                              strides=(stride_size, stride_size), padding=ptype, \n                              kernel_initializer=weight_initializer, use_bias=False))\n    model.add(BatchNormalization())\n    model.add(ReLU())\n    return model\n\ndef convSN(model, out_channels, ksize, stride_size):\n    model.add(ConvSN2D(out_channels, (ksize, ksize), strides=(stride_size, stride_size), padding='same',\n                     kernel_initializer=weight_initializer, use_bias=False))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=leaky_relu_slope))\n    #model.add(Dropout(dropout_rate))\n    return model\n\ndef conv(model, out_channels, ksize, stride_size):\n    model.add(Conv2D(out_channels, (ksize, ksize), strides=(stride_size, stride_size), padding='same',\n                     kernel_initializer=weight_initializer, use_bias=False))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=leaky_relu_slope))\n    #model.add(Dropout(dropout_rate))\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:39.576081Z","iopub.status.idle":"2022-02-09T22:58:39.576671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The Generator","metadata":{}},{"cell_type":"code","source":"def DogGenerator():\n    model = Sequential()\n    model.add(Dense(image_width // scale_factor * image_height // scale_factor * 128,\n                    input_shape=(noise_dim,), kernel_initializer=weight_initializer))\n    #model.add(BatchNormalization(epsilon=BN_EPSILON, momentum=BN_MOMENTUM))\n    #model.add(LeakyReLU(alpha=leaky_relu_slope))\n    model.add(Reshape((image_height // scale_factor, image_width // scale_factor, 128)))\n    \n    model = transposed_conv(model, 512, ksize=5, stride_size=1)\n    model.add(Dropout(dropout_rate))\n    model = transposed_conv(model, 256, ksize=5, stride_size=2)\n    model.add(Dropout(dropout_rate))\n    model = transposed_conv(model, 128, ksize=5, stride_size=2)\n    model = transposed_conv(model, 64, ksize=5, stride_size=2)\n    model = transposed_conv(model, 32, ksize=5, stride_size=2)\n    \n    model.add(Dense(3, activation='tanh', kernel_initializer=weight_initializer))\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:40.06391Z","iopub.execute_input":"2022-02-09T22:58:40.064235Z","iopub.status.idle":"2022-02-09T22:58:40.074388Z","shell.execute_reply.started":"2022-02-09T22:58:40.064168Z","shell.execute_reply":"2022-02-09T22:58:40.073217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dog_generator = DogGenerator()\nprint(dog_generator.summary())","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:40.527454Z","iopub.execute_input":"2022-02-09T22:58:40.527751Z","iopub.status.idle":"2022-02-09T22:58:40.565994Z","shell.execute_reply.started":"2022-02-09T22:58:40.527701Z","shell.execute_reply":"2022-02-09T22:58:40.565022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;We can confirm that the <b>Generator</b> works initially by feeding it a random noise vector of the right size. The output image should be normalized to similar black-colored pixels due to the <b>Weight Initialization</b> step described earlier <b>(they should be relatively zero-centered)</b>.","metadata":{}},{"cell_type":"code","source":"# random noise vector\nnoise = tf.random.normal([1,noise_dim])\n#sample = generate_latent_points(100, 50)\n# run the generator model with the noise vector as input\ngenerated_image = dog_generator(noise, training=False)\n# display output\nplt.imshow(generated_image[0, :, :, :])\nprint(generated_image.shape)\n#print(sample.shape, sample.mean(), sample.std())\nprint(noise.shape, tf.math.reduce_mean(noise).numpy(), tf.math.reduce_std(noise).numpy())","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:41.000677Z","iopub.execute_input":"2022-02-09T22:58:41.000983Z","iopub.status.idle":"2022-02-09T22:58:41.035266Z","shell.execute_reply.started":"2022-02-09T22:58:41.000931Z","shell.execute_reply":"2022-02-09T22:58:41.034069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The Discriminator","metadata":{}},{"cell_type":"code","source":"def DogDiscriminator(spectral_normalization=True):\n    model = Sequential()\n    if spectral_normalization:\n        model.add(ConvSN2D(64, (5, 5), strides=(1,1), padding='same', use_bias=False,\n                         input_shape=[image_height, image_width, image_channels], \n                         kernel_initializer=weight_initializer))\n        #model.add(BatchNormalization(epsilon=BN_EPSILON, momentum=BN_MOMENTUM))\n        model.add(LeakyReLU(alpha=leaky_relu_slope))\n        #model.add(Dropout(dropout_rate))\n        \n        model = convSN(model, 64, ksize=5, stride_size=2)\n        #model = convSN(model, 128, ksize=3, stride_size=1)\n        model = convSN(model, 128, ksize=5, stride_size=2)\n        #model = convSN(model, 256, ksize=3, stride_size=1)\n        model = convSN(model, 256, ksize=5, stride_size=2)\n        #model = convSN(model, 512, ksize=3, stride_size=1)\n        #model.add(Dropout(dropout_rate))\n\n        model.add(Flatten())\n        model.add(DenseSN(1, activation='sigmoid'))\n    else:\n        model.add(Conv2D(64, (4, 4), strides=(2,2), padding='same', use_bias=False,\n                         input_shape=[image_height, image_width, image_channels], \n                         kernel_initializer=weight_initializer))\n        #model.add(BatchNormalization(epsilon=BN_EPSILON, momentum=BN_MOMENTUM))\n        model.add(LeakyReLU(alpha=leaky_relu_slope))\n        #model.add(Dropout(dropout_rate))\n\n        model = conv(model, 64, ksize=4, stride_size=2)\n        #model = convSN(model, 128, ksize=3, stride_size=1)\n        model = conv(model, 128, ksize=4, stride_size=2)\n        #model = convSN(model, 256, ksize=3, stride_size=1)\n        model = conv(model, 256, ksize=4, stride_size=2)\n        #model = convSN(model, 512, ksize=3, stride_size=1)\n\n        model.add(Flatten())\n        model.add(Dense(1, activation='sigmoid'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:41.709136Z","iopub.execute_input":"2022-02-09T22:58:41.709424Z","iopub.status.idle":"2022-02-09T22:58:41.721113Z","shell.execute_reply.started":"2022-02-09T22:58:41.709374Z","shell.execute_reply":"2022-02-09T22:58:41.719845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dog_discriminator = DogDiscriminator(spectral_normalization=True)\nprint(dog_discriminator.summary())","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:41.759757Z","iopub.execute_input":"2022-02-09T22:58:41.759999Z","iopub.status.idle":"2022-02-09T22:58:41.797773Z","shell.execute_reply.started":"2022-02-09T22:58:41.759955Z","shell.execute_reply":"2022-02-09T22:58:41.796614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below we can observe, that <b>D</b> indeed outputs a probability very close to <b>0.5</b> for a noisy input.","metadata":{}},{"cell_type":"code","source":"decision = dog_discriminator(generated_image)\nprint(decision)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:42.160353Z","iopub.execute_input":"2022-02-09T22:58:42.160655Z","iopub.status.idle":"2022-02-09T22:58:42.187112Z","shell.execute_reply.started":"2022-02-09T22:58:42.160603Z","shell.execute_reply":"2022-02-09T22:58:42.185925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Further optimization tricks","metadata":{}},{"cell_type":"markdown","source":"### Label smoothing","metadata":{}},{"cell_type":"code","source":"# Label smoothing -- technique from GAN hacks, instead of assigning 1/0 as class labels, we assign a random integer in range [0.7, 1.0] for positive class\n# and [0.0, 0.3] for negative class\n\ndef smooth_positive_labels(y):\n    return y - 0.3 + (np.random.random(y.shape) * 0.5)\n\ndef smooth_negative_labels(y):\n    return y + np.random.random(y.shape) * 0.3","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:43.030353Z","iopub.execute_input":"2022-02-09T22:58:43.0308Z","iopub.status.idle":"2022-02-09T22:58:43.036258Z","shell.execute_reply.started":"2022-02-09T22:58:43.030702Z","shell.execute_reply":"2022-02-09T22:58:43.035491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Introducing some noise to the labels","metadata":{}},{"cell_type":"code","source":"# randomly flip some labels\ndef noisy_labels(y, p_flip):\n    # determine the number of labels to flip\n    n_select = int(p_flip * int(y.shape[0]))\n    # choose labels to flip\n    flip_ix = np.random.choice([i for i in range(int(y.shape[0]))], size=n_select)\n    \n    op_list = []\n    # invert the labels in place\n    #y_np[flip_ix] = 1 - y_np[flip_ix]\n    for i in range(int(y.shape[0])):\n        if i in flip_ix:\n            op_list.append(tf.subtract(1, y[i]))\n        else:\n            op_list.append(y[i])\n    \n    outputs = tf.stack(op_list)\n    return outputs","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:43.540334Z","iopub.execute_input":"2022-02-09T22:58:43.540634Z","iopub.status.idle":"2022-02-09T22:58:43.548197Z","shell.execute_reply.started":"2022-02-09T22:58:43.540584Z","shell.execute_reply":"2022-02-09T22:58:43.547165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Optimizers","metadata":{}},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;The best proven optimization algorithm for this task is <b>Adam</b> with a standard learning rate of <b>0.0002</b> for both models and a beta of <b>0.5</b>.","metadata":{}},{"cell_type":"code","source":"generator_optimizer = tf.train.AdamOptimizer(learning_rate=lr_initial_g, beta1=0.5)\ndiscriminator_optimizer = tf.train.AdamOptimizer(learning_rate=lr_initial_d, beta1=0.5)\n# This method returns a helper function to compute cross entropy loss\ncross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:44.490392Z","iopub.execute_input":"2022-02-09T22:58:44.490702Z","iopub.status.idle":"2022-02-09T22:58:44.496507Z","shell.execute_reply.started":"2022-02-09T22:58:44.490648Z","shell.execute_reply":"2022-02-09T22:58:44.495685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining the loss functions","metadata":{}},{"cell_type":"code","source":"def discriminator_loss(real_output, fake_output, loss_func, apply_label_smoothing=True, label_noise=True):\n    if label_noise and apply_label_smoothing:\n        real_output_noise = noisy_labels(tf.ones_like(real_output), 0.05)\n        fake_output_noise = noisy_labels(tf.zeros_like(fake_output), 0.05)\n        real_output_smooth = smooth_positive_labels(real_output_noise)\n        fake_output_smooth = smooth_negative_labels(fake_output_noise)\n        if loss_func == 'gan': \n            real_loss = cross_entropy(tf.ones_like(real_output_smooth), real_output)\n            fake_loss = cross_entropy(tf.zeros_like(fake_output_smooth), fake_output)\n        else:\n            if loss_func == 'ralsgan':\n                return (tf.reduce_mean(tf.square(real_output_smooth - tf.reduce_mean(fake_output_smooth) - tf.ones_like(real_output_smooth)))\n        + tf.reduce_mean(tf.square(fake_output_smooth - tf.reduce_mean(real_output_smooth) + tf.ones_like(fake_output_smooth)))) / 2.\n            elif loss_func == 'rasgan':\n                avg_fake_logit = tf.reduce_mean(fake_output_smooth)\n                avg_real_logit = tf.reduce_mean(real_output_smooth)\n                D_r_tilde = tf.nn.sigmoid(real_output_smooth - avg_fake_logit)\n                D_f_tilde = tf.nn.sigmoid(fake_output_smooth - avg_real_logit)\n                total_loss = - tf.reduce_mean(tf.log(\n                    D_r_tilde + 1e-14)) - tf.reduce_mean(tf.log(1 - D_f_tilde + 1e-14))\n                return total_loss\n            elif loss_func == 'rahinge':\n                real_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(real_output_smooth) - (real_output_smooth - tf.reduce_mean(fake_output_smooth))))\n                fake_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(fake_output_smooth) + (fake_output_smooth - tf.reduce_mean(real_output_smooth))))\n        total_loss = real_loss + fake_loss\n        return total_loss\n    elif label_noise and not apply_label_smoothing:\n        real_output_noise = noisy_labels(tf.ones_like(real_output), 0.05)\n        fake_output_noise = noisy_labels(tf.zeros_like(fake_output), 0.05)\n        if loss_func == 'gan': \n            real_loss = cross_entropy(tf.ones_like(real_output_noise), real_output)\n            fake_loss = cross_entropy(tf.zeros_like(fake_output_noise), fake_output)\n        else:\n            if loss_func == 'ralsgan':\n                return (tf.reduce_mean(tf.square(real_output_noise - tf.reduce_mean(fake_output_noise) - tf.ones_like(real_output_noise)))\n        + tf.reduce_mean(tf.square(fake_output_noise - tf.reduce_mean(real_output_noise) + tf.ones_like(fake_output_noise)))) / 2.\n            elif loss_func == 'rasgan':\n                avg_fake_logit = tf.reduce_mean(fake_output_noise)\n                avg_real_logit = tf.reduce_mean(real_output_noise)\n                D_r_tilde = tf.nn.sigmoid(real_output_noise - avg_fake_logit)\n                D_f_tilde = tf.nn.sigmoid(fake_output_noise - avg_real_logit)\n                total_loss = - tf.reduce_mean(tf.log(\n                    D_r_tilde + 1e-14)) - tf.reduce_mean(tf.log(1 - D_f_tilde + 1e-14))\n                return total_loss\n            elif loss_func == 'rahinge':\n                real_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(real_output_noise) - (real_output_noise - tf.reduce_mean(fake_output_noise))))\n                fake_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(fake_output_noise) + (fake_output_noise - tf.reduce_mean(real_output_noise))))\n        total_loss = real_loss + fake_loss\n        return total_loss\n    elif apply_label_smoothing and not label_noise:\n        real_output_smooth = smooth_positive_labels(tf.ones_like(real_output))\n        fake_output_smooth = smooth_negative_labels(tf.zeros_like(fake_output))\n        if loss_func == 'gan': \n            real_loss = cross_entropy(tf.ones_like(real_output_smooth), real_output)\n            fake_loss = cross_entropy(tf.zeros_like(fake_output_smooth), fake_output)\n        else:\n            if loss_func == 'ralsgan':\n                return (tf.reduce_mean(tf.square(real_output_smooth - tf.reduce_mean(fake_output_smooth) - tf.ones_like(real_output_smooth)))\n        + tf.reduce_mean(tf.square(fake_output_smooth - tf.reduce_mean(real_output_smooth) + tf.ones_like(fake_output_smooth)))) / 2.\n            elif loss_func == 'rasgan':\n                avg_fake_logit = tf.reduce_mean(fake_output_smooth)\n                avg_real_logit = tf.reduce_mean(real_output_smooth)\n                D_r_tilde = tf.nn.sigmoid(real_output_smooth - avg_fake_logit)\n                D_f_tilde = tf.nn.sigmoid(fake_output_smooth - avg_real_logit)\n                total_loss = - tf.reduce_mean(tf.log(\n                    D_r_tilde + 1e-14)) - tf.reduce_mean(tf.log(1 - D_f_tilde + 1e-14))\n                return total_loss\n            elif loss_func == 'rahinge':\n                real_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(real_output_smooth) - (real_output_smooth - tf.reduce_mean(fake_output_smooth))))\n                fake_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(fake_output_smooth) + (fake_output_smooth - tf.reduce_mean(real_output_smooth))))\n        total_loss = real_loss + fake_loss\n        return total_loss    \n    else:\n        if loss_func == 'gan': \n            real_loss = cross_entropy(tf.ones_like(real_output), real_output)\n            fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)\n        else:\n            if loss_func == 'ralsgan':\n                return (tf.reduce_mean(tf.square(real_output - tf.reduce_mean(fake_output) - tf.ones_like(real_output)))\n        + tf.reduce_mean(tf.square(fake_output - tf.reduce_mean(real_output) + tf.ones_like(fake_output)))) / 2.\n            elif loss_func == 'rasgan':\n                avg_fake_logit = tf.reduce_mean(fake_output)\n                avg_real_logit = tf.reduce_mean(real_output)\n                D_r_tilde = tf.nn.sigmoid(real_output - avg_fake_logit)\n                D_f_tilde = tf.nn.sigmoid(fake_output - avg_real_logit)\n                total_loss = - tf.reduce_mean(tf.log(\n                    D_r_tilde + 1e-14)) - tf.reduce_mean(tf.log(1 - D_f_tilde + 1e-14))\n                return total_loss\n            elif loss_func == 'rahinge':\n                real_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(real_output) - (real_output - tf.reduce_mean(fake_output))))\n                fake_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(fake_output) + (fake_output - tf.reduce_mean(real_output))))\n        total_loss = real_loss + fake_loss\n        return total_loss","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:46.159232Z","iopub.execute_input":"2022-02-09T22:58:46.159529Z","iopub.status.idle":"2022-02-09T22:58:46.195262Z","shell.execute_reply.started":"2022-02-09T22:58:46.159478Z","shell.execute_reply":"2022-02-09T22:58:46.194175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generator_loss(real_output, fake_output, loss_func, apply_label_smoothing=True):\n    if apply_label_smoothing:\n        fake_output_smooth = smooth_negative_labels(tf.ones_like(fake_output))\n        if loss_func == 'gan':\n            return cross_entropy(tf.ones_like(fake_output_smooth), fake_output)\n        else:\n            if loss_func == 'ralsgan':\n                return (tf.reduce_mean(tf.square(real_output - tf.reduce_mean(fake_output_smooth) + tf.ones_like(real_output)))\n        + tf.reduce_mean(tf.square(fake_output_smooth - tf.reduce_mean(real_output) - tf.ones_like(fake_output_smooth)))) / 2.\n            elif loss_func == 'rasgan':\n                avg_fake_logit = tf.reduce_mean(fake_output_smooth)\n                avg_real_logit = tf.reduce_mean(real_output)\n                D_r_tilde = tf.nn.sigmoid(real_output - avg_fake_logit)\n                D_f_tilde = tf.nn.sigmoid(fake_output_smooth - avg_real_logit)\n                total_loss = - tf.reduce_mean(tf.log(\n                    D_f_tilde + 1e-14)) - tf.reduce_mean(tf.log(1 - D_r_tilde + 1e-14))\n                return total_loss\n            elif loss_func == 'rahinge':\n                fake_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(fake_output_smooth) - (fake_output_smooth - tf.reduce_mean(real_output))))\n                real_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(real_output) + (real_output - tf.reduce_mean(fake_output_smooth))))\n                loss = fake_loss + real_loss\n                return loss\n    else:           \n        if loss_func == 'gan':\n            return cross_entropy(tf.ones_like(fake_output), fake_output)\n        else:\n            if loss_func == 'ralsgan':\n                return (tf.reduce_mean(tf.square(real_output - tf.reduce_mean(fake_output) + tf.ones_like(real_output)))\n        + tf.reduce_mean(tf.square(fake_output - tf.reduce_mean(real_output) - tf.ones_like(fake_output)))) / 2.\n            elif loss_func == 'rasgan':\n                avg_fake_logit = tf.reduce_mean(fake_output)\n                avg_real_logit = tf.reduce_mean(real_output)\n                D_r_tilde = tf.nn.sigmoid(real_output - avg_fake_logit)\n                D_f_tilde = tf.nn.sigmoid(fake_output - avg_real_logit)\n                total_loss = - tf.reduce_mean(tf.log(\n                    D_f_tilde + 1e-14)) - tf.reduce_mean(tf.log(1 - D_r_tilde + 1e-14))\n                return total_loss\n            elif loss_func == 'rahinge':\n                fake_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(fake_output) - (fake_output - tf.reduce_mean(real_output))))\n                real_loss = tf.reduce_mean(tf.nn.relu(tf.ones_like(real_output) + (real_output - tf.reduce_mean(fake_output))))\n                loss = fake_loss + real_loss\n                return loss ","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:46.196827Z","iopub.execute_input":"2022-02-09T22:58:46.197336Z","iopub.status.idle":"2022-02-09T22:58:46.217399Z","shell.execute_reply.started":"2022-02-09T22:58:46.197289Z","shell.execute_reply":"2022-02-09T22:58:46.216666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define a checkpointer","metadata":{}},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;Let's finally add a checkpointer to be able to save and reuse our model.","metadata":{}},{"cell_type":"code","source":"checkpoint_dir = '/training_checkpoints'\ncheckpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\ncheckpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n                                 discriminator_optimizer=discriminator_optimizer,\n                                 generator=dog_generator,\n                                 discriminator=dog_discriminator)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:47.055535Z","iopub.execute_input":"2022-02-09T22:58:47.056053Z","iopub.status.idle":"2022-02-09T22:58:47.09212Z","shell.execute_reply.started":"2022-02-09T22:58:47.055866Z","shell.execute_reply":"2022-02-09T22:58:47.087921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Main training loop","metadata":{}},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;Let's also fix the number of epochs for training and the number of images to feed to the <b>Generator</b> for visualizing results during that training.","metadata":{}},{"cell_type":"code","source":"EPOCHS = 280\nnum_examples_to_generate = 64\nseed = tf.random.normal([num_examples_to_generate, noise_dim])","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:47.938362Z","iopub.execute_input":"2022-02-09T22:58:47.938666Z","iopub.status.idle":"2022-02-09T22:58:47.944096Z","shell.execute_reply.started":"2022-02-09T22:58:47.938614Z","shell.execute_reply":"2022-02-09T22:58:47.943252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_step(images, loss_type='gan'):\n    noise = tf.random.normal([batch_size, noise_dim])\n\n    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n        generated_images = dog_generator(noise, training=True)\n        \n        real_output = dog_discriminator(images, training=True)\n        fake_output = dog_discriminator(generated_images, training=True)\n        \n        gen_loss = generator_loss(real_output, fake_output, loss_type, apply_label_smoothing=True)\n        disc_loss = discriminator_loss(real_output, fake_output, loss_type, \n                                       apply_label_smoothing=True, label_noise=True)\n \n    gradients_of_generator = gen_tape.gradient(gen_loss, dog_generator.trainable_variables)\n    gradients_of_discriminator = disc_tape.gradient(disc_loss, dog_discriminator.trainable_variables)\n\n    generator_optimizer.apply_gradients(zip(gradients_of_generator, dog_generator.trainable_variables))\n    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, dog_discriminator.trainable_variables))\n    \n    return gen_loss, disc_loss","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:48.41662Z","iopub.execute_input":"2022-02-09T22:58:48.416943Z","iopub.status.idle":"2022-02-09T22:58:48.425417Z","shell.execute_reply.started":"2022-02-09T22:58:48.416888Z","shell.execute_reply":"2022-02-09T22:58:48.424621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;Let's also define some functions to visualize the model losses by epoch and as a whole.","metadata":{}},{"cell_type":"code","source":"def plot_losses(G_losses, D_losses, all_gl, all_dl, epoch):\n    plt.figure(figsize=(10,5))\n    plt.title(\"Generator and Discriminator Loss - EPOCH {}\".format(epoch))\n    plt.plot(G_losses,label=\"G\")\n    plt.plot(D_losses,label=\"D\")\n    plt.xlabel(\"Iterations\")\n    plt.ylabel(\"Loss\")\n    plt.legend()\n    ymax = plt.ylim()[1]\n    plt.show()\n    \n    plt.figure(figsize=(10,5))\n    plt.plot(np.arange(len(all_gl)),all_gl,label='G')\n    plt.plot(np.arange(len(all_dl)),all_dl,label='D')\n    plt.legend()\n    #plt.ylim((0,np.min([1.1*np.max(all_gl),2*ymax])))\n    plt.title('All Time Loss')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:49.266379Z","iopub.execute_input":"2022-02-09T22:58:49.266817Z","iopub.status.idle":"2022-02-09T22:58:49.274676Z","shell.execute_reply.started":"2022-02-09T22:58:49.266758Z","shell.execute_reply":"2022-02-09T22:58:49.273973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;We can also use the following function to plot a grid of the generated images.","metadata":{}},{"cell_type":"code","source":"def generate_and_save_images(model, epoch, test_input, rows, cols):\n    # Notice `training` is set to False.\n    # This is so all layers run in inference mode (batchnorm).\n    predictions = model(test_input, training=False)\n    fig = plt.figure(figsize=(14,14))\n    for i in range(predictions.shape[0]):\n        plt.subplot(rows, cols, i+1)\n        plt.imshow((predictions[i, :, :, :] * 127.5 + 127.5) / 255.)\n        plt.axis('off') \n        \n    plt.subplots_adjust(wspace=0, hspace=0)\n    plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:49.745275Z","iopub.execute_input":"2022-02-09T22:58:49.745613Z","iopub.status.idle":"2022-02-09T22:58:49.752695Z","shell.execute_reply.started":"2022-02-09T22:58:49.745552Z","shell.execute_reply":"2022-02-09T22:58:49.751891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;To generate a single test image, we can also reuse the same method.","metadata":{}},{"cell_type":"code","source":"def generate_test_image(model, noise_dim=noise_dim):\n    test_input = tf.random.normal([1, noise_dim])\n    # Notice `training` is set to False.\n    # This is so all layers run in inference mode (batchnorm).\n    predictions = model(test_input, training=False)\n    fig = plt.figure(figsize=(5,5))\n    plt.imshow((predictions[0, :, :, :] * 127.5 + 127.5) / 255.)\n    plt.axis('off') \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:50.304383Z","iopub.execute_input":"2022-02-09T22:58:50.30469Z","iopub.status.idle":"2022-02-09T22:58:50.311775Z","shell.execute_reply.started":"2022-02-09T22:58:50.304638Z","shell.execute_reply":"2022-02-09T22:58:50.31087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluating GANs - FID and MIFID","metadata":{}},{"cell_type":"markdown","source":"### Inception Score","metadata":{}},{"cell_type":"markdown","source":"### FID (Frechet Inception Distance)","metadata":{}},{"cell_type":"code","source":"class KernelEvalException(Exception):\n    pass\n\n\n@dataclass\nclass MiFIDEvaluator(object):\n    model_path: str\n    train_images_path: str\n    feature_path: str = None\n    imsize: int = 64\n    output_layer: str = 'Pretrained_Net/pool_3:0'\n    input_layer: str = 'Pretrained_Net/ExpandDims:0'\n    output_shape: int = 2048\n    cosine_distance_eps: float = 0.1\n    batch_size: int = 50\n    fid_epsilon: float = 1e-14\n    \n    def __post_init__(self):\n        tf.reset_default_graph()\n        self.create_model_graph()\n        with tf.Session() as sess:\n            if self.feature_path is None:\n                self.mu2, self.sigma2, self.features2 = self._handle_path_memorization(\n                    self.train_images_path, sess, is_checksize=False, is_check_png=False)\n            else:\n                with np.load(self.feature_path) as f:\n                    self.mu2, self.sigma2, self.features2 = f['m'], f['s'], f['features']\n    \n    def create_model_graph(self):\n        with tf.gfile.FastGFile(self.model_path, 'rb') as f:\n            graph_def = tf.GraphDef()\n            graph_def.ParseFromString(f.read())\n            _ = tf.import_graph_def(graph_def, name='Pretrained_Net')\n            \n    def img_read_checks(self, filename, is_checksize=False, is_check_png=False):\n        im = Image.open(str(filename))\n        if is_checksize and im.size != (self.imsize, self.imsize):\n            raise KernelEvalException(f'The images are not of size {check_imsize}')\n        if is_check_png and im.format != 'PNG':\n            raise KernelEvalException('Only PNG images should be submitted.')\n\n        if self.imsize is None:\n            return im\n        else:\n            return im.resize((self.imsize, self.imsize), Image.ANTIALIAS)\n        \n    def _get_model_layer(self, sess):\n        layer = sess.graph.get_tensor_by_name(self.output_layer)\n        ops = layer.graph.get_operations()\n        for op_idx, op in enumerate(ops):\n            for o in op.outputs:\n                shape = o.get_shape()\n                if shape._dims != []:\n                    shape = [s.value for s in shape]\n                    new_shape = []\n                    for j, s in enumerate(shape):\n                        if s == 1 and j == 0:\n                            new_shape.append(None)\n                        else:\n                            new_shape.append(s)\n                    o.__dict__['_shape_val'] = tf.TensorShape(new_shape)\n        return layer\n        \n    def get_activations(self, images, sess):\n        inception_layer = self._get_model_layer(sess)\n        n_images = images.shape[0]\n        if self.batch_size > n_images:\n            warnings.warn('batch size is bigger than the data size. setting batch size to data size')\n            self.batch_size = n_images\n        n_batches = n_images // self.batch_size + 1\n        pred_arr = np.empty((n_images, self.output_shape))\n        for i in range(n_batches):\n            start = i * self.batch_size\n            if start + self.batch_size < n_images:\n                end = start + self.batch_size\n            else:\n                end = n_images\n\n            batch = images[start:end]\n            pred = sess.run(inception_layer, {self.input_layer: batch})\n            pred_arr[start:end] = pred.reshape(-1, self.output_shape)\n        return pred_arr\n        \n    def calculate_activation_statistics(self, images, sess):\n        act = self.get_activations(images, sess)\n        mu = np.mean(act, axis=0)\n        sigma = np.cov(act, rowvar=False)\n        return mu, sigma, act\n            \n    def _handle_path_memorization(self, path, sess, is_checksize, is_check_png):\n        path = Path(path)\n        files = list(path.glob('*.jpg')) + list(path.glob('*.png'))\n\n        # In production we don't resize input images. This is just for demo purpose. \n        x = np.array([np.array(self.img_read_checks(fn, is_checksize, is_check_png)) for fn in files])\n        m, s, features = self.calculate_activation_statistics(x, sess)\n        del x\n        return m, s, features\n    \n    def calculate_frechet_distance(self, mu1, sigma1):\n        mu1 = np.atleast_1d(mu1)\n        mu2 = np.atleast_1d(self.mu2)\n        sigma1 = np.atleast_2d(sigma1)\n        sigma2 = np.atleast_2d(self.sigma2)\n\n        assert mu1.shape == mu2.shape, 'Training and test mean vectors have different lengths'\n        assert sigma1.shape == sigma2.shape, 'Training and test covariances have different dimensions'\n\n        # product might be almost singular\n        covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)\n        if not np.isfinite(covmean).all():\n            msg = f'fid calculation produces singular product; adding {self.eps} to diagonal of cov estimates'\n            warnings.warn(msg)\n            offset = np.eye(sigma1.shape[0]) * self.eps\n            covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))\n            \n        # numerical error might give slight imaginary component\n        if np.iscomplexobj(covmean):\n            if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):\n                m = np.max(np.abs(covmean.imag))\n                raise ValueError(f'Imaginary component {m}')\n            covmean = covmean.real\n        tr_covmean = np.trace(covmean)\n        return (mu1 - mu2).dot(mu1 - mu2) + np.trace(sigma1) + np.trace(sigma2) - 2 * tr_covmean\n    \n    @staticmethod\n    def normalize_rows(x):\n        return np.nan_to_num(x / np.linalg.norm(x, ord=2, axis=1, keepdims=True))\n    \n    def cosine_distance(self, features1):\n        features1_nozero = features1[np.sum(features1, axis=1) != 0]\n        features2_nozero = self.features2[np.sum(self.features2, axis=1) != 0]\n        norm_f1 = self.normalize_rows(features1_nozero)\n        norm_f2 = self.normalize_rows(features2_nozero)\n\n        d = 1.0 - np.abs(np.matmul(norm_f1, norm_f2.T))\n        mean_min_d = np.mean(np.min(d, axis=1))\n        return mean_min_d\n            \n    def calculate_kid_given_paths(self, user_images_unzipped_path):\n        with tf.Session() as sess:\n            sess.run(tf.global_variables_initializer())\n            m1, s1, features1 = self._handle_path_memorization(\n                user_images_unzipped_path, sess, is_checksize=True, is_check_png=True)\n\n            fid_value = self.calculate_frechet_distance(m1, s1)\n            distance = self.cosine_distance(features1)\n            return fid_value, distance\n        \n    def distance_thresholding(self, d):\n        if d < self.cosine_distance_eps:\n            return d\n        else:\n            return 1\n        \n    def evaluate(self, user_images_unzipped_path):\n        fid_value, distance = self.calculate_kid_given_paths(user_images_unzipped_path)\n        distance = self.distance_thresholding(distance)\n        return fid_value, distance, fid_value / (distance + self.fid_epsilon)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:52.035554Z","iopub.execute_input":"2022-02-09T22:58:52.036064Z","iopub.status.idle":"2022-02-09T22:58:52.080109Z","shell.execute_reply.started":"2022-02-09T22:58:52.035815Z","shell.execute_reply":"2022-02-09T22:58:52.079346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image zipping and saving functions","metadata":{}},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;I'm just going to add two more for zipping the final <b>10K</b> images for submission and generating temporary images to calculate the <b>MIFID</b> in between certain epochs during training.","metadata":{}},{"cell_type":"code","source":"def zip_images(filename='images.zip'):\n    # SAVE TO ZIP FILE NAMED IMAGES.ZIP\n    z = zipfile.PyZipFile(filename, mode='w')\n    for k in range(image_sample_size):\n        generated_image = dog_generator(tf.random.normal([1, noise_dim]), training=False)\n        f = str(k)+'.png'\n        img = np.array(generated_image)\n        img = (img[0, :, :, :] + 1.) / 2.\n        img = Image.fromarray((255*img).astype('uint8').reshape((image_height,image_width,image_channels)))\n        img.save(f,'PNG')\n        z.write(f)\n        os.remove(f)\n        #if k % 1000==0: print(k)\n    z.close()\n    print('Saved final images for submission.')\n    \ndef save_images(directory=OUT_DIR):\n    for k in range(image_sample_size):\n        generated_image = dog_generator(tf.random.normal([1, noise_dim]), training=False)\n        f = str(k)+'.png'\n        f = os.path.join(directory, f)\n        img = np.array(generated_image)\n        img = (img[0, :, :, :] + 1.) / 2.\n        img = Image.fromarray((255*img).astype('uint8').reshape((image_height,image_width,image_channels)))\n        img.save(f,'PNG')\n        #if k % 1000==0: print(k)\n    print('Saved temporary images for evaluation.')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:58:59.748701Z","iopub.execute_input":"2022-02-09T22:58:59.749013Z","iopub.status.idle":"2022-02-09T22:58:59.760257Z","shell.execute_reply.started":"2022-02-09T22:58:59.748962Z","shell.execute_reply":"2022-02-09T22:58:59.759478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model training and avoiding Mode Collapse","metadata":{}},{"cell_type":"markdown","source":"### Learning rate decay","metadata":{}},{"cell_type":"code","source":"display_results = 40\ncalculate_mifid = 100\nreplay_step = 50\ndecay_step = 50\n\ndef train(dataset, epochs):\n    all_gl = np.array([]); all_dl = np.array([])\n    \n    exp_replay = []\n    for epoch in tqdm(range(epochs)):\n        \n        G_loss = []; D_loss = []\n        \n        start = time.time()\n        new_lr_d = lr_initial_d\n        new_lr_g = lr_initial_g\n        global_step = 0\n        \n        for image_batch in dataset:\n            g_loss, d_loss = train_step(image_batch)\n            global_step = global_step + 1\n            G_loss.append(g_loss); D_loss.append(d_loss)\n            all_gl = np.append(all_gl,np.array([G_loss]))\n            all_dl = np.append(all_dl,np.array([D_loss]))\n        \n        # generate an extra image for each epoch and store it in memory for experience replay\n        \n        '''\n        generated_image = dog_generator(tf.random.normal([1, noise_dim]), training=False)\n        exp_replay.append(generated_image)\n        if len(exp_replay) == replay_step:\n            print('Executing experience replay..')\n            replay_images = np.array([p[0] for p in exp_replay])\n            dog_discriminator(replay_images, training=True)\n            exp_replay = []    \n        '''\n         \n        #display.clear_output(wait=True)\n        if (epoch + 1) % display_results == 0 or epoch == 0:\n            plot_losses(G_loss, D_loss, all_gl, all_dl, epoch + 1)\n            generate_and_save_images(dog_generator, epoch + 1, seed, rows=8, cols=8)\n        \n        if (epoch + 1) % calculate_mifid == 0:            \n            OUT_DIR.mkdir(exist_ok=True)\n            save_images(OUT_DIR)\n            evaluator = MiFIDEvaluator(MODEL_PATH, TRAIN_DIR)\n            fid_value, distance, mi_fid_score = evaluator.evaluate(OUT_DIR)\n            print(f'FID: {fid_value:.5f}')\n            print(f'distance: {distance:.5f}')\n            print(f'MiFID: {mi_fid_score:.5f}')\n            shutil.rmtree(OUT_DIR)\n            print('Removed temporary image directory.')\n        \n        # Cosine learning rate decay\n        if (epoch + 1) % decay_step == 0:\n            new_lr_d = tf.train.cosine_decay(new_lr_d, min(global_step, lr_decay_steps), lr_decay_steps)\n            new_lr_g = tf.train.cosine_decay(new_lr_g, min(global_step, lr_decay_steps), lr_decay_steps)\n            generator_optimizer = tf.train.AdamOptimizer(learning_rate=new_lr_d, beta1=0.5)\n            discriminator_optimizer = tf.train.AdamOptimizer(learning_rate=new_lr_g, beta1=0.5)          \n\n        print('Epoch: {} computed for {} sec'.format(epoch + 1, time.time() - start))\n        print('Gen_loss mean: ', np.mean(G_loss),' std: ', np.std(G_loss))\n        print('Disc_loss mean: ', np.mean(D_loss),' std: ', np.std(D_loss))\n\n    # Generate after the final epoch\n    #display.clear_output(wait=True)\n    #final_seed = tf.random.normal([64, noise_dim])\n    generate_and_save_images(dog_generator, epochs, seed, rows=8, cols=8)\n    checkpoint.save(file_prefix = checkpoint_prefix)\n    \n    \n    OUT_DIR.mkdir(exist_ok=True)\n    save_images(OUT_DIR)\n    evaluator = MiFIDEvaluator(MODEL_PATH, TRAIN_DIR)\n    fid_value, distance, mi_fid_score = evaluator.evaluate(OUT_DIR)\n    print(f'FID: {fid_value:.5f}')\n    print(f'distance: {distance:.5f}')\n    print(f'MiFID: {mi_fid_score:.5f}')\n    shutil.rmtree(OUT_DIR)\n    print('Removed temporary image directory.')\n    \n    \n    print('Final epoch.')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:02.171715Z","iopub.execute_input":"2022-02-09T22:59:02.172078Z","iopub.status.idle":"2022-02-09T22:59:02.191197Z","shell.execute_reply.started":"2022-02-09T22:59:02.172015Z","shell.execute_reply":"2022-02-09T22:59:02.190154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\ntrain(dog_features_data, EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:03.006596Z","iopub.execute_input":"2022-02-09T22:59:03.0069Z","iopub.status.idle":"2022-02-09T22:59:03.030744Z","shell.execute_reply.started":"2022-02-09T22:59:03.00683Z","shell.execute_reply":"2022-02-09T22:59:03.029314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate a test image","metadata":{}},{"cell_type":"markdown","source":"&nbsp;&nbsp;&nbsp;We can also reuse one of the previous functions to generate a single random dog image.","metadata":{}},{"cell_type":"code","source":"checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:05.251585Z","iopub.execute_input":"2022-02-09T22:59:05.251914Z","iopub.status.idle":"2022-02-09T22:59:05.27833Z","shell.execute_reply.started":"2022-02-09T22:59:05.251855Z","shell.execute_reply":"2022-02-09T22:59:05.277075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"generate_test_image(dog_generator)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:05.909764Z","iopub.execute_input":"2022-02-09T22:59:05.910091Z","iopub.status.idle":"2022-02-09T22:59:05.932613Z","shell.execute_reply.started":"2022-02-09T22:59:05.910039Z","shell.execute_reply":"2022-02-09T22:59:05.931356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate 10000 images for submission and save them to a zip file","metadata":{}},{"cell_type":"code","source":"#zip_images()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:07.825346Z","iopub.execute_input":"2022-02-09T22:59:07.826262Z","iopub.status.idle":"2022-02-09T22:59:07.830598Z","shell.execute_reply.started":"2022-02-09T22:59:07.825694Z","shell.execute_reply":"2022-02-09T22:59:07.829516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create an animated GIF using the training images","metadata":{}},{"cell_type":"code","source":"# Display a single image using the epoch number\ndef display_image(epoch_no):\n    return Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:09.218857Z","iopub.execute_input":"2022-02-09T22:59:09.219389Z","iopub.status.idle":"2022-02-09T22:59:09.227124Z","shell.execute_reply.started":"2022-02-09T22:59:09.219161Z","shell.execute_reply":"2022-02-09T22:59:09.226121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_image(EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:10.086507Z","iopub.execute_input":"2022-02-09T22:59:10.086803Z","iopub.status.idle":"2022-02-09T22:59:10.126652Z","shell.execute_reply.started":"2022-02-09T22:59:10.086754Z","shell.execute_reply":"2022-02-09T22:59:10.125502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anim_file = 'dcgan.gif'\n\nwith imageio.get_writer(anim_file, mode='I') as writer:\n    filenames = glob.glob('image*.png')\n    filenames = sorted(filenames)\n    last = -1\n    for i,filename in enumerate(filenames):\n        frame = 1*(i**2)\n        if round(frame) > round(last):\n            last = frame\n        else:\n            continue\n        image = imageio.imread(filename)\n        writer.append_data(image)\n    image = imageio.imread(filename)\n    writer.append_data(image)\n\nimport IPython\nif IPython.version_info > (6,2,0,''):\n    IPython.display.Image(filename=anim_file)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T22:59:10.736599Z","iopub.execute_input":"2022-02-09T22:59:10.736912Z","iopub.status.idle":"2022-02-09T22:59:10.780875Z","shell.execute_reply.started":"2022-02-09T22:59:10.736857Z","shell.execute_reply":"2022-02-09T22:59:10.779783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}