{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is my current attempt at creating and optimizing a DCGAN using Tensorflow and Keras. My methods are mostly based on Chad Malla's DCGAN [kernel](https://www.kaggle.com/cmalla94/dcgan-generating-dog-images-with-tensorflow) and the DCGAN hacks [post](https://www.kaggle.com/c/generative-dog-images/discussion/98595). Will provide more details once I actually generate some decent results."},{"metadata":{},"cell_type":"markdown","source":"## References"},{"metadata":{},"cell_type":"markdown","source":"[1]. My previous kernel on [EDA and image preprocessing](https://www.kaggle.com/jadeblue/dog-generator-starter-eda-preprocessing).\n\n[2]. [Xml parsing and cropping to specified bounding box](https://www.kaggle.com/paulorzp/show-annotations-and-breeds).\n\n[3]. [Image cropping method with interpolation](https://www.kaggle.com/amanooo/wgan-gp-keras).\n\n[4]. [Another great Keras-based DCGAN approach](https://www.kaggle.com/cmalla94/dcgan-generating-dog-images-with-tensorflow).\n\n[5]. [DCGAN hacks for improving your model performance](https://www.kaggle.com/c/generative-dog-images/discussion/98595).\n\n[6]. [Tensorflow DCGAN tutorial](https://www.tensorflow.org/beta/tutorials/generative/dcgan)."},{"metadata":{},"cell_type":"markdown","source":"## Importing libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt, zipfile\nimport os\nimport glob\nimport math\nimport random\nimport time\nimport datetime\nfrom tqdm import tqdm, tqdm_notebook\n\nimport xml.etree.ElementTree as ET \n\nimport cv2\nfrom PIL import Image\n\nimport tensorflow as tf\nfrom tensorflow.keras.initializers import RandomNormal\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense, BatchNormalization, LeakyReLU, Reshape,\\\nConv2DTranspose, Conv2D, Flatten, Dropout, Embedding\nfrom tensorflow.keras.optimizers import Adam\n\n#from IPython import display\n\n# libraries for SpectralNorm\nfrom tensorflow.keras import backend as K\nfrom keras.engine import *\nfrom keras.legacy import interfaces\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras import constraints\nfrom keras.utils.generic_utils import func_dump\nfrom keras.utils.generic_utils import func_load\nfrom keras.utils.generic_utils import deserialize_keras_object\nfrom keras.utils.generic_utils import has_arg\nfrom keras.utils import conv_utils\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.enable_eager_execution()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting input variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_width = 64\nimage_height = 64\nimage_channels = 3\nimage_sample_size = 10000\nimage_output_dir = '../output_images/'\nimage_input_dir = '../input/all-dogs/all-dogs/'\nimage_ann_dir = \"../input/annotation/Annotation/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating the image features"},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_breed_dict = {}\nfor annotation in os.listdir(image_ann_dir):\n    annotations = annotation.split('-')\n    dog_breed_dict[annotations[0]] = annotations[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_image(src):\n    img = cv2.imread(src)\n    if img is None:\n        raise FileNotFoundError\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Crop the images and apply scaling"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_cropped_images(dog_breed_dict=dog_breed_dict, image_ann_dir=image_ann_dir, sample_size=25000, \n                        image_width=image_width, image_height=image_height, image_channels=image_channels):\n    curIdx = 0\n    breeds = []\n    dog_images_np = np.zeros((sample_size,image_width,image_height,image_channels))\n    for breed_folder in os.listdir(image_ann_dir):\n        for dog_ann in tqdm(os.listdir(image_ann_dir + breed_folder)):\n            try:\n                img = read_image(os.path.join(image_input_dir, dog_ann + '.jpg'))\n            except FileNotFoundError:\n                continue\n                \n            tree = ET.parse(os.path.join(image_ann_dir + breed_folder, dog_ann))\n            root = tree.getroot()\n            \n            size = root.find('size')\n            width = int(size.find('width').text)\n            height = int(size.find('height').text)\n            objects = root.findall('object')\n            for o in objects:\n                bndbox = o.find('bndbox') \n                \n                xmin = int(bndbox.find('xmin').text)\n                ymin = int(bndbox.find('ymin').text)\n                xmax = int(bndbox.find('xmax').text)\n                ymax = int(bndbox.find('ymax').text)\n                \n                xmin = max(0, xmin - 4)        # 4 : margin\n                xmax = min(width, xmax + 4)\n                ymin = max(0, ymin - 4)\n                ymax = min(height, ymax + 4)\n\n                w = np.min((xmax - xmin, ymax - ymin))\n                w = min(w, width, height)                     # available w\n\n                if w > xmax - xmin:\n                    xmin = min(max(0, xmin - int((w - (xmax - xmin))/2)), width - w)\n                    xmax = xmin + w\n                if w > ymax - ymin:\n                    ymin = min(max(0, ymin - int((w - (ymax - ymin))/2)), height - w)\n                    ymax = ymin + w\n                \n                img_cropped = img[ymin:ymin+w, xmin:xmin+w, :]      # [h,w,c]\n                # Interpolation method\n                if xmax - xmin > image_width:\n                    interpolation = cv2.INTER_AREA          # shrink\n                else:\n                    interpolation = cv2.INTER_CUBIC         # expansion\n                    \n                img_cropped = cv2.resize(img_cropped, (image_width, image_height), \n                                         interpolation=interpolation)  # resize\n                    \n                dog_images_np[curIdx,:,:,:] = np.asarray(img_cropped)\n                dog_breed_name = dog_breed_dict[dog_ann.split('_')[0]]\n                breeds.append(dog_breed_name)\n                curIdx += 1\n                \n    return dog_images_np, breeds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\ndog_images_np, breeds = load_cropped_images(sample_size=22125)\nest_time = round(time.time() - start_time)\nprint(\"Feature loading time: {}.\".format(str(datetime.timedelta(seconds=est_time))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Loaded features shape: ', dog_images_np.shape)\nprint('Loaded labels: ', len(breeds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_features(features, labels, image_width=image_width, image_height=image_height, \n                image_channels=image_channels,\n                examples=25, disp_labels=True): \n  \n    if not math.sqrt(examples).is_integer():\n        print('Please select a valid number of examples.')\n        return\n    \n    imgs = []\n    classes = []\n    for i in range(examples):\n        rnd_idx = np.random.randint(0, len(labels))\n        imgs.append(features[rnd_idx, :, :, :])\n        classes.append(labels[rnd_idx])\n    \n    \n    fig, axes = plt.subplots(round(math.sqrt(examples)), round(math.sqrt(examples)),figsize=(15,15),\n    subplot_kw = {'xticks':[], 'yticks':[]},\n    gridspec_kw = dict(hspace=0.3, wspace=0.01))\n    \n    for i, ax in enumerate(axes.flat):\n        if disp_labels == True:\n            ax.title.set_text(classes[i])\n        ax.imshow(imgs[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Plotting cropped images by their specified coordinates..')\nplot_features(dog_images_np / 255., breeds, examples=25, disp_labels=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Normalize the pixel values of the images"},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_images_np = (dog_images_np - 127.5) / 127.5  # normalize the pixel range to [-1, 1] ((image - 127.5) / 127.5) or [0, 1] (image / 255.) alternatively","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Plotting cropped images by their specified coordinates..')\nplot_features(dog_images_np, breeds, examples=25, disp_labels=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.max(dog_images_np[3,:,:,:]), np.min(dog_images_np[3,:,:,:]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Deprocessing back to the original values"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_features((dog_images_np * 127.5 + 127.5) / 255., breeds, examples=25, disp_labels=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tensorflow-based preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Dog features shape:\", dog_images_np.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_features_tf = tf.cast(dog_images_np, 'float32')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Set model hyperparameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_size = 22125\nbatch_size = 128\nweight_init_std = 0.02\nweight_init_mean = 0.0\nleaky_relu_slope = 0.2\ndownsize_factor = 3\ndropout_rate = 0.3\nscale_factor = 2 ** downsize_factor\nBN_MOMENTUM = 0.1\nBN_EPSILON  = 0.00002","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create tensorflow-type dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_features_data = tf.data.Dataset.from_tensor_slices(dog_features_tf).shuffle(sample_size).batch(batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dog_features_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating the generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"weight_initializer = tf.keras.initializers.RandomNormal(mean=weight_init_mean, stddev=weight_init_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DenseSN(Dense):\n    def build(self, input_shape):\n        assert len(input_shape) >= 2\n        input_dim = input_shape[-1]\n        self.kernel = self.add_weight(shape=(input_dim, self.units),\n                                      initializer=self.kernel_initializer,\n                                      name='kernel',\n                                      regularizer=self.kernel_regularizer,\n                                      constraint=self.kernel_constraint)\n        if self.use_bias:\n            self.bias = self.add_weight(shape=(self.units,),\n                                        initializer=self.bias_initializer,\n                                        name='bias',\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n        else:\n            self.bias = None\n        self.u = self.add_weight(shape=tuple([1, self.kernel.shape.as_list()[-1]]),\n                                 initializer=initializers.RandomNormal(0, 1),\n                                 name='sn',\n                                 trainable=False)\n        self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})\n        self.built = True\n        \n    def call(self, inputs, training=None):\n        def _l2normalize(v, eps=1e-12):\n            return v / (K.sum(v ** 2) ** 0.5 + eps)\n        def power_iteration(W, u):\n            _u = u\n            _v = _l2normalize(K.dot(_u, K.transpose(W)))\n            _u = _l2normalize(K.dot(_v, W))\n            return _u, _v\n        W_shape = self.kernel.shape.as_list()\n        #Flatten the Tensor\n        W_reshaped = K.reshape(self.kernel, [-1, W_shape[-1]])\n        _u, _v = power_iteration(W_reshaped, self.u)\n        #Calculate Sigma\n        sigma=K.dot(_v, W_reshaped)\n        sigma=K.dot(sigma, K.transpose(_u))\n        #normalize it\n        W_bar = W_reshaped / sigma\n        #reshape weight tensor\n        if training in {0, False}:\n            W_bar = K.reshape(W_bar, W_shape)\n        else:\n            with tf.control_dependencies([self.u.assign(_u)]):\n                 W_bar = K.reshape(W_bar, W_shape)  \n        output = K.dot(inputs, W_bar)\n        if self.use_bias:\n            output = K.bias_add(output, self.bias, data_format='channels_last')\n        if self.activation is not None:\n            output = self.activation(output)\n        return output \n    \nclass ConvSN2D(Conv2D):\n\n    def build(self, input_shape):\n        if self.data_format == 'channels_first':\n            channel_axis = 1\n        else:\n            channel_axis = -1\n        if input_shape[channel_axis] is None:\n            raise ValueError('The channel dimension of the inputs '\n                             'should be defined. Found `None`.')\n        input_dim = input_shape[channel_axis]\n        kernel_shape = self.kernel_size + (input_dim, self.filters)\n\n        self.kernel = self.add_weight(shape=kernel_shape,\n                                      initializer=self.kernel_initializer,\n                                      name='kernel',\n                                      regularizer=self.kernel_regularizer,\n                                      constraint=self.kernel_constraint)\n\n        if self.use_bias:\n            self.bias = self.add_weight(shape=(self.filters,),\n                                        initializer=self.bias_initializer,\n                                        name='bias',\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n        else:\n            self.bias = None\n            \n        self.u = self.add_weight(shape=tuple([1, self.kernel.shape.as_list()[-1]]),\n                         initializer=initializers.RandomNormal(0, 1),\n                         name='sn',\n                         trainable=False)\n        \n        # Set input spec.\n        self.input_spec = InputSpec(ndim=self.rank + 2,\n                                    axes={channel_axis: input_dim})\n        self.built = True\n    def call(self, inputs, training=None):\n        def _l2normalize(v, eps=1e-12):\n            return v / (K.sum(v ** 2) ** 0.5 + eps)\n        def power_iteration(W, u):\n            #Accroding the paper, we only need to do power iteration one time.\n            _u = u\n            _v = _l2normalize(K.dot(_u, K.transpose(W)))\n            _u = _l2normalize(K.dot(_v, W))\n            return _u, _v\n        #Spectral Normalization\n        W_shape = self.kernel.shape.as_list()\n        #Flatten the Tensor\n        W_reshaped = K.reshape(self.kernel, [-1, W_shape[-1]])\n        _u, _v = power_iteration(W_reshaped, self.u)\n        #Calculate Sigma\n        sigma=K.dot(_v, W_reshaped)\n        sigma=K.dot(sigma, K.transpose(_u))\n        #normalize it\n        W_bar = W_reshaped / sigma\n        #reshape weight tensor\n        if training in {0, False}:\n            W_bar = K.reshape(W_bar, W_shape)\n        else:\n            with tf.control_dependencies([self.u.assign(_u)]):\n                W_bar = K.reshape(W_bar, W_shape)\n                \n        outputs = K.conv2d(\n                inputs,\n                W_bar,\n                strides=self.strides,\n                padding=self.padding,\n                data_format=self.data_format,\n                dilation_rate=self.dilation_rate)\n        if self.use_bias:\n            outputs = K.bias_add(\n                outputs,\n                self.bias,\n                data_format=self.data_format)\n        if self.activation is not None:\n            return self.activation(outputs)\n        return outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transposed_conv(model, out_channels):\n    model.add(Conv2DTranspose(out_channels, (4, 4), strides=(2, 2), padding='same', \n                              kernel_initializer=weight_initializer, use_bias=False))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=leaky_relu_slope))\n    return model\n\n\ndef convSN(model, out_channels, ksize, stride_size):\n    model.add(ConvSN2D(out_channels, (ksize, ksize), strides=(stride_size, stride_size), padding='same',\n                     kernel_initializer=weight_initializer))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=leaky_relu_slope))\n    model.add(Dropout(dropout_rate))\n    return model\n\ndef conv(model, out_channels, ksize, stride_size):\n    model.add(Conv2D(out_channels, (ksize, ksize), strides=(stride_size, stride_size), padding='same',\n                     kernel_initializer=weight_initializer))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=leaky_relu_slope))\n    model.add(Dropout(dropout_rate))\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Confirm that input dimensions are correct"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(image_height // scale_factor, image_width // scale_factor, 512)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def DogGenerator():\n    model = Sequential()\n    model.add(Dense(image_width // scale_factor * image_height // scale_factor * 512,\n                    input_shape=(100,), kernel_initializer=weight_initializer, use_bias=False))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=leaky_relu_slope))\n    model.add(Reshape((image_height // scale_factor, image_width // scale_factor, 512)))\n    \n    model = transposed_conv(model, 256)\n    model = transposed_conv(model, 128)\n    model = transposed_conv(model, 64)\n    \n    model.add(Dense(3, activation='tanh', kernel_initializer=weight_initializer, use_bias=False))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_generator = DogGenerator()\nprint(dog_generator.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate points in latent space as input for the generator\ndef generate_latent_points(latent_dim, n_samples):\n    # generate points in the latent space\n    x_input = np.random.randn(latent_dim * n_samples)\n    # reshape into a batch of inputs for the network\n    x_input = x_input.reshape((n_samples, latent_dim))\n    return x_input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# random noise vector\nnoise = tf.random.normal([1,100])\n#sample = generate_latent_points(100, 50)\n# run the generator model with the noise vector as input\ngenerated_image = dog_generator(noise, training=False)\n# display output\nplt.imshow(generated_image[0, :, :, :])\nprint(generated_image.shape)\n#print(sample.shape, sample.mean(), sample.std())\nprint(noise.shape, tf.math.reduce_mean(noise).numpy(), tf.math.reduce_std(noise).numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating the discriminator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def DogDiscriminator(spectral_normalization=True):\n    model = Sequential()\n    if spectral_normalization:\n        model.add(ConvSN2D(64, (4, 4), strides=(2,2), padding='same',\n                         input_shape=[image_height, image_width, image_channels], \n                         kernel_initializer=weight_initializer))\n        model.add(BatchNormalization())\n        model.add(LeakyReLU(alpha=leaky_relu_slope))\n        model.add(Dropout(dropout_rate))\n\n        model = convSN(model, 64, ksize=4, stride_size=2)\n        model = convSN(model, 128, ksize=3, stride_size=1)\n        model = convSN(model, 128, ksize=4, stride_size=2)\n        model = convSN(model, 256, ksize=3, stride_size=1)\n        model = convSN(model, 256, ksize=4, stride_size=2)\n        model = convSN(model, 512, ksize=4, stride_size=2)\n\n        model.add(Flatten())\n        model.add(DenseSN(1, activation='sigmoid'))\n    else:\n        model.add(Conv2D(64, (4, 4), strides=(2,2), padding='same',\n                         input_shape=[image_height, image_width, image_channels], \n                         kernel_initializer=weight_initializer))\n        model.add(BatchNormalization())\n        model.add(LeakyReLU(alpha=leaky_relu_slope))\n        model.add(Dropout(dropout_rate))\n\n        model = conv(model, 64, ksize=4, stride_size=2)\n        model = conv(model, 128, ksize=3, stride_size=1)\n        model = conv(model, 128, ksize=4, stride_size=2)\n        model = conv(model, 256, ksize=3, stride_size=1)\n        model = conv(model, 256, ksize=4, stride_size=2)\n        model = conv(model, 512, ksize=4, stride_size=2)\n\n        model.add(Flatten())\n        model.add(Dense(1, activation='sigmoid'))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_discriminator = DogDiscriminator(spectral_normalization=True)\nprint(dog_discriminator.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"decision = dog_discriminator(generated_image)\nprint(decision)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Provide label smoothing (Thanks to Chad Malla)."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Label smoothing -- technique from GAN hacks, instead of assigning 1/0 as class labels, we assign a random integer in range [0.7, 1.0] for positive class\n# and [0.0, 0.3] for negative class\n\ndef smooth_positive_labels(y):\n    return y - 0.3 + (np.random.random(y.shape) * 0.5)\n\ndef smooth_negative_labels(y):\n    return y + np.random.random(y.shape) * 0.3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Randomly flip labels to introduce more noise to the discriminator"},{"metadata":{"trusted":true},"cell_type":"code","source":"# randomly flip some labels\ndef noisy_labels(y, p_flip):\n    # determine the number of labels to flip\n    n_select = int(p_flip * int(y.shape[0]))\n    # choose labels to flip\n    flip_ix = np.random.choice([i for i in range(int(y.shape[0]))], size=n_select)\n    \n    op_list = []\n    # invert the labels in place\n    #y_np[flip_ix] = 1 - y_np[flip_ix]\n    for i in range(int(y.shape[0])):\n        if i in flip_ix:\n            op_list.append(tf.subtract(1, y[i]))\n        else:\n            op_list.append(y[i])\n    \n    outputs = tf.stack(op_list)\n    return outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\n# generate 'real' class labels (1)\nn_samples = 1000\ny = np.ones((n_samples, 1))\n# flip labels with 5% probability\ny = noisy_labels(y, 0.05)\n# summarize labels\nprint(y.sum())\n \n# generate 'fake' class labels (0)\ny = np.zeros((n_samples, 1))\n# flip labels with 5% probability\ny = noisy_labels(y, 0.05)\n# summarize labels\nprint(y.sum())\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizers and loss functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"generator_optimizer = Adam(lr=0.0002, beta_1=0.5)\ndiscriminator_optimizer = Adam(lr=0.0002, beta_1=0.5)\n# This method returns a helper function to compute cross entropy loss\ncross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def discriminator_loss(real_output, fake_output, apply_label_smoothing=True, label_noise=True):\n    if label_noise and apply_label_smoothing:\n        real_output_noise = noisy_labels(tf.ones_like(real_output), 0.05)\n        fake_output_noise = noisy_labels(tf.zeros_like(fake_output), 0.05)\n        real_output_smooth = smooth_positive_labels(real_output_noise)\n        fake_output_smooth = smooth_negative_labels(fake_output_noise)\n        real_loss = cross_entropy(real_output_smooth, real_output)\n        fake_loss = cross_entropy(fake_output_smooth, fake_output)\n    elif label_noise and not apply_label_smoothing:\n        real_output_noise = noisy_labels(tf.ones_like(real_output), 0.05)\n        fake_output_noise = noisy_labels(tf.zeros_like(fake_output), 0.05)\n        real_loss = cross_entropy(real_output_noise, real_output)\n        fake_loss = cross_entropy(fake_output_noise, fake_output)\n    elif apply_label_smoothing and not label_noise:\n        real_output_smooth = smooth_positive_labels(tf.ones_like(real_output))\n        fake_output_smooth = smooth_negative_labels(tf.zeros_like(fake_output))\n        real_loss = cross_entropy(real_output_smooth, real_output)\n        fake_loss = cross_entropy(fake_output_smooth, fake_output)\n    else:\n        real_loss = cross_entropy(tf.ones_like(real_output), real_output)\n        fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)\n    total_loss = real_loss + fake_loss\n    return total_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator_loss(fake_output, apply_label_smoothing=True):\n    if apply_label_smoothing:\n        fake_output_smooth = smooth_negative_labels(tf.ones_like(fake_output))\n        return cross_entropy(fake_output_smooth, fake_output)\n    return cross_entropy(tf.ones_like(fake_output), fake_output)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define a checkpointer"},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_dir = '/training_checkpoints'\ncheckpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\ncheckpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n                                 discriminator_optimizer=discriminator_optimizer,\n                                 generator=dog_generator,\n                                 discriminator=dog_discriminator)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Main training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 250\nnoise_dim = 100\nnum_examples_to_generate = 8\nseed = tf.random.normal([num_examples_to_generate, noise_dim])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_step(images, G_loss, D_loss):\n    noise = tf.random.normal([batch_size, noise_dim])\n\n    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n        generated_images = dog_generator(noise, training=True)\n        \n        real_output = dog_discriminator(images, training=True)\n        fake_output = dog_discriminator(generated_images, training=True)\n        \n        gen_loss = generator_loss(fake_output, apply_label_smoothing=False)\n        disc_loss = discriminator_loss(real_output, fake_output, apply_label_smoothing=False, label_noise=False)\n\n    G_loss.append(gen_loss.numpy())\n    D_loss.append(disc_loss.numpy())\n    \n    gradients_of_generator = gen_tape.gradient(gen_loss, dog_generator.trainable_variables)\n    gradients_of_discriminator = disc_tape.gradient(disc_loss, dog_discriminator.trainable_variables)\n\n    generator_optimizer.apply_gradients(zip(gradients_of_generator, dog_generator.trainable_variables))\n    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, dog_discriminator.trainable_variables))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function by Nanashi\ndef plot_loss(G_losses, D_losses, epoch):\n    plt.figure(figsize=(10,5))\n    plt.title(\"Generator and Discriminator Loss - EPOCH {}\".format(epoch))\n    plt.plot(G_losses,label=\"G\")\n    plt.plot(D_losses,label=\"D\")\n    plt.xlabel(\"Iterations\")\n    plt.ylabel(\"Loss\")\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_and_save_images(model, epoch, test_input):\n    # Notice `training` is set to False.\n    # This is so all layers run in inference mode (batchnorm).\n    predictions = model(test_input, training=False)\n    fig = plt.figure(figsize=(10,10))\n    for i in range(predictions.shape[0]):\n        plt.subplot(4, 2, i+1)\n        plt.imshow((predictions[i, :, :, :] * 127.5 + 127.5) / 255.)\n        plt.axis('off') \n    #plt.savefig('image_at_epoch_{}.png'.format(epoch))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_test_image(model, noise_dim=100):\n    test_input = tf.random.normal([1, noise_dim])\n    # Notice `training` is set to False.\n    # This is so all layers run in inference mode (batchnorm).\n    predictions = model(test_input, training=False)\n    fig = plt.figure(figsize=(5,5))\n    plt.imshow((predictions[0, :, :, :] * 127.5 + 127.5) / 255.)\n    plt.axis('off') \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(dataset, epochs):\n    G_loss = []\n    D_loss = []\n    for epoch in tqdm(range(epochs)):\n        \n        start = time.time()\n        for image_batch in dataset:\n            train_step(image_batch, G_loss, D_loss)\n         \n        #display.clear_output(wait=True)\n        if (epoch + 1) % 25 == 0:\n            generate_and_save_images(dog_generator, epoch + 1, seed)\n            plot_loss(G_loss, D_loss, epoch + 1)\n        \n        G_loss = []\n        D_loss = []           \n\n        print ('Epoch: {} computed for {} sec'.format(epoch + 1, time.time() - start))\n\n    # Generate after the final epoch\n    #display.clear_output(wait=True)\n    generate_and_save_images(dog_generator, epochs, seed)\n    checkpoint.save(file_prefix = checkpoint_prefix)\n    print('Final epoch.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\ntrain(dog_features_data, EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate a test image"},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_test_image(dog_generator)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate 10000 images for submission and save them to a zip file"},{"metadata":{"trusted":true},"cell_type":"code","source":"# SAVE TO ZIP FILE NAMED IMAGES.ZIP\nz = zipfile.PyZipFile('images.zip', mode='w')\nfor k in tqdm(range(image_sample_size)):\n    generated_image = dog_generator(tf.random.normal([1, noise_dim]), training=False)\n    f = str(k)+'.png'\n    img = np.array(generated_image)\n    img = (img[0, :, :, :] + 1.) / 2.\n    img = Image.fromarray((255*img).astype('uint8').reshape((image_height,image_width,image_channels)))\n    \n    #cv2.imwrite(f, img)\n    img.save(f,'PNG')\n    z.write(f)\n    os.remove(f)\n    #if k % 1000==0: print(k)\nz.close()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}