{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!unzip /kaggle/input/generative-dog-images/all-dogs.zip -d input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip /kaggle/input/generative-dog-images/Annotation.zip -d input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\ntf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xml.etree.ElementTree as ET # For parsing XML\nfrom PIL import Image # to read image\nimport glob\nfrom tqdm import tqdm_notebook\nimport urllib\nimport tarfile\nimport imageio\nfrom imageio import imread, imsave, mimsave\nimport shutil\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport numpy as np\nimport os\nfrom tensorflow.keras import layers\nimport time\nfrom IPython import display","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Code slightly modified from user: cdeotte | https://www.kaggle.com/cdeotte/supervised-generative-dog-net\n\nROOT = './input/'\n# list of all image file names in all-dogs\nIMAGES = os.listdir(ROOT + 'all-dogs/')\n# list of all the annotation directories, each directory is a dog breed\nbreeds = os.listdir(ROOT + 'Annotation/') \n\nidxIn = 0; namesIn = []\nimagesIn = np.zeros((25000,64,64,3))\n\n# CROP WITH BOUNDING BOXES TO GET DOGS ONLY\n# iterate through each directory in annotation\nfor breed in breeds:\n    # iterate through each file in the directory\n    for dog in os.listdir(ROOT+'Annotation/'+breed):\n        try: img = Image.open(ROOT+'all-dogs/'+dog+'.jpg') \n        except: continue           \n        # Element Tree library allows for parsing xml and getting specific tag values    \n        tree = ET.parse(ROOT+'Annotation/'+breed+'/'+dog)\n        # take a look at the print out of an xml previously to get what is going on\n        root = tree.getroot() # <annotation>\n        objects = root.findall('object') # <object>\n        for o in objects:\n            bndbox = o.find('bndbox') # <bndbox>\n            xmin = int(bndbox.find('xmin').text) # <xmin>\n            ymin = int(bndbox.find('ymin').text) # <ymin>\n            xmax = int(bndbox.find('xmax').text) # <xmax>\n            ymax = int(bndbox.find('ymax').text) # <ymax>\n            w = np.min((xmax - xmin, ymax - ymin))\n            img2 = img.crop((xmin, ymin, xmin+w, ymin+w))\n            img2 = img2.resize((64,64), Image.ANTIALIAS)\n            imagesIn[idxIn,:,:,:] = np.asarray(img2)\n            namesIn.append(breed)\n            idxIn += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Inspect what the previous code created\nprint(\"imagesIn is a {} with {} {} by {} rgb({}) images. Shape: {}\".format(type(imagesIn), imagesIn.shape[0], imagesIn.shape[1], imagesIn.shape[2], imagesIn.shape[3], imagesIn.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(imagesIn[1].astype(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# normalize the pixel values\nimagesIn = (imagesIn[:idxIn,:,:,:]-127.5)/127.5 # Normalize the images to [-1, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(imagesIn[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this is needed because the gradient functions from TF require float32 instead of float64\nimagesIn = tf.cast(imagesIn, 'float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef flip(x: tf.Tensor) -> (tf.Tensor):\n    x = tf.image.random_flip_left_right(x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Batch and shuffle the data\nBUFFER_SIZE = 25000\nBATCH_SIZE = 128\n\ntrain_dataset = tf.data.Dataset.from_tensor_slices(imagesIn).shuffle(BUFFER_SIZE).map(flip).batch(BATCH_SIZE,  drop_remainder=True)\nprint(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weight_init_std = 0.02\nweight_init_mean = 0.0\nweight_initializer = tf.keras.initializers.TruncatedNormal(stddev=weight_init_std, mean=weight_init_mean,\n                                                          seed=42)\nlr_initial_d = tf.Variable(0.0002)\nlr_initial_g = tf.Variable(0.0002)\nlr_decay_steps = 1000\ndecay_step = 50\nnoise_dim = 128\ndropout_rate = 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef make_generator_model():\n    model = tf.keras.Sequential()\n    model.add(layers.Dense(8*8*512, use_bias=False, input_shape=(noise_dim,)))\n    model.add(layers.BatchNormalization())\n    model.add(layers.ReLU())\n    \n    model.add(layers.Reshape((8, 8, 512)))\n    assert model.output_shape == (None, 8, 8, 512)\n    model.add(layers.Dropout(dropout_rate))\n    \n    model.add(layers.Conv2DTranspose(256, (5, 5), strides=(1, 1),  padding='same', use_bias=False))\n    assert model.output_shape == (None, 8, 8, 256)\n    model.add(layers.BatchNormalization())\n    model.add(layers.ReLU())\n    model.add(layers.Dropout(dropout_rate))\n    \n    model.add(layers.Conv2DTranspose(128, (5, 5), strides=(2, 2),  padding='same', use_bias=False))\n    assert model.output_shape == (None, 16, 16, 128)\n    model.add(layers.BatchNormalization())\n    model.add(layers.ReLU())\n    \n    model.add(layers.Conv2DTranspose(64, (5, 5), strides=(2, 2),  padding='same', use_bias=False))\n    assert model.output_shape == (None, 32, 32, 64)\n    model.add(layers.BatchNormalization())\n    model.add(layers.ReLU())\n    \n    model.add(layers.Conv2DTranspose(32, (5,5), strides=(2, 2), padding='same', use_bias=False))\n    assert model.output_shape == (None, 64, 64, 32)\n    model.add(layers.BatchNormalization())\n    model.add(layers.ReLU())\n    \n    model.add(layers.Dense(3, activation='tanh', kernel_initializer=weight_initializer))\n    print(\"GENERATOR\")\n    model.summary()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generator = make_generator_model()\n\nnoise = tf.random.normal([1, noise_dim])\n\ngenerated_image = generator(noise,training=False)\n\nplt.imshow(generated_image[0, :, :, 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(generated_image.shape)\nprint(noise.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_discriminator_model():\n    model = tf.keras.Sequential()   \n    model.add(layers.Conv2D(64, (5, 5), strides=(2, 2), padding='same',\n                                     input_shape=[64, 64, 3]))   \n    model.add(layers.LeakyReLU())\n    #model.add(layers.Dropout(0.3))\n\n    model.add(layers.Conv2D(128, (5,5), \n                             strides=(2,2),\n                             padding='same'))\n    model.add(layers.LeakyReLU())\n    #model.add(layers.Dropout(0.3))\n    \n    model.add(layers.Conv2D(256, (5,5), \n                             strides=(2,2),\n                             padding='same'))\n    model.add(layers.LeakyReLU())\n    #model.add(layers.Dropout(0.3))\n\n        \n    model.add(layers.Conv2D(512, (5,5), \n                             strides=(2,2),\n                             padding='same'))\n    model.add(layers.LeakyReLU())\n    #model.add(layers.Dropout(0.3))\n    \n    \n    \n    model.add(layers.Flatten())\n    model.add(layers.Dense(1, activation='sigmoid'))\n    \n    print(\"DISCRIMINATOR\")\n    model.summary()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndiscriminator = make_discriminator_model()\ndecision = discriminator(generated_image)\nprint (decision)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This method returns a helper funciton to compute cross entropy loss\ncross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Label smoothing -- technique from GAN hacks, instead of assigning 1/0 as class labels, we assign a random integer in range [0.7, 1.0] for positive class\n# and [0.0, 0.3] for negative class\n\ndef smooth_positive_labels(y):\n    return y - 0.3 + (np.random.random(y.shape) * 0.5)\n\ndef smooth_negative_labels(y):\n    return y + np.random.random(y.shape) * 0.3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# randomly flip some labels\ndef noisy_labels(y, p_flip):\n    # determine the number of labels to flip\n    n_select = int(p_flip * int(y.shape[0]))\n    # choose labels to flip\n    flip_ix = np.random.choice([i for i in range(int(y.shape[0]))], size=n_select)\n    \n    op_list = []\n    # invert the labels in place\n    #y_np[flip_ix] = 1 - y_np[flip_ix]\n    for i in range(int(y.shape[0])):\n        if i in flip_ix:\n            op_list.append(tf.subtract(1., y[i]))\n        else:\n            op_list.append(y[i])\n    \n    outputs = tf.stack(op_list)\n    return outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def discriminator_loss(real_output, fake_output):\n    real_output_noise = noisy_labels(tf.ones_like(real_output), 0.05)\n    fake_output_noise = noisy_labels(tf.zeros_like(fake_output), 0.05)\n    real_output_smooth = smooth_positive_labels(real_output_noise)\n    fake_output_smooth = smooth_negative_labels(fake_output_noise)\n \n    real_loss = cross_entropy(tf.ones_like(real_output_smooth), real_output)\n    fake_loss = cross_entropy(tf.zeros_like(fake_output_smooth), fake_output)\n    \n    total_loss = real_loss + fake_loss\n    return total_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator_loss(fake_output):\n    fake_output_smooth = smooth_negative_labels(tf.ones_like(fake_output))\n    return cross_entropy(tf.ones_like(fake_output_smooth), fake_output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ngenerator_optimizer = tf.optimizers.Adam(learning_rate=lr_initial_g, beta_1=0.5)\ndiscriminator_optimizer = tf.optimizers.Adam(learning_rate=lr_initial_d, beta_1=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncheckpoint_dir = './training_checkpoints'\ncheckpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\ncheckpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n                                 discriminator_optimizer=discriminator_optimizer,\n                                 generator=generator,\n                                 discriminator=discriminator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 280\nnum_examples_to_generate = 64\n\n# we will reuse this seed overtime (so it's easier)\n# to visualize progress in the animated\nseed = tf.random.normal([num_examples_to_generate, noise_dim])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Notice the use of `tf.function`\n# This annotation causes the function to be \"compiled\".\n@tf.function\ndef train_step(images):\n    noise = tf.random.normal([BATCH_SIZE, noise_dim])\n\n    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n        generated_images = generator(noise, training=True)\n        real_output = discriminator(images, training=True)\n        fake_output = discriminator(generated_images, training=True)\n        \n        gen_loss = generator_loss(fake_output)\n        disc_loss = discriminator_loss(real_output, fake_output)\n\n    gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)\n    gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)\n\n    generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(dataset, epochs):\n    for epoch in tqdm_notebook(range(epochs)):\n        start = time.time()\n        new_lr_d = lr_initial_d\n        new_lr_g = lr_initial_g  \n        global_step = 0\n        \n        for image_batch in dataset:\n            train_step(image_batch)\n            \n        # Produce images for the GIF as we go\n        #display.clear_output(wait=True)\n        generate_and_save_images(generator,\n                             epoch + 1,\n                             seed)\n        \n        # Save the model every 15 epochs\n        if (epoch + 1) % 15 == 0:\n            checkpoint.save(file_prefix = checkpoint_prefix)\n            \n        # Cosine learning rate decay\n        if (epoch + 1) % decay_step == 0:\n            new_lr_d = tf.compat.v1.train.cosine_decay(new_lr_d, min(global_step, lr_decay_steps), lr_decay_steps)\n            new_lr_g = tf.compat.v1.train.cosine_decay(new_lr_g, min(global_step, lr_decay_steps), lr_decay_steps)\n            generator_optimizer = tf.optimizers.Adam(learning_rate=new_lr_d, beta_1=0.5)\n            discriminator_optimizer = tf.optimizers.Adam(learning_rate=new_lr_g, beta_1=0.5)   \n\n        #print ('Time for epoch {} is {} sec'.format(epoch + 1, time.time()-start))\n        \n    # Generate after the final epoch     \n    #display.clear_output(wait=True)\n    generate_and_save_images(generator,\n                           epochs,\n                           seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_and_save_images(model, epoch, test_input):\n    # Notice `training` is set to False.\n    # This is so all layers run in inference mode (batchnorm).\n    predictions = model(test_input, training=False)\n    \n    fig = plt.figure(figsize=(14,14))\n    for i in range(predictions.shape[0]):\n        plt.subplot(8, 8, i + 1)\n        plt.imshow((predictions[i, :, :, :] + 1.)/2.)\n        plt.axis('off')\n    plt.savefig('dcgan/image_at_epoch_{}.png'.format(epoch))\n    plt.close(fig)\n    #plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir dcgan && mkdir training_checkpoints && mkdir output_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain(train_dataset, EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}