{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a submission to the \"Generative Dogs Images\" competition. However, I am not in the running to win the competition as I am writing out this kernel to better understand GAN and its application.\n\nI will be using DCGAN (https://arxiv.org/abs/1511.06434) as the basic model and learn how to modify it."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import packages\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport os\nimport pickle as pkl\n\nimport xml.etree.ElementTree as ET \nfrom PIL import Image \n\nfrom keras.preprocessing.image import load_img\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# List of images\nimages_folder = \"../input/all-dogs/all-dogs\"\nannotation_folder = r'../input/annotation/Annotation'\nfilenames = os.listdir(images_folder)\nbreeds = os.listdir(annotation_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load images as array (64x64x3)\n# Credit goes to : https://www.kaggle.com/paulorzp/show-annotations-and-breeds\ncrop_dogspic = True\nidx = 0\nnames = []\nimg_height, img_width = 64, 64\nimages = np.zeros((len(filenames), img_height, img_width, 3)) # No of samples x image_height x image_width x no of channels\n\n\nfor breed in breeds:\n\tpath = os.path.join(annotation_folder, breed)\n\tfor dog in os.listdir(path):\n\t\tif crop_dogspic:\n\t\t\t# Load image\n\t\t\ttry: \n\t\t\t\timage_path = os.path.join(images_folder, dog + r\".jpg\")\n\t\t\t\timg = Image.open(image_path)\n\n\t\t\t\t# Extract the bounding box \n\t\t\t\tannotation_path = os.path.join(annotation_folder, breed, dog)\n\t\t\t\ttree = ET.parse(annotation_path)\n\t\t\t\tobjects = tree.getroot().findall(\"object\")\n\n\t\t\t\tfor obj in objects:\n\t\t\t\t\t# For each object, get the bounding box coordinates\n\t\t\t\t\tbndbox = obj.find(\"bndbox\")\n\t\t\t\t\txmin = int(bndbox.find(\"xmin\").text)\n\t\t\t\t\tymin = int(bndbox.find(\"ymin\").text)\n\t\t\t\t\txmax = int(bndbox.find(\"xmax\").text)\n\t\t\t\t\tymax = int(bndbox.find(\"ymax\").text)\n\n\t\t\t\t\t# Calculate the minimum difference for cropping\n\t\t\t\t\tmin_diff = np.min((xmax - xmin, ymax - ymin))\n\n\t\t\t\t\t# Crop image\n\t\t\t\t\timg_cropped = img.crop((xmin, ymin, xmin + min_diff, ymin + min_diff))\n\t\t\t\t\timg_cropped = img_cropped.resize((img_height, img_width))\n\n\t\t\t\t\t# Save details as array\n\t\t\t\t\timages[idx, :, :, :] = np.asarray(img_cropped) # Images as array of number\n\t\t\t\t\tnames.append(breed) # Category (breed in this case)\n\n\t\t\t\t\tidx += 1\n\t\t\texcept: \n\t\t\t\tpass\n\t\telse:\n\t\t\ttry:\n\t\t\t\timage_path = os.path.join(images_folder, dog + r\".jpg\")\n\t\t\t\timg = Image.open(image_path)\n\t\t\t\timg = img.resize((img_height, img_width))\n\n\t\t\t\t# Save to array\n\t\t\t\timages[idx, :, :, :] = np.asarray(img)\n\t\t\t\tnames.append(breed)\n\t\t\t\tidx += 1\n\t\t\texcept:\n\t\t\t\tpass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Shuffle the dataset\nidx = np.arange(idx)\nnp.random.shuffle(idx)\nimages = images[idx, :, :, :]\nnames = np.array(names)[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display images\nsample_idx = np.random.randint(0, len(idx), 25)\nfig, axes = plt.subplots(5, 5, figsize = (12, 12))\nfor ii, ax in zip(sample_idx, axes.flatten()):\n\timg = Image.fromarray(images[ii, :, :, :].astype(\"uint8\"))\n\tax.imshow(img)\n\tax.xaxis.set_visible(False)\n\tax.yaxis.set_visible(False)\n\tax.title.set_text(names[ii].split(\"-\")[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Network inputs\ndef model_inputs(real_dim, z_dim):\n\tinputs_real = tf.placeholder(tf.float32, (None, *real_dim), name = \"input_real\")\n\tinputs_z = tf.placeholder(tf.float32, (None, z_dim), name = \"input_z\")\n\n\treturn inputs_real, inputs_z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generator network\ndef generator(z, output_dim, reuse = False, alpha = 0.2, training = True, initializer = tf.random_normal_initializer(0, 0.02)):\n\twith tf.variable_scope(\"generator\", reuse = reuse):\n\t\t# First fully connected layer\n\t\tx1 = tf.layers.dense(inputs = z, units = 4*4*1024, kernel_initializer = initializer)\n\t\tx1 = tf.reshape(x1, (-1, 4, 4, 1024))\n\t\tx1 = tf.layers.batch_normalization(x1, training = training)\n\t\tx1 = tf.maximum(alpha*x1, x1) # Shape = 4x4x1024 \n\t\t# Dropout line here\n\n\t\t# Layer 2\n\t\tx2 = tf.layers.conv2d_transpose(inputs = x1, filters = 512, kernel_size = 5, strides = 2, padding = \"same\", kernel_initializer = initializer)\n\t\tx2 = tf.layers.batch_normalization(x2, training = training)\n\t\tx2 = tf.maximum(alpha*x2, x2) # 8x8x512 \n\t\t# Dropout line here\n\n\t\t# Layer 3\n\t\tx3 = tf.layers.conv2d_transpose(inputs = x2, filters = 256, kernel_size = 5, strides = 2, padding = \"same\", kernel_initializer = initializer)\n\t\tx3 = tf.layers.batch_normalization(x3, training = training)\n\t\tx3 = tf.maximum(alpha*x3, x3) # 16x16x256\n\t\t# Dropout line here\n\n\t\t# Layer 4\n\t\tx4 = tf.layers.conv2d_transpose(inputs = x3, filters = 128, kernel_size = 5, strides = 2, padding = \"same\", kernel_initializer = initializer)\n\t\tx4 = tf.layers.batch_normalization(x4, training = training)\n\t\tx4 = tf.maximum(alpha*x4, x4) # 32x32x128\n\t\t# Dropout line here\n\n\t\t# Output layer\n\t\tlogits = tf.layers.conv2d_transpose(inputs = x4, filters = output_dim, kernel_size = 5, strides = 2, padding = \"same\", kernel_initializer = initializer)\n\t\tout = tf.tanh(logits) # 64x64x3\n\n\t\treturn out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Discriminator network\ndef discriminator(x, reuse = False, alpha = 0.2, initializer = tf.random_normal_initializer(0, 0.02)):\n\twith tf.variable_scope(\"discriminator\", reuse = reuse):\n\t\t# Input layer is 64x64x3\n\t\tx1 = tf.layers.conv2d(inputs = x, filters = 64, kernel_size = 5, strides = 2, padding = \"same\", kernel_initializer = initializer)\n\t\trelu1 = tf.maximum(alpha*x1, x1) # 32x32x64\n\t\t# Dropout line here\n\n\t\t# Layer 2\n\t\tx2 = tf.layers.conv2d(inputs = relu1, filters = 128, kernel_size = 5, strides = 2, padding = \"same\", kernel_initializer = initializer)\n\t\tbn2 = tf.layers.batch_normalization(inputs = x2, training = True)\n\t\trelu2 = tf.maximum(alpha*bn2, bn2) # 16x16x128\n\t\t# Dropout line here\n\n\t\t# Layer 3\n\t\tx3 = tf.layers.conv2d(inputs = relu2, filters = 256, kernel_size = 5, strides = 2, padding = \"same\", kernel_initializer = initializer)\n\t\tbn3 = tf.layers.batch_normalization(inputs = x3, training = True)\n\t\trelu3 = tf.maximum(alpha*bn3, bn3) # 8x8x256\n\t\t# Dropout line here\n\n\t\t# Flatten it\n\t\tflat = tf.reshape(relu3, (-1, 8*8*256))\n\t\tlogits = tf.layers.dense(inputs = flat, units = 1, kernel_initializer = initializer)\n\t\tout = tf.sigmoid(logits)\n\n\t\treturn out, logits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model loss function\ndef model_loss(input_real, input_z, output_dim, alpha = 0.2, smooth = 0.1, initializer = tf.random_normal_initializer(0, 0.02), drop_threshold = 0.2):\n\t\"\"\"\n\tGet the loss for discriminator and generator\n\t:param input_real: images from the real dataset\n\t:param input_z: Z input\n\t:param out_channel_dim: the number of channels in the output image\n\t:param alpha: alpha in Leaky ReLU\n\t:param smooth: range (0.0, 1.0) - used for label smoothing\n\t:param initializer: kernel initializer for both generator and discriminator network\n\t:param drop_threshold: threshold probability for uniform distribution that converts some of the labels of real images from 1 to 0\n\t:return: A tuple of (discriminator loss, generator loss)\n\t\"\"\"\n\n\tg_model = generator(input_z, output_dim, alpha = alpha, initializer = initializer) # Fake input to discriminator\n\td_model_real, d_logits_real = discriminator(input_real, alpha = alpha, initializer = initializer) # Training on real images\n\td_model_fake, d_logits_fake = discriminator(g_model, alpha = alpha, reuse = True, initializer = initializer) # Training on fake images\n\n\t#d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = d_logits_real, labels = tf.ones_like(d_model_real)*(1 - smooth)))\n\trandom_drop = tf.cast(tf.random.uniform(shape = tf.shape(d_model_real), minval = 0.0, maxval = 1.0) >= drop_threshold, tf.float32)\n\td_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = d_logits_real, labels = tf.ones_like(d_model_real)*random_drop*(1 - smooth)))\n\td_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = d_logits_fake, labels = tf.zeros_like(d_model_fake)))\n\tg_loss \t\t= tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = d_logits_fake, labels = tf.ones_like(d_model_fake)))\n\n\td_loss = d_loss_real + d_loss_fake\n\n\treturn d_loss, g_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Optimizers\ndef model_opt(d_loss, g_loss, learning_rate, beta1):\n\t\"\"\"\n\tGet optimization operations\n\t:param d_loss: Discriminator loss Tensor\n\t:param g_loss: Generator loss Tensor\n\t:param learning_rate: Learning Rate Placeholder\n\t:param betq: The exponential decay rate for the 1st moment in the optimizer\n\t:return: A tuple of (discriminator training operation, generator training operation)\n\t\"\"\"\n\n\t# Get weights and bias to update\n\tt_vars = tf.trainable_variables()\n\td_vars = [var for var in t_vars if var.name.startswith(\"discriminator\")]\n\tg_vars = [var for var in t_vars if var.name.startswith(\"generator\")]\n\n\t# Optimize\n\twith tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):\n\t\td_train_opt = tf.train.AdamOptimizer(learning_rate, beta1 = beta1).minimize(d_loss, var_list = d_vars)\n\t\tg_train_opt = tf.train.AdamOptimizer(learning_rate, beta1 = beta1).minimize(g_loss, var_list = g_vars)\n\n\treturn d_train_opt, g_train_opt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Building the model\nclass GAN:\n\tdef __init__(self, real_size, z_size, learning_rate, alpha = 0.2, beta1 = 0.5, smooth = 0.1, initializer = tf.random_normal_initializer(0, 0.02), drop_threshold = 0.2):\n\t\t\"\"\"\n\t\t:param real_size: real image dimension (H, W, channels)\n\t\t:param z_size: Latent space dimension (integer)\n\t\t:param learning_rate: Learning rate for optimizer\n\t\t:param alpha: Leaky ReLU parameter\n\t\t:param beta1: The exponential decay rate for the 1st moment in the optimizer\n\t\t:param smooth: Label smoothing, range (0.0, 1.0)\n\t\t:param initializer: kernel initializer for the generator and discrminator network\n\t\t\"\"\"\n\n\t\ttf.reset_default_graph()\n\t\tself.input_real, self.input_z = model_inputs(real_size, z_size)\n\t\tself.d_loss, self.g_loss = model_loss(self.input_real, self.input_z, real_size[2], alpha = alpha, smooth = smooth, initializer = initializer, drop_threshold = drop_threshold)\n\t\tself.d_opt, self.g_opt = model_opt(self.d_loss, self.g_loss, learning_rate, beta1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to display generated images\ndef view_samples(epoch, samples, nrows, ncols, figsize = (10, 10)):\n\tfig, axes = plt.subplots(figsize = figsize, nrows = nrows, ncols = ncols, sharey = True, sharex = True)\n\n\tfor ax, img in zip(axes.flatten(), samples[epoch]):\n\t\tax.axis(\"off\")\n\t\timg = ((img - img.min())*255 / (img.max() - img.min())).astype(np.uint8)\n\t\tax.set_adjustable(\"box-forced\")\n\t\tim = ax.imshow(img, aspect = \"equal\")\n\n\tplt.subplots_adjust(wspace = 0, hspace = 0)\n\treturn fig, axes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data preprocessing\n# Rescale to [-1, 1] as output of our generator is in that range\ndef scale(x, feature_range = (-1, 1)):\n\t# scale to (0, 1)\n\tx = ((x - x.min())/(255 - x.min()))\n\n\t# scale to feature_range\n\tmin, max = feature_range\n\tx = x*(max - min) + min\n\treturn x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset:\n\t# initialize the attributes of the class\n\tdef __init__(self, images, names, val_frac = 0.5, shuffle = False, scale_func = None):\n\t\t# Split the filenames into train and valid set\n\t\tsplit_idx = int(len(names)*(1 - val_frac))\n\t\tself.train_x, self.valid_x = images[:split_idx, :, :, :], images[split_idx:, :, :, :] # Split the images\n\t\tself.train_y, self.valid_y = names[:split_idx], names[split_idx:] # Split the names accordingly\n\t\tself.shuffle = shuffle\n\n\t\t# Scaling function\n\t\tif scale_func is None:\n\t\t\tself.scaler = scale\n\t\telse:\n\t\t\tself.scaler = scale_func\n\n\n\tdef batches(self, batch_size):\n\t\t# Shuffle if True\n\t\tif self.shuffle:\n\t\t\tindex = np.arange(len(self.train_x))\n\t\t\tnp.random.shuffle(index)\n\t\t\tself.train_x = self.train_x[index]\n\t\t\tself.train_y = self.train_y[index]\n\n\t\t# Provide the minibatch\n\t\tn_batches = len(self.train_y)//batch_size\n\t\tfor ii in range(0, len(self.train_y), batch_size):\n\t\t\tx = self.train_x[ii: ii + batch_size]\n\t\t\ty = self.train_y[ii: ii + batch_size]\n\n\t\t\tyield self.scaler(x), y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to train the network\ndef train(net, dataset, batch_size, epochs, figsize = (12, 12), print_every = 10, show_every = 100):\n#\tsaver = tf.train.Saver()\n#\tsample_z = np.random.uniform(-1, 1, size = (72, z_size))\n\tsample_z = np.random.normal(size = (72, z_size))\n\n\tsamples, losses = [], []\n\tsteps = 0\n\n\twith tf.Session() as sess:\n\t\tsess.run(tf.global_variables_initializer())\n\t\tfor e in range(epochs):\n\t\t\tfor x, y in dataset.batches(batch_size):\n\t\t\t\tsteps += 1\n\n\t\t\t\t# Sample random noise for G\n#\t\t\t\tbatch_z = np.random.uniform(-1, 1, size = (batch_size, z_size))\n\t\t\t\tbatch_z = np.random.normal(size = (batch_size, z_size))\n\n\t\t\t\t# Run optimizers\n\t\t\t\t_ = sess.run(net.d_opt, feed_dict = {net.input_real: x, net.input_z: batch_z})\n\t\t\t\t_ = sess.run(net.g_opt, feed_dict = {net.input_z: batch_z, net.input_real: x})\n\n\n\t\t\t\t# Calculate the generative and discriminator model loss after every n1 steps\n\t\t\t\tif steps % print_every == 0:\n\t\t\t\t\t# At the end of each epoch, get the losses and print them out\n\t\t\t\t\ttrain_loss_d = net.d_loss.eval({net.input_z: batch_z, net.input_real: x})\n\t\t\t\t\ttrain_loss_g = net.g_loss.eval({net.input_z: batch_z})\n\n\t\t\t\t\tprint(\"Epoch {}/{}...\".format(e+1, epochs),\n\t\t\t\t\t\t\"Step {}\".format(steps),\n\t\t\t\t\t\t\"Discriminator Loss: {:.4f}...\".format(train_loss_d),\n\t\t\t\t\t\t\"Generator Loss: {:.4f}\".format(train_loss_g))\n\n\t\t\t\t\t# Save losses to view after training\n\t\t\t\t\tlosses.append((train_loss_d, train_loss_g))\n\n\t\t\t\t# Show the generated picture after every n2 steps\n\t\t\t\tif steps % show_every == 0:\n\t\t\t\t\tgen_samples = sess.run(generator(net.input_z, 3, reuse = True, training = False), feed_dict = {net.input_z: sample_z})\n\t\t\t\t\tsamples.append(gen_samples)\n\t\t\t\t\t_ = view_samples(-1, samples, 1, 6, figsize = figsize)\n\t\t\t\t\tplt.show()\n\n#\t\tsaver.save(sess, \"./checkpoints/dcgan_generator.ckpt\")\n\n#\twith open(\"samples.pkl\", \"wb\") as f:\n#\t\tpkl.dump(samples, f)\n\n\treturn losses, samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyper parameters\nreal_size = (64, 64, 3)\nz_size = 100\nlearning_rate = 0.0002\nbatch_size = 128\nepochs = 70\nalpha = 0.2\nbeta1 = 0.5\nsmooth = 0.1\ndrop_threshold = 0.1\ninitializer = tf.random_normal_initializer(0, 0.02)\n\n# Create the network\nnet = GAN(real_size, z_size, learning_rate, alpha = alpha, beta1 = beta1, smooth = smooth, initializer = initializer, drop_threshold = drop_threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training\ndataset = Dataset(images, names, val_frac = 0.3, shuffle = True)\nlosses, samples = train(net, dataset, batch_size, epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualization of training losses\nfig, ax = plt.subplots()\nlosses = np.array(losses)\nplt.plot(losses.T[0], label = \"Discriminator\", alpha = 0.5)\nplt.plot(losses.T[1], label = \"Generator\", alpha = 0.5)\nplt.ylim([0, 4])\nplt.title(\"Training losses\")\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = view_samples(-1, samples, 4, 4, figsize = (7, 7))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}