{"cells":[{"metadata":{},"cell_type":"markdown","source":"# GAN Basic using MNIST\nreference: https://github.com/udacity/deep-learning/tree/master/gan_mnist"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport pickle as pkl\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.examples.tutorials.mnist import input_data\nmnist = input_data.read_data_sets('MNIST_data')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Inputs"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_inputs(real_dim, z_dim):\n    inputs_real = tf.placeholder(tf.float32, (None, real_dim), name='input_real') \n    inputs_z = tf.placeholder(tf.float32, (None, z_dim), name='input_z')\n    \n    return inputs_real, inputs_z","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generator network"},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator(z, out_dim, n_units=128, reuse=False, alpha=0.01):\n    with tf.variable_scope('generator', reuse=reuse):\n        # Hidden layer\n        h1 = tf.layers.dense(z, n_units, activation=None)\n        # Leaky ReLU\n        h1 = tf.maximum(alpha * h1, h1)\n        \n        # Logits and tanh output\n        logits = tf.layers.dense(h1, out_dim, activation=None)\n        out = tf.tanh(logits)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Discriminator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def discriminator(x, n_units=128, reuse=False, alpha=0.01):\n    with tf.variable_scope('discriminator', reuse=reuse):\n        # Hidden layer\n        h1 = tf.layers.dense(x, n_units, activation=None)\n        # Leaky ReLU\n        h1 = tf.maximum(alpha * h1, h1)\n        \n        logits = tf.layers.dense(h1, 1, activation=None)\n        out = tf.sigmoid(logits)\n        \n        return out, logits","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Size of input image to discriminator\ninput_size = 784\n# Size of latent vector to generator\nz_size = 100\n# Sizes of hidden layers in generator and discriminator\ng_hidden_size = 128\nd_hidden_size = 128\n# Leak factor for leaky ReLU\nalpha = 0.01\n# Smoothing \nsmooth = 0.1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build network"},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.reset_default_graph()\n# Create our input placeholders\ninput_real, input_z = model_inputs(input_size, z_size)\n\n# Build the model\ng_model = generator(input_z, input_size, n_units=g_hidden_size, alpha=alpha)\n# g_model is the generator output\n\nd_model_real, d_logits_real = discriminator(input_real, n_units=d_hidden_size, alpha=alpha)\nd_model_fake, d_logits_fake = discriminator(g_model, reuse=True, n_units=d_hidden_size, alpha=alpha)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Discriminator and Generator Losses"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate losses\nd_loss_real = tf.reduce_mean(\n                  tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_real, \n                                                          labels=tf.ones_like(d_logits_real) * (1 - smooth)))\nd_loss_fake = tf.reduce_mean(\n                  tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake, \n                                                          labels=tf.zeros_like(d_logits_real)))\nd_loss = d_loss_real + d_loss_fake\n\ng_loss = tf.reduce_mean(\n             tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake,\n                                                     labels=tf.ones_like(d_logits_fake)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimizers"},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate = 0.002\n\n# Get the trainable_variables, split into G and D parts\nt_vars = tf.trainable_variables()\ng_vars = [var for var in t_vars if var.name.startswith('generator')]\nd_vars = [var for var in t_vars if var.name.startswith('discriminator')]\n\nd_train_opt = tf.train.AdamOptimizer(learning_rate).minimize(d_loss, var_list=d_vars)\ng_train_opt = tf.train.AdamOptimizer(learning_rate).minimize(g_loss, var_list=g_vars)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 100\nepochs = 100\nsamples = []\nlosses = []\n# Only save generator variables\nsaver = tf.train.Saver(var_list=g_vars)\nwith tf.Session() as sess:\n    sess.run(tf.global_variables_initializer())\n    for e in range(epochs):\n        for ii in range(mnist.train.num_examples//batch_size):\n            batch = mnist.train.next_batch(batch_size)\n            \n            # Get images, reshape and rescale to pass to D\n            batch_images = batch[0].reshape((batch_size, 784))\n            batch_images = batch_images*2 - 1\n            \n            # Sample random noise for G\n            batch_z = np.random.uniform(-1, 1, size=(batch_size, z_size))\n            \n            # Run optimizers\n            _ = sess.run(d_train_opt, feed_dict={input_real: batch_images, input_z: batch_z})\n            _ = sess.run(g_train_opt, feed_dict={input_z: batch_z})\n        \n        # At the end of each epoch, get the losses and print them out\n        train_loss_d = sess.run(d_loss, {input_z: batch_z, input_real: batch_images})\n        train_loss_g = g_loss.eval({input_z: batch_z})\n            \n        print(\"Epoch {}/{}...\".format(e+1, epochs),\n              \"Discriminator Loss: {:.4f}...\".format(train_loss_d),\n              \"Generator Loss: {:.4f}\".format(train_loss_g))    \n        # Save losses to view after training\n        losses.append((train_loss_d, train_loss_g))\n        \n        # Sample from generator as we're training for viewing afterwards\n        sample_z = np.random.uniform(-1, 1, size=(16, z_size))\n        gen_samples = sess.run(\n                       generator(input_z, input_size, n_units=g_hidden_size, reuse=True, alpha=alpha),\n                       feed_dict={input_z: sample_z})\n        samples.append(gen_samples)\n        saver.save(sess, './checkpoints/generator.ckpt')\n\n# Save training generator samples\nwith open('train_samples.pkl', 'wb') as f:\n    pkl.dump(samples, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots()\nlosses = np.array(losses)\nplt.plot(losses.T[0], label='Discriminator')\nplt.plot(losses.T[1], label='Generator')\nplt.title(\"Training Losses\")\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generator samples from training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_samples(epoch, samples):\n    fig, axes = plt.subplots(figsize=(7,7), nrows=4, ncols=4, sharey=True, sharex=True)\n    for ax, img in zip(axes.flatten(), samples[epoch]):\n        ax.xaxis.set_visible(False)\n        ax.yaxis.set_visible(False)\n        im = ax.imshow(img.reshape((28,28)), cmap='Greys_r')\n    \n    return fig, axes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load samples from generator taken while training\nwith open('train_samples.pkl', 'rb') as f:\n    samples = pkl.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = view_samples(-1, samples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rows, cols = 10, 6\nfig, axes = plt.subplots(figsize=(7,12), nrows=rows, ncols=cols, sharex=True, sharey=True)\n\nfor sample, ax_row in zip(samples[::int(len(samples)/rows)], axes):\n    for img, ax in zip(sample[::int(len(sample)/cols)], ax_row):\n        ax.imshow(img.reshape((28,28)), cmap='Greys_r')\n        ax.xaxis.set_visible(False)\n        ax.yaxis.set_visible(False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sampling from the generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"saver = tf.train.Saver(var_list=g_vars)\nwith tf.Session() as sess:\n    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n    sample_z = np.random.uniform(-1, 1, size=(16, z_size))\n    gen_samples = sess.run(\n                   generator(input_z, input_size, n_units=g_hidden_size, reuse=True, alpha=alpha),\n                   feed_dict={input_z: sample_z})\n_ = view_samples(0, [gen_samples])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}