{"cells":[{"metadata":{},"cell_type":"markdown","source":"# DCGAN Basic using Street View House Numbers (SVHN) dataset\nreference: https://github.com/udacity/deep-learning/tree/master/dcgan-svhn"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport pickle as pkl\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom scipy.io import loadmat\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Getting the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from urllib.request import urlretrieve\nfrom os.path import isfile, isdir\nfrom tqdm import tqdm\n\ndata_dir = 'data/'\n\nif not isdir(data_dir):\n    raise Exception(\"Data directory doesn't exist!\")\n\nclass DLProgress(tqdm):\n    last_block = 0\n\n    def hook(self, block_num=1, block_size=1, total_size=None):\n        self.total = total_size\n        self.update((block_num - self.last_block) * block_size)\n        self.last_block = block_num\n\nif not isfile(data_dir + \"train_32x32.mat\"):\n    with DLProgress(unit='B', unit_scale=True, miniters=1, desc='SVHN Training Set') as pbar:\n        urlretrieve(\n            'http://ufldl.stanford.edu/housenumbers/train_32x32.mat',\n            data_dir + 'train_32x32.mat',\n            pbar.hook)\n\nif not isfile(data_dir + \"test_32x32.mat\"):\n    with DLProgress(unit='B', unit_scale=True, miniters=1, desc='SVHN Testing Set') as pbar:\n        urlretrieve(\n            'http://ufldl.stanford.edu/housenumbers/test_32x32.mat',\n            data_dir + 'test_32x32.mat',\n            pbar.hook)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainset = loadmat(data_dir + 'train_32x32.mat')\ntestset = loadmat(data_dir + 'test_32x32.mat')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = np.random.randint(0, trainset['X'].shape[3], size=36)\nfig, axes = plt.subplots(6, 6, sharex=True, sharey=True, figsize=(5,5),)\nfor ii, ax in zip(idx, axes.flatten()):\n    ax.imshow(trainset['X'][:,:,:,ii], aspect='equal')\n    ax.xaxis.set_visible(False)\n    ax.yaxis.set_visible(False)\nplt.subplots_adjust(wspace=0, hspace=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def scale(x, feature_range=(-1, 1)):\n    # scale to (0, 1)\n    x = ((x - x.min())/(255 - x.min()))\n    \n    # scale to feature_range\n    min, max = feature_range\n    x = x * (max - min) + min\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset:\n    def __init__(self, train, test, val_frac=0.5, shuffle=False, scale_func=None):\n        split_idx = int(len(test['y'])*(1 - val_frac))\n        self.test_x, self.valid_x = test['X'][:,:,:,:split_idx], test['X'][:,:,:,split_idx:]\n        self.test_y, self.valid_y = test['y'][:split_idx], test['y'][split_idx:]\n        self.train_x, self.train_y = train['X'], train['y']\n        \n        self.train_x = np.rollaxis(self.train_x, 3)\n        self.valid_x = np.rollaxis(self.valid_x, 3)\n        self.test_x = np.rollaxis(self.test_x, 3)\n        \n        if scale_func is None:\n            self.scaler = scale\n        else:\n            self.scaler = scale_func\n        self.shuffle = shuffle\n        \n    def batches(self, batch_size):\n        if self.shuffle:\n            idx = np.arange(len(self.train_x))\n            np.random.shuffle(idx)\n            self.train_x = self.train_x[idx]\n            self.train_y = self.train_y[idx]\n        \n        n_batches = len(self.train_y)//batch_size\n        for ii in range(0, len(self.train_y), batch_size):\n            x = self.train_x[ii:ii+batch_size]\n            y = self.train_y[ii:ii+batch_size]\n            \n            yield self.scaler(x), y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Network Inputs"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_inputs(real_dim, z_dim):\n    inputs_real = tf.placeholder(tf.float32, (None, *real_dim), name='input_real')\n    inputs_z = tf.placeholder(tf.float32, (None, z_dim), name='input_z')\n    \n    return inputs_real, inputs_z","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator(z, output_dim, reuse=False, alpha=0.2, training=True):\n    with tf.variable_scope('generator', reuse=reuse):\n        # First fully connected layer\n        x1 = tf.layers.dense(z, 4*4*512)\n        # Reshape it to start the convolutional stack\n        x1 = tf.reshape(x1, (-1, 4, 4, 512))\n        x1 = tf.layers.batch_normalization(x1, training=training)\n        x1 = tf.maximum(alpha * x1, x1)\n        # 4x4x512 now\n        \n        x2 = tf.layers.conv2d_transpose(x1, 256, 5, strides=2, padding='same')\n        x2 = tf.layers.batch_normalization(x2, training=training)\n        x2 = tf.maximum(alpha * x2, x2)\n        # 8x8x256 now\n        \n        x3 = tf.layers.conv2d_transpose(x2, 128, 5, strides=2, padding='same')\n        x3 = tf.layers.batch_normalization(x3, training=training)\n        x3 = tf.maximum(alpha * x3, x3)\n        # 16x16x128 now\n        \n        # Output layer\n        logits = tf.layers.conv2d_transpose(x3, output_dim, 5, strides=2, padding='same')\n        # 32x32x3 now\n        \n        out = tf.tanh(logits)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Discriminator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def discriminator(x, reuse=False, alpha=0.2):\n    with tf.variable_scope('discriminator', reuse=reuse):\n        # Input layer is 32x32x3\n        x1 = tf.layers.conv2d(x, 64, 5, strides=2, padding='same')\n        relu1 = tf.maximum(alpha * x1, x1)\n        # 16x16x64\n        \n        x2 = tf.layers.conv2d(relu1, 128, 5, strides=2, padding='same')\n        bn2 = tf.layers.batch_normalization(x2, training=True)\n        relu2 = tf.maximum(alpha * bn2, bn2)\n        # 8x8x128\n        \n        x3 = tf.layers.conv2d(relu2, 256, 5, strides=2, padding='same')\n        bn3 = tf.layers.batch_normalization(x3, training=True)\n        relu3 = tf.maximum(alpha * bn3, bn3)\n        # 4x4x256\n\n        # Flatten it\n        flat = tf.reshape(relu3, (-1, 4*4*256))\n        logits = tf.layers.dense(flat, 1)\n        out = tf.sigmoid(logits)\n        \n        return out, logits","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_loss(input_real, input_z, output_dim, alpha=0.2):\n    \"\"\"\n    Get the loss for the discriminator and generator\n    :param input_real: Images from the real dataset\n    :param input_z: Z input\n    :param out_channel_dim: The number of channels in the output image\n    :return: A tuple of (discriminator loss, generator loss)\n    \"\"\"\n    g_model = generator(input_z, output_dim, alpha=alpha)\n    d_model_real, d_logits_real = discriminator(input_real, alpha=alpha)\n    d_model_fake, d_logits_fake = discriminator(g_model, reuse=True, alpha=alpha)\n\n    d_loss_real = tf.reduce_mean(\n        tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_real, labels=tf.ones_like(d_model_real)))\n    d_loss_fake = tf.reduce_mean(\n        tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake, labels=tf.zeros_like(d_model_fake)))\n    g_loss = tf.reduce_mean(\n        tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake, labels=tf.ones_like(d_model_fake)))\n\n    d_loss = d_loss_real + d_loss_fake\n\n    return d_loss, g_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimizers"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_opt(d_loss, g_loss, learning_rate, beta1):\n    \"\"\"\n    Get optimization operations\n    :param d_loss: Discriminator loss Tensor\n    :param g_loss: Generator loss Tensor\n    :param learning_rate: Learning Rate Placeholder\n    :param beta1: The exponential decay rate for the 1st moment in the optimizer\n    :return: A tuple of (discriminator training operation, generator training operation)\n    \"\"\"\n    # Get weights and bias to update\n    t_vars = tf.trainable_variables()\n    d_vars = [var for var in t_vars if var.name.startswith('discriminator')]\n    g_vars = [var for var in t_vars if var.name.startswith('generator')]\n\n    # Optimize\n    with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):\n        d_train_opt = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(d_loss, var_list=d_vars)\n        g_train_opt = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(g_loss, var_list=g_vars)\n\n    return d_train_opt, g_train_opt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class GAN:\n    def __init__(self, real_size, z_size, learning_rate, alpha=0.2, beta1=0.5):\n        tf.reset_default_graph()\n        \n        self.input_real, self.input_z = model_inputs(real_size, z_size)\n        \n        self.d_loss, self.g_loss = model_loss(self.input_real, self.input_z,\n                                              real_size[2], alpha=alpha)\n        \n        self.d_opt, self.g_opt = model_opt(self.d_loss, self.g_loss, learning_rate, beta1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_samples(epoch, samples, nrows, ncols, figsize=(5,5)):\n    fig, axes = plt.subplots(figsize=figsize, nrows=nrows, ncols=ncols, \n                             sharey=True, sharex=True)\n    for ax, img in zip(axes.flatten(), samples[epoch]):\n        ax.axis('off')\n        img = ((img - img.min())*255 / (img.max() - img.min())).astype(np.uint8)\n        ax.set_adjustable('box-forced')\n        im = ax.imshow(img, aspect='equal')\n   \n    plt.subplots_adjust(wspace=0, hspace=0)\n    return fig, axes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(net, dataset, epochs, batch_size, print_every=10, show_every=100, figsize=(5,5)):\n    saver = tf.train.Saver()\n    sample_z = np.random.uniform(-1, 1, size=(72, z_size))\n\n    samples, losses = [], []\n    steps = 0\n\n    with tf.Session() as sess:\n        sess.run(tf.global_variables_initializer())\n        for e in range(epochs):\n            for x, y in dataset.batches(batch_size):\n                steps += 1\n\n                # Sample random noise for G\n                batch_z = np.random.uniform(-1, 1, size=(batch_size, z_size))\n\n                # Run optimizers\n                _ = sess.run(net.d_opt, feed_dict={net.input_real: x, net.input_z: batch_z})\n                _ = sess.run(net.g_opt, feed_dict={net.input_z: batch_z, net.input_real: x})\n\n                if steps % print_every == 0:\n                    # At the end of each epoch, get the losses and print them out\n                    train_loss_d = net.d_loss.eval({net.input_z: batch_z, net.input_real: x})\n                    train_loss_g = net.g_loss.eval({net.input_z: batch_z})\n\n                    print(\"Epoch {}/{}...\".format(e+1, epochs),\n                          \"Discriminator Loss: {:.4f}...\".format(train_loss_d),\n                          \"Generator Loss: {:.4f}\".format(train_loss_g))\n                    # Save losses to view after training\n                    losses.append((train_loss_d, train_loss_g))\n\n                if steps % show_every == 0:\n                    gen_samples = sess.run(\n                                   generator(net.input_z, 3, reuse=True, training=False),\n                                   feed_dict={net.input_z: sample_z})\n                    samples.append(gen_samples)\n                    _ = view_samples(-1, samples, 6, 12, figsize=figsize)\n                    plt.show()\n\n        saver.save(sess, './checkpoints/generator.ckpt')\n\n    with open('samples.pkl', 'wb') as f:\n        pkl.dump(samples, f)\n    \n    return losses, samples","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"real_size = (32,32,3)\nz_size = 100\nlearning_rate = 0.0002\nbatch_size = 128\nepochs = 25\nalpha = 0.2\nbeta1 = 0.5\n\n# Create the network\nnet = GAN(real_size, z_size, learning_rate, alpha=alpha, beta1=beta1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = Dataset(trainset, testset)\n\nlosses, samples = train(net, dataset, epochs, batch_size, figsize=(10,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots()\nlosses = np.array(losses)\nplt.plot(losses.T[0], label='Discriminator', alpha=0.5)\nplt.plot(losses.T[1], label='Generator', alpha=0.5)\nplt.title(\"Training Losses\")\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = view_samples(-1, samples, 6, 12, figsize=(10,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}