{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os, os.path\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from xml.etree import ElementTree as ET\n\ndef parse_annotation(fname):\n    objects = []\n    for child in ET.parse(fname).findall(\"object\"):\n        dog = {}\n        dog['name'] = child.find('name').text\n        dog['pose'] = child.find('pose').text\n        dog['difficult'] = int(child.find('difficult').text)\n        dog['truncated'] = int(child.find('truncated').text)\n        \n        bbox = child.find('bndbox')\n        dog['bbox'] = [\n            int(bbox.find('xmin').text),\n            int(bbox.find('ymin').text),\n            int(bbox.find('xmax').text),\n            int(bbox.find('ymax').text)\n        ]\n        objects.append(dog)\n    return objects","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_DIR = '../input/all-dogs/all-dogs'\ndog_imgs = pd.DataFrame(os.listdir(IMAGE_DIR), columns=['filename'])\ndog_imgs['basename'] = dog_imgs['filename'].str.split('.').apply(lambda x: x[0])\ndog_imgs[['class','id']] = dog_imgs['basename'].str.split(\"_\",expand=True,)\ndog_imgs = dog_imgs.set_index('basename').sort_index()\n\nANNOTATION_DIR = '../input/annotation/Annotation'\ndog_breeds = pd.DataFrame(os.listdir(ANNOTATION_DIR), columns=['dirname'])\ndog_breeds[['class', 'breedname']] = dog_breeds['dirname'].str.split(\"-\",1,expand=True)\ndog_breeds = dog_breeds.set_index('class').sort_index()\n\ndog_imgs['annotation_filename'] = dog_imgs.apply(lambda x: os.path.join(ANNOTATION_DIR, dog_breeds.loc[x['class']]['dirname'], x.name), axis=1)\ndog_imgs['objects'] = dog_imgs['annotation_filename'].apply(parse_annotation)\n\ndisplay(dog_imgs.head())\ndisplay(dog_breeds.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"doggo = dog_imgs.sample(1).iloc[0]\n\nimport matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\nfrom PIL import Image\n\nimport imgaug.augmenters as iaa\n\npil_im = Image.open(os.path.join(IMAGE_DIR, doggo['filename']))\nim = np.asarray(pil_im)\n\nfig,ax = plt.subplots(1)\n\nax.imshow(im)\n\nh,w,c = im.shape\nfor dog in doggo['objects']:\n    xmin, ymin, xmax, ymax = dog['bbox']\n    print(h,w,\":\",xmin,ymin,xmax,ymax)\n    bbox = patches.Rectangle((xmin, ymin), xmax-xmin, ymax-ymin, linewidth=1, edgecolor='r', facecolor='none')\n    ax.add_patch(bbox)\n\nplt.show()\n\nfig,ax = plt.subplots(1)\n\ndog = doggo.objects[0]\n\nh,w,c = im.shape\nxmin, ymin, xmax, ymax = dog['bbox']\n\n#im = im[ymin:ymax,xmin:xmax]\npil_crop = pil_im.crop((xmin, ymin, xmax, ymax)).resize((64, 64))\nim2 = np.asarray(pil_crop)\n\nax.imshow(im2)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\nfrom PIL import Image\n\nimport imgaug.augmenters as iaa\nfrom tqdm import tqdm, tqdm_notebook\n\ndef get_truth_images():\n    all_imgs = []\n    \n    for _,doggo in tqdm_notebook(dog_imgs.iterrows(), total=len(dog_imgs)):\n        pil_im = Image.open(os.path.join(IMAGE_DIR, doggo['filename']))\n        h,w,c = im.shape\n        \n        for dog in doggo['objects']:\n            border = 4#int(min(h,w)*.1)\n            \n            xmin, ymin, xmax, ymax = dog['bbox']\n            \n            xmin = max(0, xmin-border)\n            ymin = max(0, ymin-border)\n            xmax = min(w, xmax+border)\n            ymax = min(h, ymax+border)\n\n            pil_crop = pil_im.crop((xmin, ymin, xmax, ymax)).resize((64, 64))\n            all_imgs.append(np.asarray(pil_crop))\n\n    return np.stack(all_imgs)\n\ntruth_imgs = get_truth_images()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"truth_nrm_imgs = (truth_imgs-127.5)/127.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.optimizers import Adam\nfrom keras import backend as K\n\n# adapted from keras.optimizers.Adam\nclass AdamWithWeightnorm(Adam):\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr *= (1. / (1. + self.decay * K.cast(self.iterations, K.floatx())))\n\n        t = K.cast(self.iterations + 1, K.floatx())\n        lr_t = lr * K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))\n\n        shapes = [K.get_variable_shape(p) for p in params]\n        ms = [K.zeros(shape) for shape in shapes]\n        vs = [K.zeros(shape) for shape in shapes]\n        self.weights = [self.iterations] + ms + vs\n\n        for p, g, m, v in zip(params, grads, ms, vs):\n\n            # if a weight tensor (len > 1) use weight normalized parameterization\n            # this is the only part changed w.r.t. keras.optimizers.Adam\n            ps = K.get_variable_shape(p)\n            if len(ps)>1:\n\n                # get weight normalization parameters\n                V, V_norm, V_scaler, g_param, grad_g, grad_V = get_weightnorm_params_and_grads(p, g)\n\n                # Adam containers for the 'g' parameter\n                V_scaler_shape = K.get_variable_shape(V_scaler)\n                m_g = K.zeros(V_scaler_shape)\n                v_g = K.zeros(V_scaler_shape)\n\n                # update g parameters\n                m_g_t = (self.beta_1 * m_g) + (1. - self.beta_1) * grad_g\n                v_g_t = (self.beta_2 * v_g) + (1. - self.beta_2) * K.square(grad_g)\n                new_g_param = g_param - lr_t * m_g_t / (K.sqrt(v_g_t) + self.epsilon)\n                self.updates.append(K.update(m_g, m_g_t))\n                self.updates.append(K.update(v_g, v_g_t))\n\n                # update V parameters\n                m_t = (self.beta_1 * m) + (1. - self.beta_1) * grad_V\n                v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(grad_V)\n                new_V_param = V - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n                self.updates.append(K.update(m, m_t))\n                self.updates.append(K.update(v, v_t))\n\n                # if there are constraints we apply them to V, not W\n                if getattr(p, 'constraint', None) is not None:\n                    new_V_param = p.constraint(new_V_param)\n\n                # wn param updates --> W updates\n                add_weightnorm_param_updates(self.updates, new_V_param, new_g_param, p, V_scaler)\n\n            else: # do optimization normally\n                m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n                v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n                self.updates.append(K.update(m, m_t))\n                self.updates.append(K.update(v, v_t))\n\n                new_p = p_t\n                # apply constraints\n                if getattr(p, 'constraint', None) is not None:\n                    new_p = p.constraint(new_p)\n                self.updates.append(K.update(p, new_p))\n        return self.updates\n\nimport tensorflow as tf\n    \ndef get_weightnorm_params_and_grads(p, g):\n    ps = K.get_variable_shape(p)\n\n    # construct weight scaler: V_scaler = g/||V||\n    V_scaler_shape = (ps[-1],)  # assumes we're using tensorflow!\n    V_scaler = K.ones(V_scaler_shape)  # init to ones, so effective parameters don't change\n\n    # get V parameters = ||V||/g * W\n    norm_axes = [i for i in range(len(ps) - 1)]\n    V = p / tf.reshape(V_scaler, [1] * len(norm_axes) + [-1])\n\n    # split V_scaler into ||V|| and g parameters\n    V_norm = tf.sqrt(tf.reduce_sum(tf.square(V), norm_axes))\n    g_param = V_scaler * V_norm\n\n    # get grad in V,g parameters\n    grad_g = tf.reduce_sum(g * V, norm_axes) / V_norm\n    grad_V = tf.reshape(V_scaler, [1] * len(norm_axes) + [-1]) * \\\n             (g - tf.reshape(grad_g / V_norm, [1] * len(norm_axes) + [-1]) * V)\n\n    return V, V_norm, V_scaler, g_param, grad_g, grad_V\n\ndef add_weightnorm_param_updates(updates, new_V_param, new_g_param, W, V_scaler):\n    ps = K.get_variable_shape(new_V_param)\n    norm_axes = [i for i in range(len(ps) - 1)]\n\n    # update W and V_scaler\n    new_V_norm = tf.sqrt(tf.reduce_sum(tf.square(new_V_param), norm_axes))\n    new_V_scaler = new_g_param / new_V_norm\n    new_W = tf.reshape(new_V_scaler, [1] * len(norm_axes) + [-1]) * new_V_param\n    updates.append(K.update(W, new_W))\n    updates.append(K.update(V_scaler, new_V_scaler))\n\n# data based initialization for a given Keras model\ndef data_based_init(model, input):\n    # input can be dict, numpy array, or list of numpy arrays\n    if type(input) is dict:\n        feed_dict = input\n    elif type(input) is list:\n        feed_dict = {tf_inp: np_inp for tf_inp,np_inp in zip(model.inputs,input)}\n    else:\n        feed_dict = {model.inputs[0]: input}\n\n    # add learning phase if required\n    if model.uses_learning_phase and K.learning_phase() not in feed_dict:\n        feed_dict.update({K.learning_phase(): 1})\n\n    # get all layer name, output, weight, bias tuples\n    layer_output_weight_bias = []\n    for l in model.layers:\n        trainable_weights = l.trainable_weights\n        if len(trainable_weights) == 2:\n            W,b = trainable_weights\n            assert(l.built)\n            layer_output_weight_bias.append((l.name,l.get_output_at(0),W,b)) # if more than one node, only use the first\n\n    # iterate over our list and do data dependent init\n    sess = K.get_session()\n    for l,o,W,b in layer_output_weight_bias:\n        print('Performing data dependent initialization for layer ' + l)\n        m,v = tf.nn.moments(o, [i for i in range(len(o.get_shape())-1)])\n        s = tf.sqrt(v + 1e-10)\n        updates = tf.group(W.assign(W/tf.reshape(s,[1]*(len(W.get_shape())-1)+[-1])), b.assign((b-m)/s))\n        sess.run(updates, feed_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model, Sequential\nfrom keras.layers import (\n    Dense, Conv2D, Flatten, Concatenate, UpSampling2D,\n    Dropout, LeakyReLU, ReLU, Reshape, Input, Conv2DTranspose\n)\nfrom keras.initializers import RandomNormal\n\nfrom keras import backend as K\nimport tensorflow as tf\n\ndef make_discriminator_model(input_shape=(64, 64, 3)):\n    init = RandomNormal(mean=0.0, stddev=0.02)\n    \n    model = Sequential()\n    # 64x64 => in\n    model.add(Conv2D(32, \n                     kernel_size=4,\n                     strides=2,\n                     padding='same',\n                     kernel_initializer=init,\n                     input_shape=input_shape,))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n    # out => 32x32\n    \n    # 32x32 => in\n    model.add(Conv2D(64, \n                     kernel_size=4,\n                     strides=2,\n                     padding='same',\n                     kernel_initializer=init,))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n    # out => 16x16\n    \n    # 16x16 => in\n    model.add(Conv2D(128, \n                     kernel_size=4,\n                     strides=2,\n                     padding='same',\n                     kernel_initializer=init,))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n    # out => 8x8\n    \n    # 8x8 => in\n    model.add(Conv2D(256, \n                     kernel_size=4,\n                     strides=2,\n                     padding='same',\n                     kernel_initializer=init,))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n    # out => 4x4\n    \n    model.add(Flatten())\n    model.add(Dense(1, \n                    activation='linear', \n                    kernel_initializer=init))\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_generator_model(random_dim=128, start_shape=(4, 4, 64)):\n    init = RandomNormal(mean=0.0, stddev=0.02)\n    \n    model = Sequential()\n    \n    a,b,c = start_shape; start_dim = a*b*c\n    \n    model.add(Dense(start_dim,\n                    kernel_initializer=init,\n                    input_dim=random_dim))\n    model.add(Reshape(start_shape))\n    \n    #model.add(Conv2DTranspose(512, kernel_size=4, padding='same', \n    #                          kernel_initializer=init))\n    \n    # => 8x8\n    model.add(UpSampling2D(interpolation='bilinear'))\n    model.add(Conv2D(512, \n                     kernel_size=4,\n                     padding='same',\n                     kernel_initializer=init))\n    model.add(ReLU())\n    \n    # => 16x16\n    model.add(UpSampling2D(interpolation='bilinear'))\n    model.add(Conv2D(256, \n                     kernel_size=4,\n                     padding='same',\n                     kernel_initializer=init))\n    model.add(ReLU())\n    \n    # => 32x32\n    #model.add(Conv2DTranspose(128,\n    #                          kernel_size=3,\n    #                          strides=2,\n    #                          padding='same',\n    #                          kernel_initializer=init))\n    model.add(UpSampling2D(interpolation='bilinear'))\n    model.add(Conv2D(128, \n                     kernel_size=4,\n                     padding='same',\n                     kernel_initializer=init))\n    model.add(ReLU())\n    \n    # => 64x64\n    #model.add(Conv2DTranspose(128,\n    #                          kernel_size=3,\n    #                          strides=2,\n    #                          padding='same',\n    #                          kernel_initializer=init))\n    model.add(UpSampling2D(interpolation='bilinear'))\n    model.add(Conv2D(64, \n                     kernel_size=4,\n                     padding='same',\n                     kernel_initializer=init))\n    model.add(ReLU())\n    \n    model.add(Conv2D(3, \n                     kernel_size=3,\n                     activation='tanh',\n                     padding='same',\n                     kernel_initializer=init))\n    model.summary()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_gan_model(dis_model, gen_model, random_dim=128):\n    dis_model.trainable = False\n    gan_input = Input(shape=(random_dim,))\n    gen_output = gen_model(gan_input)\n    gan_output = dis_model(gen_output)\n    \n    gan_model = Model(inputs=gan_input, outputs=gan_output)\n    \n    return gan_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def gen_input(random_dim, n_samples):\n    noise = np.random.randn(random_dim * n_samples)\n    noise = noise.reshape((n_samples, random_dim))\n    \n    return noise\n\ndef plot_gen_noise(gen_model, random_dim=128, examples=25, dim=(5,5)):\n    gen_imgs = gen_model.predict(gen_input(128, 25))\n    gen_imgs = ((gen_imgs + 1)*127.5).astype('uint8')\n    \n    plt.figure(figsize=(12,8))\n    for i, img in enumerate(gen_imgs):\n        plt.subplot(dim[0], dim[1], i+1)\n        plt.imshow(img, interpolation='bilinear')\n        plt.axis('off')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RANDOM_DIM = 128\nRAW_BATCH_SIZE = 32\nDIS_TRAIN_RATIO = 2\n\nMINI_BATCH_SIZE = RAW_BATCH_SIZE//DIS_TRAIN_RATIO\n\ndis_model = make_discriminator_model()\ngen_model = make_generator_model()\n\nbatch_count = truth_nrm_imgs.shape[0] // RAW_BATCH_SIZE\n\nadam_nrm_op = AdamWithWeightnorm(lr=0.0002, beta_1=0.5, beta_2=0.999)\n\nreal_inp = Input(shape=truth_nrm_imgs.shape[1:])\nnois_inp = Input(shape=(RANDOM_DIM,))\nfake_inp = gen_model(nois_inp)\n\ndisc_r = dis_model(real_inp)\ndisc_f = dis_model(fake_inp)\n\n# Relative GAN\ndef rel_dis_loss(_y_real, _y_pred):\n    epsilon = K.epsilon()\n    return -(\n        K.mean(K.log(  K.sigmoid(disc_r - K.mean(disc_f, axis=0)) + epsilon), axis=0) +\n        K.mean(K.log(1-K.sigmoid(disc_f - K.mean(disc_r, axis=0)) + epsilon), axis=0)\n    )\n\ndef rel_gen_loss(_y_real, _y_pred):\n    epsilon = K.epsilon()\n    return -(\n        K.mean(K.log(  K.sigmoid(disc_f - K.mean(disc_r, axis=0)) + epsilon), axis=0) +\n        K.mean(K.log(1-K.sigmoid(disc_r - K.mean(disc_f, axis=0)) + epsilon), axis=0)\n    )\n\n# RaLSGAN\nREAL_LABEL = 0.8\ndef rals_dis_loss(_y_real, _y_pred):\n    return K.mean(\n        K.pow(disc_r - K.mean(disc_f, axis=0) - REAL_LABEL, 2) +\n        K.pow(disc_f - K.mean(disc_r, axis=0) + REAL_LABEL, 2)\n    )\n\ndef rals_gen_loss(_y_real, _y_pred):\n    return K.mean(\n        K.pow(disc_r - K.mean(disc_f, axis=0) + REAL_LABEL, 2) +\n        K.pow(disc_f - K.mean(disc_r, axis=0) - REAL_LABEL, 2)\n    )\n\ngen_train = Model([nois_inp, real_inp], [disc_r, disc_f])\ndis_model.trainable = False\ngen_train.compile(adam_nrm_op, loss=[rals_gen_loss, None])\ngen_train.summary()\n\ndis_train = Model([nois_inp, real_inp], [disc_r, disc_f])\ngen_model.trainable = False\ndis_model.trainable = True\ndis_train.compile(adam_nrm_op, loss=[rals_dis_loss, None])\ndis_train.summary()\n\ngen_loss = []\ndis_loss = []\n\ndummy_y = np.zeros((RAW_BATCH_SIZE, 1), dtype=np.float32)\ndummy_mini_y = np.zeros((MINI_BATCH_SIZE, 1), dtype=np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import imgaug.augmenters as iaa\n\naug_seq = iaa.Sequential([\n    iaa.Affine(rotate=(-8,8)),\n    iaa.Fliplr(0.5),\n], random_order=True)\n\ndef augment(images):\n    return aug_seq.augment_images(images=images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch_i in range(300):\n    epoch_idx = np.arange(truth_nrm_imgs.shape[0])\n    np.random.shuffle(epoch_idx)\n\n    for batch_i in tqdm_notebook(range(batch_count)):\n\n        # train the discriminator\n        dis_model.trainable = True\n        gen_model.trainable = False\n        \n        loss = 0\n        for mini_j in range(DIS_TRAIN_RATIO):\n            mini_oset = batch_i*DIS_TRAIN_RATIO + mini_j\n            \n            z_fake = gen_input(RANDOM_DIM, MINI_BATCH_SIZE)\n            X_real = augment(truth_nrm_imgs[epoch_idx[\n                mini_oset*MINI_BATCH_SIZE:(mini_oset+1)*MINI_BATCH_SIZE]])\n\n            loss += dis_train.train_on_batch([z_fake, X_real], dummy_mini_y)[0]\n        dis_loss.append(loss/DIS_TRAIN_RATIO)\n\n        dis_model.trainable = False\n        gen_model.trainable = True\n            \n        z_fake = gen_input(RANDOM_DIM, RAW_BATCH_SIZE)\n        X_real = augment(truth_nrm_imgs[epoch_idx[batch_i*RAW_BATCH_SIZE:(batch_i+1)*RAW_BATCH_SIZE]])\n        \n        gen_loss.append(gen_train.train_on_batch([z_fake, X_real], dummy_y)[0])\n\n    if epoch_i%5 == 0:\n        plot_gen_noise(gen_model)\n        plt.suptitle('Epoch {}'.format(epoch_i+1), x=0.5, y=1.0)\n        plt.savefig('dog_at_epoch_{}.png'.format(epoch_i+1))\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(gen_loss, label='Generator loss')\nplt.plot(dis_loss, label=\"Discriminator loss\")\nplt.legend()\nplt.ylim([0, 15])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_gen_noise(gen_model)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\nfrom PIL import Image\n\nN_OUTPUT = 10000\nz = zipfile.PyZipFile('images.zip', mode='w')\ngenerated_images = gen_model.predict(gen_input(RANDOM_DIM, N_OUTPUT))\n\nfor k, img_arr in tqdm_notebook(enumerate(generated_images), total=N_OUTPUT):\n    image = Image.fromarray(((img_arr+1)*127.5).astype('uint8'))\n    \n    fname = str(k)+'.png'\n    image.save(fname, 'PNG')\n    z.write(fname)\n    os.remove(fname)\nz.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}