{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport shutil\nimport matplotlib.pyplot as plt\nimport zipfile\nimport tensorflow as tf\nimport xml.etree.ElementTree as ET\nfrom tqdm import tqdm\nfrom keras.models import Model \nfrom keras.models import Sequential\nfrom keras.layers.core import Dense\nfrom keras.layers.core import Dropout\nfrom keras.layers import Input\nfrom keras.layers import BatchNormalization\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Reshape\nfrom keras.layers import Flatten\nfrom keras.layers import Conv2D\nfrom keras.layers import Conv2DTranspose\nfrom keras.layers import UpSampling2D\nfrom keras.layers import ReLU\nfrom keras.layers.advanced_activations import LeakyReLU\nfrom keras.initializers import RandomNormal\nfrom keras.optimizers import Adam\nfrom keras import backend as K\nfrom PIL import Image\nfrom time import time\nimport torchvision","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = '../input/'\nIMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')\nBREEDS = os.listdir(ROOT + 'annotation/Annotation/') \nBREEDS_DIR=ROOT+'annotation/Annotation/'\nIMAGES_DIR=ROOT+'all-dogs/all-dogs/'\nstart=time()\n\nimg_rows = 64\nimg_cols = 64\nchannels = 3\nimg_shape = (img_rows, img_cols, channels)\nlatent_dim = 128\nrandom_dim =128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_images():\n    # Place holder for output \n    all_images = np.zeros((22250, 64, 64, 3))\n    namesIn = []\n    \n    # Index\n    index = 0\n    \n    required_transforms = torchvision.transforms.Compose([\n            torchvision.transforms.Resize(64),\n            torchvision.transforms.CenterCrop(64),\n    ])\n    \n    for breed in BREEDS:\n        for dog in os.listdir(BREEDS_DIR + breed):\n            try: img = Image.open(IMAGES_DIR + dog + '.jpg') \n            except: continue  \n                \n            tree = ET.parse(BREEDS_DIR + breed + '/' + dog)\n            root = tree.getroot()\n            objects = root.findall('object')\n            for o in objects:\n                bndbox = o.find('bndbox') \n                xmin = int(bndbox.find('xmin').text)\n                ymin = int(bndbox.find('ymin').text)\n                xmax = int(bndbox.find('xmax').text)\n                ymax = int(bndbox.find('ymax').text)\n                \n                w = np.min((xmax - xmin, ymax - ymin))\n                bbox = (xmin, ymin, xmin+w, ymin+w)\n                object_img = required_transforms(img.crop(bbox))\n                #object_img = object_img.resize((64,64), Image.ANTIALIAS)                \n                all_images[index,:]=object_img\n                index += 1\n                namesIn.append(breed)\n\n                # Determine each side\n                xdelta = xmax - xmin\n                ydelta = ymax - ymin\n                \n                # Take the mean of the sides\n                #w = int((xdelta + ydelta) / 2)\n                \n                \n#                 # Filter out images where bounding box is below 64 pixels.\n#                 # This filters out a couple of 100 images but prevents using low resolution images.\n#                 if xdelta >= 64 and ydelta >= 64:\n#                     img2 = img.crop((xmin, ymin, xmax, ymax))\n#                     img2 = img2.resize((64, 64), Image.ANTIALIAS)\n#                     image = np.asarray(img2)\n                    \n#                     #    # Normalize to range[-1, 1]\n#                     #all_images[index,:] = (image.astype(np.float32) - 127.5)/127.5\n#                     all_images[index,:]=image\n#                     index += 1\n#                     namesIn.append(breed)\n        \n                # Plot Status\n                if index % 5000 == 0:\n                    print('Processed Images: {}'.format(index))\n\n    print('Total Processed Images: {}'.format(index))\n    \n    \n#     for breed in BREEDS:\n#         for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):\n#             try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg') \n#             except: continue           \n#             tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)\n#             root = tree.getroot()\n#             objects = root.findall('object')\n#             for o in objects:\n#                 bndbox = o.find('bndbox') \n#                 xmin = int(bndbox.find('xmin').text)\n#                 ymin = int(bndbox.find('ymin').text)\n#                 xmax = int(bndbox.find('xmax').text)\n#                 ymax = int(bndbox.find('ymax').text)\n#                 w = np.min((xmax - xmin, ymax - ymin))\n#                 img2 = img.crop((xmin, ymin, xmin+w, ymin+w))\n#                 img2 = img2.resize((64,64), Image.ANTIALIAS)\n#                 imagesIn[idxIn,:,:,:] = np.asarray(img2)\n#                 #if idxIn%1000==0: print(idxIn)\n#                 namesIn.append(breed)\n#                 idxIn += 1\n#     idx = np.arange(idxIn)\n#     np.random.shuffle(idx)\n#     imagesIn = imagesIn[idx,:,:,:]\n#     namesIn = np.array(namesIn)[idx]\n\n    return all_images,namesIn,index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train,namesIn,idxIn=load_images()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#DISPLAY CROPPED IMAGES\nx = np.random.randint(0,idxIn,15)\nfor k in range(3):\n    plt.figure(figsize=(15,3))\n    for j in range(5):\n        plt.subplot(1,5,j+1)\n        img = Image.fromarray(X_train[x[k*5+j],:,:,:].astype('uint8') )\n        plt.axis('off')\n        plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)\n        plt.imshow(img)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# adapted from keras.optimizers.Adam\nclass AdamWithWeightnorm(Adam):\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr *= (1. / (1. + self.decay * K.cast(self.iterations, K.floatx())))\n\n        t = K.cast(self.iterations + 1, K.floatx())\n        lr_t = lr * K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))\n\n        shapes = [K.get_variable_shape(p) for p in params]\n        ms = [K.zeros(shape) for shape in shapes]\n        vs = [K.zeros(shape) for shape in shapes]\n        self.weights = [self.iterations] + ms + vs\n\n        for p, g, m, v in zip(params, grads, ms, vs):\n\n            # if a weight tensor (len > 1) use weight normalized parameterization\n            # this is the only part changed w.r.t. keras.optimizers.Adam\n            ps = K.get_variable_shape(p)\n            if len(ps)>1:\n\n                # get weight normalization parameters\n                V, V_norm, V_scaler, g_param, grad_g, grad_V = get_weightnorm_params_and_grads(p, g)\n\n                # Adam containers for the 'g' parameter\n                V_scaler_shape = K.get_variable_shape(V_scaler)\n                m_g = K.zeros(V_scaler_shape)\n                v_g = K.zeros(V_scaler_shape)\n\n                # update g parameters\n                m_g_t = (self.beta_1 * m_g) + (1. - self.beta_1) * grad_g\n                v_g_t = (self.beta_2 * v_g) + (1. - self.beta_2) * K.square(grad_g)\n                new_g_param = g_param - lr_t * m_g_t / (K.sqrt(v_g_t) + self.epsilon)\n                self.updates.append(K.update(m_g, m_g_t))\n                self.updates.append(K.update(v_g, v_g_t))\n\n                # update V parameters\n                m_t = (self.beta_1 * m) + (1. - self.beta_1) * grad_V\n                v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(grad_V)\n                new_V_param = V - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n                self.updates.append(K.update(m, m_t))\n                self.updates.append(K.update(v, v_t))\n\n                # if there are constraints we apply them to V, not W\n                if getattr(p, 'constraint', None) is not None:\n                    new_V_param = p.constraint(new_V_param)\n\n                # wn param updates --> W updates\n                add_weightnorm_param_updates(self.updates, new_V_param, new_g_param, p, V_scaler)\n\n            else: # do optimization normally\n                m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n                v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n                self.updates.append(K.update(m, m_t))\n                self.updates.append(K.update(v, v_t))\n\n                new_p = p_t\n                # apply constraints\n                if getattr(p, 'constraint', None) is not None:\n                    new_p = p.constraint(new_p)\n                self.updates.append(K.update(p, new_p))\n        return self.updates\n\ndef get_weightnorm_params_and_grads(p, g):\n    ps = K.get_variable_shape(p)\n\n    # construct weight scaler: V_scaler = g/||V||\n    V_scaler_shape = (ps[-1],)  # assumes we're using tensorflow!\n    V_scaler = K.ones(V_scaler_shape)  # init to ones, so effective parameters don't change\n\n    # get V parameters = ||V||/g * W\n    norm_axes = [i for i in range(len(ps) - 1)]\n    V = p / tf.reshape(V_scaler, [1] * len(norm_axes) + [-1])\n\n    # split V_scaler into ||V|| and g parameters\n    V_norm = tf.sqrt(tf.reduce_sum(tf.square(V), norm_axes))\n    g_param = V_scaler * V_norm\n\n    # get grad in V,g parameters\n    grad_g = tf.reduce_sum(g * V, norm_axes) / V_norm\n    grad_V = tf.reshape(V_scaler, [1] * len(norm_axes) + [-1]) * \\\n             (g - tf.reshape(grad_g / V_norm, [1] * len(norm_axes) + [-1]) * V)\n\n    return V, V_norm, V_scaler, g_param, grad_g, grad_V\n\ndef add_weightnorm_param_updates(updates, new_V_param, new_g_param, W, V_scaler):\n    ps = K.get_variable_shape(new_V_param)\n    norm_axes = [i for i in range(len(ps) - 1)]\n\n    # update W and V_scaler\n    new_V_norm = tf.sqrt(tf.reduce_sum(tf.square(new_V_param), norm_axes))\n    new_V_scaler = new_g_param / new_V_norm\n    new_W = tf.reshape(new_V_scaler, [1] * len(norm_axes) + [-1]) * new_V_param\n    updates.append(K.update(W, new_W))\n    updates.append(K.update(V_scaler, new_V_scaler))\n\n# data based initialization for a given Keras model\ndef data_based_init(model, input):\n    # input can be dict, numpy array, or list of numpy arrays\n    if type(input) is dict:\n        feed_dict = input\n    elif type(input) is list:\n        feed_dict = {tf_inp: np_inp for tf_inp,np_inp in zip(model.inputs,input)}\n    else:\n        feed_dict = {model.inputs[0]: input}\n\n    # add learning phase if required\n    if model.uses_learning_phase and K.learning_phase() not in feed_dict:\n        feed_dict.update({K.learning_phase(): 1})\n\n    # get all layer name, output, weight, bias tuples\n    layer_output_weight_bias = []\n    for l in model.layers:\n        trainable_weights = l.trainable_weights\n        if len(trainable_weights) == 2:\n            W,b = trainable_weights\n            assert(l.built)\n            layer_output_weight_bias.append((l.name,l.get_output_at(0),W,b)) # if more than one node, only use the first\n\n    # iterate over our list and do data dependent init\n    sess = K.get_session()\n    for l,o,W,b in layer_output_weight_bias:\n        print('Performing data dependent initialization for layer ' + l)\n        m,v = tf.nn.moments(o, [i for i in range(len(o.get_shape())-1)])\n        s = tf.sqrt(v + 1e-10)\n        updates = tf.group(W.assign(W/tf.reshape(s,[1]*(len(W.get_shape())-1)+[-1])), b.assign((b-m)/s))\n        sess.run(updates, feed_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#optimizer = tf.keras.optimizers.Adam(0.001, 0.5)\nadamWithWeightnorm=AdamWithWeightnorm(lr = 0.0002, beta_1 = 0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_generator():\n\n    # Random Normal Weight Initialization\n    init = RandomNormal(mean = 0.0, stddev = 0.02)\n\n    # Model\n    model = Sequential()\n\n    # Start at 4 * 4\n    start_shape = 64 * 4 * 4\n    model.add(Dense(start_shape, kernel_initializer = init, input_dim = random_dim))\n    model.add(Reshape((4, 4, 64)))\n    \n    # Upsample => 8 * 8 \n    model.add(UpSampling2D())\n    model.add(Conv2D(128, kernel_size = 3, padding = \"same\", kernel_initializer = init))\n    model.add(ReLU())\n    \n    # Upsample => 16 * 16 \n    model.add(UpSampling2D())\n    model.add(Conv2D(128, kernel_size = 3, padding = \"same\", kernel_initializer = init))\n    model.add(ReLU())\n    \n    # Upsample => 32 * 32\n    model.add(UpSampling2D())\n    model.add(Conv2D(128, kernel_size = 3, padding = \"same\", kernel_initializer = init))\n    model.add(ReLU())\n    \n    # Upsample => 64 * 64\n    model.add(UpSampling2D())\n    model.add(Conv2D(128, kernel_size = 3, padding = \"same\", kernel_initializer = init))\n    model.add(ReLU())\n    \n    # output\n    model.add(Conv2D(3, kernel_size = 3, activation = 'tanh', padding = 'same', kernel_initializer=init))\n    model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = 0.0002, beta_1 = 0.5))\n    print(model.summary())\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_discriminator():\n\n    input_shape = (64, 64, 3)\n\n    # Random Normal Weight Initialization\n    init = RandomNormal(mean = 0.0, stddev = 0.02)\n\n    # Define Model\n    model = Sequential()\n\n    # Downsample ==> 32 * 32\n    model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init, input_shape = input_shape))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n\n    # Downsample ==> 16 * 16\n    model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n    \n    # Downsample => 8 * 8\n    model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n    \n    # Downsample => 4 * 4\n    model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))\n    model.add(LeakyReLU(0.2))\n    model.add(Dropout(0.25))\n    \n    # Final Layers\n    model.add(Flatten())\n    model.add(Dense(1, activation = 'sigmoid', kernel_initializer = init))\n\n    # Compile model\n    model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = 0.0002, beta_1 = 0.5))\n    \n    print(model.summary())\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_images( epoch):\n    r, c = 5, 5\n    noise = np.random.normal(0, 1, (r * c, latent_dim))\n    gen_imgs = generator.predict(noise)\n\n    # Rescale images 0 - 1\n    gen_imgs = 0.5 * gen_imgs + 0.5\n\n    fig, axs = plt.subplots(r, c)\n    cnt = 0\n    for i in range(r):\n        for j in range(c):\n            axs[i,j].imshow(image.array_to_img(gen_imgs[cnt]))\n            axs[i,j].axis('off')\n            cnt += 1\n    #fig.savefig(\"mnist_%d.png\" % epoch)\n    plt.show()\n    plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(epochs, batch_size=128, save_interval=50):\n\n    # Adversarial ground truths\n    valid = np.ones((batch_size, 1))\n    fake = np.zeros((batch_size, 1))\n    \n    valid[:]=0.9\n    \n    for epoch in range(epochs):\n        \n        end = time()\n        if (end -start) > 31900 :\n            break\n\n        # ---------------------\n        #  Train Discriminator\n        # ---------------------\n\n        # Select a random half of images\n        idx = np.random.randint(0, X_train.shape[0], batch_size)\n        imgs = X_train[idx]\n        \n        #imgs = imgs/255\n        imgs = (imgs -127.5) / 127.5\n\n        # Sample noise and generate a batch of new images\n        noise = np.random.normal(0, 1, (batch_size, latent_dim))\n        gen_imgs = generator.predict(noise)\n\n        # Train the discriminator (real classified as ones and generated as zeros)\n        discriminator.trainable = True\n        d_loss_real = discriminator.train_on_batch(imgs, valid)\n        discriminator.trainable = True\n        d_loss_fake = discriminator.train_on_batch(gen_imgs, fake)\n        d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)\n        \n#         print(\"d_loss_real:\",d_loss_real)\n#         print(\"d_loss_fake:\",d_loss_fake)\n#         print(\"d_loss:\",d_loss)\n\n        # ---------------------\n        #  Train Generator\n        # ---------------------\n        discriminator.trainable = False\n        # Train the generator (wants discriminator to mistake images as real)\n        y_gen =  np.ones((batch_size, 1))        \n        g_loss = combined.train_on_batch(noise, y_gen)\n        #g_loss = combined.train_on_batch(noise, valid)\n\n        # Plot the progress\n        #print (\"%d [D loss: %f, acc.: %.2f%%] [G loss: %f]\" % (epoch, d_loss[0], 100*d_loss[1], g_loss))\n\n        # If at save interval => save generated image samples\n        if (epoch+1) % save_interval == 0  or  epoch == 0:\n            print (\"%d [D loss: %f, acc.: %.2f%%] [G loss: %f]\" % (epoch, d_loss, 100*d_loss, g_loss))\n            #show_images(epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build and compile the discriminator\ndiscriminator = build_discriminator()\n#discriminator.compile(loss='binary_crossentropy',\n#    optimizer=AdamWithWeightnorm(lr = 0.0002, beta_1 = 0.5))\n    #metrics=['accuracy'])\n\n# Build the generator\ngenerator = build_generator()\n\n# The generator takes noise as input and generates imgs\nz = Input(shape=(latent_dim,))\nimg = generator(z)\n\n# For the combined model we will only train the generator\ndiscriminator.trainable = False\n\n# The discriminator takes generated images as input and determines validity\nvalid = discriminator(img)\n\n# The combined model  (stacked generator and discriminator)\n# Trains the generator to fool the discriminator\ncombined = Model(z, valid)\ncombined.compile(loss='binary_crossentropy', optimizer=AdamWithWeightnorm(lr = 0.0002, beta_1 = 0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain(epochs=2000000, batch_size=32, save_interval=10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k in range(5):\n    noise = np.random.normal(0, 1, (1, latent_dim))\n    gen_imgs = generator.predict(noise)\n\n    # Rescale images 0 - 1\n    #gen_imgs = 0.5 * gen_imgs + 0.5\n    \n    image = Image.fromarray(((gen_imgs + 1) * 127.5).astype('uint8').reshape(64, 64, 3))\n\n    #plt.imshow(image.array_to_img(gen_imgs[0]))\n    plt.imshow(image, interpolation = 'nearest')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"z = zipfile.PyZipFile('images.zip', mode='w')\nfor k in range(10000):\n    noise = np.random.normal(0, 1, (1, latent_dim))\n    gen_imgs = generator.predict(noise)\n    # Rescale images 0 - 1\n#     gen_imgs = 0.5 * gen_imgs + 0.5\n#     save_image=image.array_to_img(gen_imgs[0])\n    save_image = Image.fromarray(((gen_imgs + 1) * 127.5).astype('uint8').reshape(64, 64, 3))\n    #plt.imshow(save_image)\n    #plt.show()\n    f = str(k)+'.png'\n    save_image.save(f,'PNG'); z.write(f); os.remove(f)\n    #if k % 1000==0: print(k)\nz.close()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}