{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\n#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport xml.etree.ElementTree as ET \nimport matplotlib.pyplot as plt, zipfile \nfrom PIL import Image \nimport dask.array as da\nimport h5py\nimport tqdm\n\nimport os\nprint(os.listdir(\"../input\"))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dirlist = os.listdir(\".\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NN related imports\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.optimizers import SGD, Adam\nfrom keras import regularizers\nfrom keras.layers import Layer, Input, Conv2D, MaxPooling2D,AveragePooling2D, UpSampling2D, Conv2DTranspose,BatchNormalization\nfrom keras.layers import Dense, Dropout, Flatten, Reshape , LeakyReLU , Lambda, PReLU, Concatenate\nfrom keras.initializers import RandomNormal, Constant","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MODEL HYPERPARAMETERS.\nLATENT_DIMS = 100\nACTIVATION = 'relu'\nBATCH_SIZE=256\nEPOCHS=100\nLR=0.0005\n\n# OTHER CONSTANTS.\nCHUNK_SIZE = 20*1024\nLOSS_FACTOR = 10000\n\nOUT_DIR = \"./cropped_images/\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# extract the dog images and annotations into working directory.\nif \"croppedImages.h5\" not in dirlist:\n    import zipfile\n\n    with zipfile.ZipFile(\"../input/generative-dog-images/all-dogs.zip\",\"r\") as z:\n        z.extractall(\".\")\n    with zipfile.ZipFile(\"../input/generative-dog-images/Annotation.zip\",\"r\") as z:\n        z.extractall(\".\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if \"croppedImages.h5\" not in dirlist:\n    PATH = \"./all-dogs/\"\n    # get name of all dog image files.\n    imageNames = os.listdir(PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# crop the images based on bounding boxes given. Then resize (and crop if necessary) to 64*64*3\nif \"croppedImages.h5\" not in dirlist:\n    PATH_ANNO = './Annotation/'\n    breeds = os.listdir(PATH_ANNO)\n    imagesInput = np.zeros((len(imageNames)*2,64,64,3))\n    images_breed = []\n    i = 0\n    print(imagesInput.shape)\n    for breed in breeds:\n        for dog in os.listdir(PATH_ANNO+breed):\n            tree = ET.parse(PATH_ANNO + breed + '/' + dog)\n            root = tree.getroot()\n            try: img = Image.open(PATH + root.find('filename').text +'.jpg')\n            except: continue\n            for obj in root.findall('object'):\n                bndbox = obj.find('bndbox')\n                xmin = int(bndbox.find('xmin').text)\n                ymin = int(bndbox.find('ymin').text)\n                xmax = int(bndbox.find('xmax').text)\n                ymax = int(bndbox.find('ymax').text)\n                img_crop = img.crop((xmin, ymin, xmax, ymax))\n                w = img_crop.size[0]; h = img_crop.size[1];\n                a=0; b=0\n                if w<h:\n                    w2 = 64; h2 = int((64/w)*h)\n                    #b = np.random.randint(0,(h2-64)) if (h2-64 > 0) else 0\n                else:\n                    h2 = 64; w2 = int((64/h)*w)\n                    #a = np.random.randint(0,(w2-64)) if (w2-64 > 0) else 0\n                img_crop = img_crop.resize((w2,h2), Image.ANTIALIAS)\n                img_crop = img_crop.crop((0+a, 0+b, 64+a, 64+b))\n                imagesInput[i,:,:,:] = np.asarray(img_crop)\n                images_breed.append(obj.find('name').text)\n                i += 1\n    imagesInput = imagesInput[:i,:,:,:]   \n    print(imagesInput.shape)\n    flip_imagesInput = np.flip(imagesInput,2)\n    imagesInput = np.vstack((imagesInput,flip_imagesInput))\n    flip_imagesInput = None\n    images_breed = images_breed + images_breed\n#     imagesInput = imagesInput / (255 / 2) - 1\n    imagesInput = imagesInput / 255\n#     print(imagesInput.shape,len(images_breed))\n\n####################################\n#     # dask enhancement.\n    \n#     # write to file.\n    \n#     with h5py.File('croppedImages.h5', 'w') as h5f:\n#         h5f.create_dataset('croppedImages', data=imagesInput)\n#         h5f.close()\n\n# #     del imagesInput\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# write each image as a file to the directory.\nfor i in range(imageInput.size[0]):\n    try: img = Image.open(PATH + root.find('filename').text +'.jpg')\n                except: continue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(imagesInput[0])\nplt.imshow(imagesInput[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ii = imagesInput[1]/255\nplt.imshow(ii)\nprint(ii)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(ii)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imagesInput = imagesInput/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read the saved cropped images as dask array\nh5f = h5py.File('croppedImages.h5','r')\n# b = h5f[\"croppedImages\"]\ndaskInput = da.from_array(h5f[\"croppedImages\"], CHUNK_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"daskInput.visualize()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print a random image to check.\nrnd = np.random.randint(0,imagesInput.shape[0])\nplt.imshow(imagesInput[rnd])\nprint(imagesInput[0].shape)\n\n# print a random image to check.\n# rnd = np.random.randint(0,CHUNK_SIZE)\n# plt.imshow(daskInput[rnd])\n# print(daskInput[0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.array()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Sampling(Layer):\n    \"\"\"Uses (z_mean, z_log_var) to sample z, the vector encoding a digit.\"\"\"\n\n    def call(self, inputs):\n        z_mean, z_log_var = inputs\n        batch = tf.shape(z_mean)[0]\n        dim = tf.shape(z_mean)[1]\n        epsilon = tf.keras.backend.random_normal(shape=(batch, dim))\n        return z_mean + tf.exp(0.5 * z_log_var) * epsilon\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# noise generator to get latent representation.\ndef gen_noise(batch_size=1):\n#     noise = z_mean + tf.exp(0.5 * z_log_var) * epsilon\n    noise = np.random.normal(0,1,size=(batch_size, LATENT_DIMS))\n    return noise","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"init = RandomNormal(mean=0.0, stddev=0.02)\n\n# encoder\n\nimg_input = Input(shape=(64, 64, 3,))\nx = Conv2D(32, (5,5), strides=2, padding='same', kernel_initializer=init)(img_input)\n# x = PReLU(alpha_initializer=Constant(value=0.3))(x)\nx = LeakyReLU()(x)\n\nx = Conv2D(64, (2, 2), strides=2, padding='same', kernel_initializer=init)(x)\n# x = PReLU(alpha_initializer=Constant(value=0.3))(x)\nx = LeakyReLU()(x)\n\nx = Flatten()(x)\n\nx = Dense(64, activation=ACTIVATION)(x)\n\nz_mean = Dense(LATENT_DIMS, name=\"z_mean\")(x)\nz_log_var = Dense(LATENT_DIMS, name=\"z_log_var\")(x)\nz = Sampling()([z_mean, z_log_var])\nencoder = Model(img_input, [z_mean, z_log_var, z], name=\"encoder\")\nencoder.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# decoder\ncode_input = Input(shape=(LATENT_DIMS,))\nx = Dense(16*16*64, activation=ACTIVATION)(code_input)\nx = Reshape((16, 16, 64))(x)\n\nx = Conv2DTranspose(64, (2, 2), strides=2, padding=\"same\", kernel_initializer=init)(x)\n# x = PReLU(alpha_initializer=Constant(value=0.3))(x)\nx = LeakyReLU()(x)\nx = Conv2DTranspose(32, (5, 5), strides=2, padding=\"same\", kernel_initializer=init)(x)\n# x = PReLU(alpha_initializer=Constant(value=0.3))(x)\nx = LeakyReLU()(x)\n\nx = Conv2DTranspose(3, (1,1), strides=1, padding=\"same\", kernel_initializer=init, activation=\"sigmoid\")(x)\n\ndecoder = Model(code_input, x, name=\"decoder\")\ndecoder.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VAE(Model):\n    def __init__(self, encoder, decoder, **kwargs):\n        super(VAE, self).__init__(**kwargs)\n        self.encoder = encoder\n        self.decoder = decoder\n        \n    def train_step(self, data):\n        if isinstance(data, tuple):\n            data = data[0]\n        \n        with tf.GradientTape() as tape:\n            z_mean, z_log_var, z = encoder(data)\n            reconstruction = decoder(z)\n            reconstruction_loss = tf.reduce_mean(\n#                 keras.losses.binary_crossentropy(data, reconstruction)\n                keras.losses.mean_squared_error(data, reconstruction)\n            )\n            reconstruction_loss *= 64*64*3 # check later.\n            kl_loss = 1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var)\n#             kl_loss = tf.reduce_mean(kl_loss)\n            kl_loss = tf.reduce_sum(kl_loss)\n            kl_loss *= -0.5\n        \n#             def r_loss(y_true, y_pred):\n#                 return tf.keras.backend.mean(tf.square(y_true - y_pred), axis=[1,2,3])\n            \n#             rec_loss = r_loss(data, reconstruction)\n            \n            total_loss = reconstruction_loss + kl_loss\n#             total_loss = LOSS_FACTOR*rec_loss + kl_loss\n#             total_loss = kl_loss\n            \n        grads = tape.gradient(total_loss, self.trainable_weights)\n        self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n        return {\n            \"loss\": total_loss,\n            \"reconstruction_loss\": reconstruction_loss,\n            \"kl_loss\": kl_loss,\n        }\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # training\nvae = VAE(encoder, decoder)\nvae.compile(optimizer=Adam())\n\nnoise = gen_noise()\nloss, reconstruction_loss, kl_loss = [] , [], []\nprogress = []\nfor i in tqdm.tqdm(range(0, EPOCHS, 5)):\n    history = vae.fit(imagesInput, epochs=5, batch_size=BATCH_SIZE)\n    loss = history.history[\"loss\"]\n    reconstruction_loss = history.history[\"reconstruction_loss\"]\n    kl_loss = history.history[\"kl_loss\"]\n    \n    progress.append(np.array(decoder.predict(noise)[0]))\n\n# # dask enhancement.\n\n# # training\n# vae = VAE(encoder, decoder)\n# vae.compile(optimizer=Adam(lr=LR))\n\n# noise = gen_noise()\n# loss, reconstruction_loss, kl_loss = [] , [], []\n# progress = []\n# for i in tqdm.tqdm(range(1, EPOCHS + 1)):\n#     k = 0\n#     loss_acc, reconstruction_loss_acc, kl_loss_acc = 0, 0, 0\n#     for j in range(len(daskInput.chunks[0])):\n#         history = vae.fit(daskInput[k: k + CHUNK_SIZE, :, :, :], epochs=1, batch_size=BATCH_SIZE)\n#         k += CHUNK_SIZE\n#         loss_acc += history.history[\"loss\"][0]\n#         reconstruction_loss_acc += history.history[\"reconstruction_loss\"][0]\n#         kl_loss_acc += history.history[\"kl_loss\"][0]\n    \n#     loss_acc /= j\n#     reconstruction_loss_acc /= j\n#     kl_loss_acc /= j\n    \n#     loss.append(loss_acc)\n#     reconstruction_loss.append(reconstruction_loss_acc)\n#     kl_loss.append(kl_loss_acc)\n    \n#     progress.append(np.array(decoder.predict(noise)[0]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save model.\nvae.save(\"vae_dog_images\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show progress\ncolumns = 6 ; rows = min(6,(len(progress) // columns) + 1);\nfig=plt.figure(figsize=(32, 5 * rows))\nj=0\nfor i in range(0 , min(36,len(progress))):\n#     print(progress[int(j)])\n    fig.add_subplot(rows,columns,i+1)\n    plt.imshow(progress[int(j)])\n    j += max(1,len(progress) / 36)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(progress)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(daskInput[0,:,:,:])\nn = np.array(daskInput[44,:,:,:])\nprint(n)\nplt.imshow(n)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(n.shape)\nnn = np.array([n])\nprint(nn.shape)\ncode = encoder.predict(nn)\ncode","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ngen_img = decoder(code)[0]\nplt.imshow(gen_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(loss)\nplt.plot(reconstruction_loss)\nplt.plot(kl_loss)\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['tota loss', 'reconstruction loss', 'kl loss'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\ndef plot_latent(encoder, decoder):\n    # display a n*n 2D manifold of digits\n    n = 30\n    digit_size = 64\n    scale = 2.0\n    figsize = 15\n    figure = np.zeros((digit_size * n, digit_size * n, 3*n))\n    # linearly spaced coordinates corresponding to the 2D plot\n    # of digit classes in the latent space\n    grid_x = np.linspace(-scale, scale, n)\n    grid_y = np.linspace(-scale, scale, n)[::-1]\n\n    for i, yi in enumerate(grid_y):\n        for j, xi in enumerate(grid_x):\n            z_sample = np.array([[xi, yi]])\n            x_decoded = decoder.predict(z_sample)\n            digit = x_decoded[0].reshape(digit_size, digit_size)\n            figure[\n                i * digit_size : (i + 1) * digit_size,\n                j * digit_size : (j + 1) * digit_size,\n            ] = digit\n\n    plt.figure(figsize=(figsize, figsize))\n    start_range = digit_size // 2\n    end_range = n * digit_size + start_range + 1\n    pixel_range = np.arange(start_range, end_range, digit_size)\n    sample_range_x = np.round(grid_x, 1)\n    sample_range_y = np.round(grid_y, 1)\n    plt.xticks(pixel_range, sample_range_x)\n    plt.yticks(pixel_range, sample_range_y)\n    plt.xlabel(\"z[0]\")\n    plt.ylabel(\"z[1]\")\n    plt.imshow(figure, cmap=\"Greys_r\")\n    plt.show()\n\n\nplot_latent(encoder, decoder)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}