{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":15062,"databundleVersionId":545987,"sourceType":"competition"},{"sourceId":7445922,"sourceType":"datasetVersion","datasetId":4333969}],"dockerImageVersionId":28450,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Hamster Autoencoder\n\nBazat pe https://www.kaggle.com/code/cdeotte/dog-autoencoder#How-is-LB-scored\n\n# Dataset\n\nDin https://images.cv/dataset/hamster-image-classification-dataset","metadata":{}},{"cell_type":"code","source":"ComputeLB = False\n\nimport os, gc, zipfile\nimport numpy as np, pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\n# if ComputeLB: PATH = '../input/generative-dog-images/all-dogs/all-dogs/'\n# else: PATH = '../input/all-dogs/all-dogs/'\n\nPATH = '/kaggle/input/hamters/for-kaggle-all-hamsters/'\n\nIMAGES = os.listdir(PATH)\n\nnum_img = len(IMAGES)\n\nprint('There are',num_img,'images. Here are 5 example filesnames:')\nprint(IMAGES[:5])\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-21T08:57:22.871878Z","iopub.execute_input":"2024-01-21T08:57:22.872272Z","iopub.status.idle":"2024-01-21T08:57:22.887366Z","shell.execute_reply.started":"2024-01-21T08:57:22.872202Z","shell.execute_reply":"2024-01-21T08:57:22.886543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isdir('../tmp'):\n    os.mkdir('../tmp')\nif not os.path.isdir('../tmp/images'):\n    os.mkdir('../tmp/images')\n\n# CREATE RANDOMLY CROPPED IMAGES\nfor i in range(num_img):\n    img = Image.open(PATH + IMAGES[i%num_img])\n    img = img.resize(( 100,int(img.size[1]/(img.size[0]/100) )), Image.ANTIALIAS)\n    w = img.size[0]; h = img.size[1]; a=0; b=0\n    if w>64: a = np.random.randint(0,w-64)\n    if h>64: b = np.random.randint(0,h-64)\n    img = img.crop((a, b, 64+a, 64+b))\n    img.save('../tmp/images/'+str(i)+'.png','PNG')\n    if i%num_img==0: print('created',i,'cropped images')\nprint(f'created {num_img} cropped images')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-21T08:57:22.889027Z","iopub.execute_input":"2024-01-21T08:57:22.889265Z","iopub.status.idle":"2024-01-21T08:57:27.689324Z","shell.execute_reply.started":"2024-01-21T08:57:22.889218Z","shell.execute_reply":"2024-01-21T08:57:27.688379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Autoencoder\nThis code is inspired by Keras' tutorial [here][1]\n\n[1]: https://blog.keras.io/building-autoencoders-in-keras.html","metadata":{}},{"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler\n\nBATCH_SIZE = 256; EPOCHS = 50\ntrain_datagen = ImageDataGenerator(rescale=1./255)\ntrain_batches = train_datagen.flow_from_directory('../tmp/',\n        target_size=(64,64), shuffle=True, class_mode='input', batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T08:57:27.690717Z","iopub.execute_input":"2024-01-21T08:57:27.690994Z","iopub.status.idle":"2024-01-21T08:57:27.803321Z","shell.execute_reply.started":"2024-01-21T08:57:27.690942Z","shell.execute_reply":"2024-01-21T08:57:27.802616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ENCODER\ninput_img = Input(shape=(64, 64, 3))  \nx = Conv2D(48, (3, 3), activation='relu', padding='same')(input_img)\nx = MaxPooling2D((2, 2), padding='same')(x)\nx = Conv2D(96, (3, 3), activation='relu', padding='same')(x)\nx = MaxPooling2D((2, 2), padding='same')(x)\nx = Conv2D(192, (3, 3), activation='relu', padding='same')(x)\nx = MaxPooling2D((2, 2), padding='same')(x)\nencoded = Conv2D(32, (1, 1), activation='relu', padding='same')(x)\n\n# LATENT SPACE\nlatentSize = (8,8,32)\n\n# DECODER\ndirect_input = Input(shape=latentSize)\nx = Conv2D(192, (1, 1), activation='relu', padding='same')(direct_input)\nx = UpSampling2D((2, 2))(x)\nx = Conv2D(192, (3, 3), activation='relu', padding='same')(x)\nx = UpSampling2D((2, 2))(x)\nx = Conv2D(96, (3, 3), activation='relu', padding='same')(x)\nx = UpSampling2D((2, 2))(x)\nx = Conv2D(48, (3, 3), activation='relu', padding='same')(x)\ndecoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(x)\n\n# COMPILE\nencoder = Model(input_img, encoded)\ndecoder = Model(direct_input, decoded)\nautoencoder = Model(input_img, decoder(encoded))\n\nautoencoder.compile(optimizer='Adam', loss='binary_crossentropy')","metadata":{"execution":{"iopub.status.busy":"2024-01-21T08:57:27.806266Z","iopub.execute_input":"2024-01-21T08:57:27.806659Z","iopub.status.idle":"2024-01-21T08:57:28.030419Z","shell.execute_reply.started":"2024-01-21T08:57:27.806484Z","shell.execute_reply":"2024-01-21T08:57:28.029765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Autoencoder\n\nAm mărit numărul de epoci la 50, sperând la rezultate mai clare. Loss ~= 0.56","metadata":{}},{"cell_type":"code","source":"history = autoencoder.fit_generator(train_batches,\n        steps_per_epoch = train_batches.samples // BATCH_SIZE,\n        epochs = EPOCHS, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T08:58:10.631600Z","iopub.execute_input":"2024-01-21T08:58:10.632074Z","iopub.status.idle":"2024-01-21T08:58:43.253459Z","shell.execute_reply.started":"2024-01-21T08:58:10.631873Z","shell.execute_reply":"2024-01-21T08:58:43.252722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View Reconstruction\nOur encoder works by mapping images from 12288 dimensional space (64 x 64 x 3) into 2048 dimensional space (latent image). This is a 6x compression rate. Our decoder works by mapping our latent image back into 12288 dimensional space. Below are examples. (Note that a decoder is like a GAN generator).","metadata":{}},{"cell_type":"code","source":"images = next(iter(train_batches))[0]\nfor i in range(5):\n\n    plt.figure(figsize=(15,5))\n    plt.subplot(1,3,1)\n    \n    # ORIGINAL IMAGE\n    orig = images[i,:,:,:].reshape((-1,64,64,3))\n    img = Image.fromarray( (255*orig).astype('uint8').reshape((64,64,3)))\n    plt.title('Original')\n    plt.imshow(img)\n\n    # LATENT IMAGE\n    latent_img = encoder.predict(orig)\n    mx = np.max( latent_img[0] )\n    mn = np.min( latent_img[0] )\n    latent_flat = ((latent_img[0] - mn) * 255/(mx - mn)).flatten(order='F')\n    img = Image.fromarray( latent_flat[:2025].astype('uint8').reshape((45,45)), mode='L') \n    plt.subplot(1,3,2)\n    plt.title('Latent')\n    plt.xlim((-10,55))\n    plt.ylim((-10,55))\n    plt.axis('off')\n    plt.imshow(img)\n\n    # RECONSTRUCTED IMAGE\n    decoded_imgs = decoder.predict(latent_img[0].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.subplot(1,3,3)\n    plt.title('Reconstructed')\n    plt.imshow(img)\n    \n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-21T08:58:54.393551Z","iopub.execute_input":"2024-01-21T08:58:54.393902Z","iopub.status.idle":"2024-01-21T08:58:57.992006Z","shell.execute_reply.started":"2024-01-21T08:58:54.393841Z","shell.execute_reply":"2024-01-21T08:58:57.990919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Latent Space and Hamster Generation\nWe choose the size of latent space. In our autoencoder, we chose for latent space to be 2048 dimensional (6x compression). If we map all 20,000 images into latent space, they would cluster inside a 2048 dimensional hyperellipsoid. That ellipsoid would represent dog images. (Cat images would form a different ellipsoid). Below I have plotted 256 of our dog images in latent space as blue dots and drew their ellipsoid. (Note that latent space is being projected onto 2D for display in this kernel). (Note because of ReLU, you may see clipping).\n\nIf we would like to generate a new dog image, we can chose a new random point (different from existing training image dots) inside this ellipsoid and then decode it. For example, we could choose the 9 red points below and then convert them into dog images.","metadata":{}},{"cell_type":"code","source":"from matplotlib.patches import Ellipse\n\n# PROJECT LATENT INTO 2D, AVOID DEAD RELU\nlatent_img = encoder.predict(images)\nlatent_img2 = latent_img.reshape((-1,latentSize[0]*latentSize[1]*latentSize[2]))\nd = 0; s = 0\nwhile s<0.1:\n    x = latent_img2[:,d]\n    s = np.std(x); d += 1\ns = 0\nwhile s<0.1:\n    y = latent_img2[:,d]\n    s = np.std(y); d += 1\n\n# CALCULATE ELLIPSOID FROM 256 IMAGES\ncov = np.cov(x, y)\nlambda_, v = np.linalg.eig(cov)\nlambda_ = np.sqrt(lambda_)\nfor j in [1,2,3]:\n    ell = Ellipse(xy=(np.mean(x), np.mean(y)), width=lambda_[0]*j*2, \n            height=lambda_[1]*j*2, angle=np.rad2deg(np.arccos(v[0, 0])))\n    ell.set_facecolor('None')\n    ell.set_edgecolor('black')\n    plt.gca().add_artist(ell)\n    \n# PLOT 256 IMAGES AS DOTS IN LATENT SPACE\nplt.scatter(x,y)\nd = np.random.multivariate_normal([np.mean(x),np.mean(y)],cov,9)\nplt.scatter(d[:,0],d[:,1],color='red',s=100)\nplt.title('Dog Images form an Ellipsoid in Latent Space')\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-21T08:59:06.348249Z","iopub.execute_input":"2024-01-21T08:59:06.348573Z","iopub.status.idle":"2024-01-21T08:59:06.680739Z","shell.execute_reply.started":"2024-01-21T08:59:06.348526Z","shell.execute_reply":"2024-01-21T08:59:06.679475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE num_img CROPPED IMAGES\nx = np.random.choice(np.arange(num_img),num_img)\nimages = np.zeros((num_img,64,64,3))\n\n# print(x)\n\nfor i in range(len(x)):\n    img = Image.open(PATH + IMAGES[x[i]])\n    img = img.resize((100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)\n    img = img.crop((18, 0, 82, 64))\n    images[i,:,:,:] = np.asarray(img).astype('float32') / 255.\n    #if i%1000==0: print(i)\n        \n# CALCULATE ELLIPSOID FROM num_img IMAGES        \nencoded_imgs = encoder.predict(images)\nsz = latentSize[0] * latentSize[1] * latentSize[2]\nencoded_imgs = encoded_imgs.reshape((-1,sz))\nmm = np.mean(encoded_imgs,axis=0)\nss = np.cov(encoded_imgs,rowvar=False)\n\n# GENERATE 9 RANDOM HAMSTER IMAGES\ngenerated = np.random.multivariate_normal(mm,ss,9)\ngenerated = generated.reshape((-1,latentSize[0],latentSize[1],latentSize[2]))","metadata":{"execution":{"iopub.status.busy":"2024-01-21T08:57:32.715089Z","iopub.status.idle":"2024-01-21T08:57:32.715633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PLOT 9 RANDOM HAMSTER IMAGES\nfor k in range(3):\n    plt.figure(figsize=(15,5))\n    plt.subplot(1,3,1)\n    decoded_imgs = decoder.predict(generated[k*3].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,2)\n    decoded_imgs = decoder.predict(generated[k*3+1].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,3)\n    decoded_imgs = decoder.predict(generated[k*3+2].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-21T08:59:12.693181Z","iopub.execute_input":"2024-01-21T08:59:12.693494Z","iopub.status.idle":"2024-01-21T08:59:13.877799Z","shell.execute_reply.started":"2024-01-21T08:59:12.693447Z","shell.execute_reply":"2024-01-21T08:59:13.877172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How is LB scored?\nWhat is the FID (Fréchet Inception Distance) metric? Now that we understand latent space, the FID metric is easy to understand. After we train our autoencoder, we can compute the latent hyper-ellipsoid of our dog training data. Then if someone generates dog images, we could feed them into our autoencoder and calculate the latent hyper-ellipsoid of their generated dog images. Then FID is basically the distance between the two latent hyper-ellipsoids squared.\n\nLet `e1` be the training dog ellipsoid and `e2` be the generated dog ellipsoid. Let `x0, x1, ..., x2047` be the 2048 features of latent space. Each vector `x_k` from training dataset `e1` has approx 20000 values, and `e2` has 10000. Then the distance between the ellipsoids is approx the sum of the difference of means (ellipse centers) and sum of the difference of standard deviations (ellipse widths) of each 2048 `x_k`, i.e. `(mean(x_k)_e1 - mean(x_k)_e2)` plus `(std(x_k)_e1 - std(x_k)_e2)` for each of the 2048 `k` values. Finally `FID = this sum^2`.\n\n# Generating Better Dog Images\nThe generated dog images above are not very recognizable. Instead of choosing completely random points in latent space, we can choose points near existing training images. Let's try that.","metadata":{}},{"cell_type":"code","source":"# DISTANCE TO MOVE AWAY FROM EXISTING TRAIN IMAGES\nbeta = 0.35\n# GENERATE 9 RANDOM HAMSTER IMAGES\ngenerated = np.random.multivariate_normal(mm,ss,9)\ngenerated = beta*generated + (1-beta)*encoded_imgs[:9]","metadata":{"execution":{"iopub.status.busy":"2024-01-21T08:59:30.487544Z","iopub.execute_input":"2024-01-21T08:59:30.488114Z","iopub.status.idle":"2024-01-21T08:59:33.084330Z","shell.execute_reply.started":"2024-01-21T08:59:30.487888Z","shell.execute_reply":"2024-01-21T08:59:33.083054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in range(3):\n    plt.figure(figsize=(15,5))\n    plt.subplot(1,3,1)\n    decoded_imgs = decoder.predict(generated[k*3].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,2)\n    decoded_imgs = decoder.predict(generated[k*3+1].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,3)\n    decoded_imgs = decoder.predict(generated[k*3+2].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-21T08:59:33.086852Z","iopub.execute_input":"2024-01-21T08:59:33.087657Z","iopub.status.idle":"2024-01-21T08:59:34.335051Z","shell.execute_reply.started":"2024-01-21T08:59:33.087577Z","shell.execute_reply":"2024-01-21T08:59:34.334212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beta = 0.2\n# GENERATE num_img RANDOM DOG IMAGES FOR KAGGLE\ngenerated = np.random.multivariate_normal(mm,ss,num_img)\nencoded_imgs = beta*generated + (1-beta)*encoded_imgs\ndecoded_imgs = decoder.predict(encoded_imgs.reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\ndecoded_imgs.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-21T08:59:36.531786Z","iopub.execute_input":"2024-01-21T08:59:36.532094Z","iopub.status.idle":"2024-01-21T08:59:39.490144Z","shell.execute_reply.started":"2024-01-21T08:59:36.532051Z","shell.execute_reply":"2024-01-21T08:59:39.488900Z"},"trusted":true},"execution_count":null,"outputs":[]}]}