{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":7454367,"sourceType":"datasetVersion","datasetId":4338928}],"dockerImageVersionId":28450,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Sharks Autoencoder\nIn this kernel, we learn about autoencoders. By understanding autoencoders, we will better understand GANs (Generative Adversarial Networks) and VAEs (Variational Autoencoders). We will also learn how to use an autoencoder to generate images of dogs.\n  \n![image](http://playagricola.com/Kaggle/ae7119.jpg)\n  \n\n# Load Data and Augment\nWe will randomly crop the original 20,000 images and make 500,000 new training images.","metadata":{}},{"cell_type":"code","source":"ComputeLB = False\n\nimport os, gc, zipfile\nimport numpy as np, pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nif ComputeLB: PATH = '../input/sharks/'\nelse: PATH = '../input/sharks/images.cv_fcm70hkcygb1t2j99zi589/data/train/tiger_shark/'\nIMAGES = os.listdir(PATH)\nnr_images = len(IMAGES)\nprint('There are',nr_images,'images. Here are example filesnames:')\nprint(IMAGES[:5])","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-22T10:22:21.192187Z","iopub.execute_input":"2024-01-22T10:22:21.192507Z","iopub.status.idle":"2024-01-22T10:22:21.365606Z","shell.execute_reply.started":"2024-01-22T10:22:21.192463Z","shell.execute_reply":"2024-01-22T10:22:21.364807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isdir('../tmp'):\n    os.mkdir('../tmp')\nif not os.path.isdir('../tmp/images'):\n    os.mkdir('../tmp/images')\n\n# CREATE RANDOMLY CROPPED IMAGES\nfor i in range(nr_images):\n    img = Image.open(PATH + IMAGES[i%nr_images])\n    img = img.resize(( 100,int(img.size[1]/(img.size[0]/100) )), Image.ANTIALIAS)\n    w = img.size[0]; h = img.size[1]; a=0; b=0\n    if w>64: a = np.random.randint(0,w-64)\n    if h>64: b = np.random.randint(0,h-64)\n    img = img.crop((a, b, 64+a, 64+b))\n    img.save('../tmp/images/'+str(i)+'.png','PNG')\n    if i%nr_images==0: print('created',i,'cropped images')\nprint('There are', nr_images, 'cropped images')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-22T10:22:25.22161Z","iopub.execute_input":"2024-01-22T10:22:25.221955Z","iopub.status.idle":"2024-01-22T10:22:28.52302Z","shell.execute_reply.started":"2024-01-22T10:22:25.221894Z","shell.execute_reply":"2024-01-22T10:22:28.522344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Autoencoder\nThis code is inspired by Keras' tutorial [here][1]\n\n[1]: https://blog.keras.io/building-autoencoders-in-keras.html","metadata":{}},{"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler\n\nBATCH_SIZE = 256; EPOCHS = 10\ntrain_datagen = ImageDataGenerator(rescale=1./255)\ntrain_batches = train_datagen.flow_from_directory('../tmp/',\n        target_size=(64,64), shuffle=True, class_mode='input', batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:22:45.27919Z","iopub.execute_input":"2024-01-22T10:22:45.279521Z","iopub.status.idle":"2024-01-22T10:22:46.852726Z","shell.execute_reply.started":"2024-01-22T10:22:45.279465Z","shell.execute_reply":"2024-01-22T10:22:46.851897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ENCODER\ninput_img = Input(shape=(64, 64, 3))  \nx = Conv2D(48, (3, 3), activation='relu', padding='same')(input_img)\nx = MaxPooling2D((2, 2), padding='same')(x)\nx = Conv2D(96, (3, 3), activation='relu', padding='same')(x)\nx = MaxPooling2D((2, 2), padding='same')(x)\nx = Conv2D(192, (3, 3), activation='relu', padding='same')(x)\nx = MaxPooling2D((2, 2), padding='same')(x)\nencoded = Conv2D(32, (1, 1), activation='relu', padding='same')(x)\n\n# LATENT SPACE\nlatentSize = (8,8,32)\n\n# DECODER\ndirect_input = Input(shape=latentSize)\nx = Conv2D(192, (1, 1), activation='relu', padding='same')(direct_input)\nx = UpSampling2D((2, 2))(x)\nx = Conv2D(192, (3, 3), activation='relu', padding='same')(x)\nx = UpSampling2D((2, 2))(x)\nx = Conv2D(96, (3, 3), activation='relu', padding='same')(x)\nx = UpSampling2D((2, 2))(x)\nx = Conv2D(48, (3, 3), activation='relu', padding='same')(x)\ndecoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(x)\n\n# COMPILE\nencoder = Model(input_img, encoded)\ndecoder = Model(direct_input, decoded)\nautoencoder = Model(input_img, decoder(encoded))\n\nautoencoder.compile(optimizer='Adam', loss='binary_crossentropy')","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:22:56.445075Z","iopub.execute_input":"2024-01-22T10:22:56.445522Z","iopub.status.idle":"2024-01-22T10:22:56.715521Z","shell.execute_reply.started":"2024-01-22T10:22:56.445458Z","shell.execute_reply":"2024-01-22T10:22:56.71487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Autoencoder","metadata":{}},{"cell_type":"code","source":"history = autoencoder.fit_generator(train_batches,\n        steps_per_epoch = train_batches.samples // BATCH_SIZE,\n        epochs = EPOCHS, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:23:07.613298Z","iopub.execute_input":"2024-01-22T10:23:07.613636Z","iopub.status.idle":"2024-01-22T10:23:20.052579Z","shell.execute_reply.started":"2024-01-22T10:23:07.613589Z","shell.execute_reply":"2024-01-22T10:23:20.05168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View Reconstruction\nOur encoder works by mapping images from 12288 dimensional space (64 x 64 x 3) into 2048 dimensional space (latent image). This is a 6x compression rate. Our decoder works by mapping our latent image back into 12288 dimensional space. Below are examples. (Note that a decoder is like a GAN generator).","metadata":{}},{"cell_type":"code","source":"images = next(iter(train_batches))[0]\nfor i in range(5):\n\n    plt.figure(figsize=(15,5))\n    plt.subplot(1,3,1)\n    \n    # ORIGINAL IMAGE\n    orig = images[i,:,:,:].reshape((-1,64,64,3))\n    img = Image.fromarray( (255*orig).astype('uint8').reshape((64,64,3)))\n    plt.title('Original')\n    plt.imshow(img)\n\n    # LATENT IMAGE\n    latent_img = encoder.predict(orig)\n    mx = np.max( latent_img[0] )\n    mn = np.min( latent_img[0] )\n    latent_flat = ((latent_img[0] - mn) * 255/(mx - mn)).flatten(order='F')\n    img = Image.fromarray( latent_flat[:2025].astype('uint8').reshape((45,45)), mode='L') \n    plt.subplot(1,3,2)\n    plt.title('Latent')\n    plt.xlim((-10,55))\n    plt.ylim((-10,55))\n    plt.axis('off')\n    plt.imshow(img)\n\n    # RECONSTRUCTED IMAGE\n    decoded_imgs = decoder.predict(latent_img[0].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.subplot(1,3,3)\n    plt.title('Reconstructed')\n    plt.imshow(img)\n    \n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-22T10:23:55.491004Z","iopub.execute_input":"2024-01-22T10:23:55.491359Z","iopub.status.idle":"2024-01-22T10:23:57.916113Z","shell.execute_reply.started":"2024-01-22T10:23:55.491305Z","shell.execute_reply":"2024-01-22T10:23:57.915265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Latent Space and Shark Generation\n","metadata":{}},{"cell_type":"code","source":"from matplotlib.patches import Ellipse\n\n# PROJECT LATENT INTO 2D, AVOID DEAD RELU\nlatent_img = encoder.predict(images)\nlatent_img2 = latent_img.reshape((-1,latentSize[0]*latentSize[1]*latentSize[2]))\nd = 0; s = 0\nwhile s<0.1:\n    x = latent_img2[:,d]\n    s = np.std(x); d += 1\ns = 0\nwhile s<0.1:\n    y = latent_img2[:,d]\n    s = np.std(y); d += 1\n\n# CALCULATE ELLIPSOID FROM 256 IMAGES\ncov = np.cov(x, y)\nlambda_, v = np.linalg.eig(cov)\nlambda_ = np.sqrt(lambda_)\nfor j in [1,2,3]:\n    ell = Ellipse(xy=(np.mean(x), np.mean(y)), width=lambda_[0]*j*2, \n            height=lambda_[1]*j*2, angle=np.rad2deg(np.arccos(v[0, 0])))\n    ell.set_facecolor('None')\n    ell.set_edgecolor('black')\n    plt.gca().add_artist(ell)\n    \n# PLOT 256 IMAGES AS DOTS IN LATENT SPACE\nplt.scatter(x,y)\nd = np.random.multivariate_normal([np.mean(x),np.mean(y)],cov,9)\nplt.scatter(d[:,0],d[:,1],color='red',s=100)\nplt.title('Shark Images form an Ellipsoid in Latent Space')\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-22T10:24:46.290618Z","iopub.execute_input":"2024-01-22T10:24:46.290936Z","iopub.status.idle":"2024-01-22T10:24:46.608524Z","shell.execute_reply.started":"2024-01-22T10:24:46.290889Z","shell.execute_reply":"2024-01-22T10:24:46.607463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE 10000 CROPPED IMAGES\nx = np.random.choice(np.arange(nr_images),nr_images)\nimages = np.zeros((nr_images,64,64,3))\nfor i in range(len(x)):\n    img = Image.open(PATH + IMAGES[x[i]])\n    img = img.resize((100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)\n    img = img.crop((18, 0, 82, 64))\n    images[i,:,:,:] = np.asarray(img).astype('float32') / 255.\n    #if i%1000==0: print(i)\n        \n# CALCULATE ELLIPSOID FROM nr_images IMAGES        \nencoded_imgs = encoder.predict(images)\nsz = latentSize[0] * latentSize[1] * latentSize[2]\nencoded_imgs = encoded_imgs.reshape((-1,sz))\nmm = np.mean(encoded_imgs,axis=0)\nss = np.cov(encoded_imgs,rowvar=False)\n\n# GENERATE 9 RANDOM SHARK IMAGES\ngenerated = np.random.multivariate_normal(mm,ss,9)\ngenerated = generated.reshape((-1,latentSize[0],latentSize[1],latentSize[2]))","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:26:53.742652Z","iopub.execute_input":"2024-01-22T10:26:53.742952Z","iopub.status.idle":"2024-01-22T10:26:57.680939Z","shell.execute_reply.started":"2024-01-22T10:26:53.74291Z","shell.execute_reply":"2024-01-22T10:26:57.679714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PLOT 9 RANDOM SHARK IMAGES\nfor k in range(3):\n    plt.figure(figsize=(15,5))\n    plt.subplot(1,3,1)\n    decoded_imgs = decoder.predict(generated[k*3].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,2)\n    decoded_imgs = decoder.predict(generated[k*3+1].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,3)\n    decoded_imgs = decoder.predict(generated[k*3+2].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-22T10:27:17.97736Z","iopub.execute_input":"2024-01-22T10:27:17.977663Z","iopub.status.idle":"2024-01-22T10:27:19.838925Z","shell.execute_reply.started":"2024-01-22T10:27:17.977622Z","shell.execute_reply":"2024-01-22T10:27:19.837487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How is LB scored?\nWhat is the FID (Fréchet Inception Distance) metric? Now that we understand latent space, the FID metric is easy to understand. After we train our autoencoder, we can compute the latent hyper-ellipsoid of our dog training data. Then if someone generates dog images, we could feed them into our autoencoder and calculate the latent hyper-ellipsoid of their generated dog images. Then FID is basically the distance between the two latent hyper-ellipsoids squared.\n\nLet `e1` be the training dog ellipsoid and `e2` be the generated dog ellipsoid. Let `x0, x1, ..., x2047` be the 2048 features of latent space. Each vector `x_k` from training dataset `e1` has approx 20000 values, and `e2` has 10000. Then the distance between the ellipsoids is approx the sum of the difference of means (ellipse centers) and sum of the difference of standard deviations (ellipse widths) of each 2048 `x_k`, i.e. `(mean(x_k)_e1 - mean(x_k)_e2)` plus `(std(x_k)_e1 - std(x_k)_e2)` for each of the 2048 `k` values. Finally `FID = this sum^2`.\n\nIn the image below, `FID approx = ||c2-c1||^2 + (h2-h1)^2 + (w2-w1)^2`. (Note this is exact when features are uncorrelated). MiFID adds a multiplier penalty if your dog images are too similar to the training data. `MiFID = FID * penalty`.\n\n![image](http://playagricola.com/Kaggle/ellipsesb7119.jpg)\n\n# Generating Better Shark Images\nThe generated dog images above are not very recognizable. Instead of choosing completely random points in latent space, we can choose points near existing training images. Let's try that.","metadata":{}},{"cell_type":"code","source":"# DISTANCE TO MOVE AWAY FROM EXISTING TRAIN IMAGES\nbeta = 0.35\n# GENERATE 9 RANDOM SHARK IMAGES\ngenerated = np.random.multivariate_normal(mm,ss,9)\ngenerated = beta*generated + (1-beta)*encoded_imgs[:9]","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:27:59.729499Z","iopub.execute_input":"2024-01-22T10:27:59.729958Z","iopub.status.idle":"2024-01-22T10:28:02.44477Z","shell.execute_reply.started":"2024-01-22T10:27:59.729886Z","shell.execute_reply":"2024-01-22T10:28:02.443267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in range(3):\n    plt.figure(figsize=(15,5))\n    plt.subplot(1,3,1)\n    decoded_imgs = decoder.predict(generated[k*3].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,2)\n    decoded_imgs = decoder.predict(generated[k*3+1].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.subplot(1,3,3)\n    decoded_imgs = decoder.predict(generated[k*3+2].reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\n    img = Image.fromarray( (255*decoded_imgs[0]).astype('uint8').reshape((64,64,3)))\n    plt.imshow(img)\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-22T10:28:11.296802Z","iopub.execute_input":"2024-01-22T10:28:11.297105Z","iopub.status.idle":"2024-01-22T10:28:13.085395Z","shell.execute_reply.started":"2024-01-22T10:28:11.297061Z","shell.execute_reply":"2024-01-22T10:28:13.084203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beta = 0.2\n# GENERATE nr_images RANDOM DOG IMAGES FOR KAGGLE\ngenerated = np.random.multivariate_normal(mm,ss,nr_images)\nencoded_imgs = beta*generated + (1-beta)*encoded_imgs\ndecoded_imgs = decoder.predict(encoded_imgs.reshape((-1,latentSize[0],latentSize[1],latentSize[2])))\ndecoded_imgs.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:28:30.579198Z","iopub.execute_input":"2024-01-22T10:28:30.579493Z","iopub.status.idle":"2024-01-22T10:28:33.762946Z","shell.execute_reply.started":"2024-01-22T10:28:30.579451Z","shell.execute_reply":"2024-01-22T10:28:33.76191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}