{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image, ImageOps \n\n\nsize = 64, 64\n#size = 256, 256\nfor f in os.listdir('/kaggle/input/pokemon-images-dataset/pokemon_jpg/pokemon_jpg'):\n    im = Image.open('/kaggle/input/pokemon-images-dataset/pokemon_jpg/pokemon_jpg/'+f).resize(size, Image.ANTIALIAS)\n    break\n    \n    \nbig_arr = np.array([np.array(im)]).reshape(1, 64, 64, 3)\nprint(big_arr.shape)\ni = 0\nfor f in os.listdir('/kaggle/input/pokemon-images-dataset/pokemon_jpg/pokemon_jpg')[1:]:\n    big_arr = np.append(big_arr, [np.array(Image.open('/kaggle/input/pokemon-images-dataset/pokemon_jpg/pokemon_jpg/'+f).resize(size, Image.ANTIALIAS)).reshape(64, 64, 3) ], axis=0)\n    i += 1\n    \narr = np.array(im)\nprint(arr.shape)\nplt.imshow(big_arr[3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"big_arr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers import LeakyReLU\nimport tensorflow as tf\n    \n\nunits = 128\nbottleneck_units = 64\nrestoration_units = 256\nregularizer_weight = 0\ninput_shape = [64, 64, 3]\n\n\nclass BottleneckPriorScale(keras.layers.Layer):\n    def __init__(self, bottleneck_units, units, regularizer_weight):\n        super(BottleneckPriorScale, self).__init__()\n        #self.layer1 = layers.Conv2D(bottleneck_units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding='same') #no activation\n        #self.layer2 = layers.Conv2D(units, kernel_size=(3, 3), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n        #activations after every conv!!!!\n        \n        self.layer5 = layers.Conv2D(bottleneck_units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n        self.layera = layers.LeakyReLU()\n        self.layerb = layers.Dropout(.3)\n        self.layerc = layers.BatchNormalization(renorm=True)\n        \n        self.layer8 = layers.Conv2D(units, kernel_size=(3, 3), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n        self.layer9 = layers.LeakyReLU()\n        self.layer10 = layers.Dropout(.3)\n        self.layer11 = layers.BatchNormalization(renorm=True)\n        self.layer12 = layers.Conv2D(bottleneck_units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding='same')\n        \n        self.layerd = layers.LeakyReLU()\n        self.layere = layers.Dropout(.3)\n        self.layerf = layers.BatchNormalization(renorm=True)\n        self.layer13 = layers.Conv2D(units, kernel_size=(3, 3), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n\n        #self.layer16 = layers.Conv2D(bottleneck_units, kernel_size=(1, 3), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n\n        #self.layer19 = layers.Conv2D(units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n        self.layer20 = layers.LeakyReLU()\n        self.layer21 = layers.Dropout(.3)\n        self.layer22 = layers.BatchNormalization(renorm=True)\n\n    def call(self, inputs):\n        #x = self.layer1(inputs)\n        #x = self.layer2(x)\n\n        x = self.layer5(inputs)\n        x = self.layera(x)\n        x = self.layerb(x)\n        x = self.layerc(x)\n\n        x = self.layer8(x)\n        x = self.layer9(x)\n        x = self.layer10(x)\n        x = self.layer11(x)\n        x = self.layer12(x)\n        x = self.layerd(x)\n        x = self.layere(x)\n        x = self.layerf(x)\n        \n        x = self.layer13(x)\n\n        #x = self.layer16(x)\n\n        #x = self.layer19(x)\n        x = self.layer20(x)\n        x = self.layer21(x)\n        x = self.layer22(x)\n        \n        return x\n\n    \n    \nimage_reconstruction_64 = keras.Sequential([\n    layers.BatchNormalization(input_shape=input_shape, renorm=True),\n    layers.Conv2D(units, kernel_size=(3, 3), input_shape=input_shape, padding=\"same\"),\n\n    \n    BottleneckPriorScale(bottleneck_units, units*2, regularizer_weight),\n    layers.MaxPooling2D(),\n    \n    #Projection!\n    layers.Conv2D(units*8, kernel_size=(1, 1), padding=\"same\"),\n    layers.LeakyReLU(),\n    layers.BatchNormalization(renorm=True),\n    BottleneckPriorScale(bottleneck_units, units*4, regularizer_weight),\n    layers.MaxPooling2D(),\n    \n    #Projection!\n    layers.Conv2D(units*16, kernel_size=(1, 1),padding=\"same\"),\n    layers.LeakyReLU(),\n    layers.BatchNormalization(renorm=True),\n    BottleneckPriorScale(bottleneck_units, units*4, regularizer_weight),\n    layers.MaxPooling2D(),\n    \n    #Projection!\n    layers.Conv2D(units*16, kernel_size=(1, 1), padding=\"same\"),\n    layers.LeakyReLU(),\n    layers.BatchNormalization(renorm=True),\n    BottleneckPriorScale(bottleneck_units, units*8, regularizer_weight),\n    layers.MaxPooling2D(),\n\n    #Projection!\n    layers.Conv2D(units*16, kernel_size=(1, 1), padding=\"same\"),\n    layers.LeakyReLU(),\n    layers.BatchNormalization(renorm=True),\n    BottleneckPriorScale(bottleneck_units, units*8, regularizer_weight),\n    layers.MaxPooling2D(),\n    \n    BottleneckPriorScale(bottleneck_units, units*16, regularizer_weight),\n    layers.MaxPooling2D(),\n    \n    #BottleneckPriorScale(bottleneck_units, units*128, regularizer_weight),\n    \n    #BottleneckPriorScale(bottleneck_units, units*128, regularizer_weight),\n\n    \n    #layers.Dense(units*16),\n    #layers.LeakyReLU(),\n    #layers.BatchNormalization(renorm=True),\n    layers.Dense(units*32, name=\"half\"),\n    layers.LeakyReLU(),\n    layers.BatchNormalization(renorm=True),\n    layers.Dense(1024),\n    layers.LeakyReLU(),\n    layers.BatchNormalization(renorm=True),\n    \n    layers.Dense(64*64*3),\n    layers.LeakyReLU(),\n    layers.Reshape(input_shape, name=\"decoder_output\")\n    \n   # layers.Conv2D(input_shape[-1], (1, 1), strides=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight),\n   #               \n   #               #activation='relu', \n   #               activation=layers.LeakyReLU(), \n   #               \n   #               padding=\"same\", name='last_layer'),\n\n])\n\n\n\ndef reconstruction_loss_func(old, new):\n    #1 for each wrong pixel, harsh!\n    \n    reconstruction_loss = keras.losses.binary_crossentropy(tf.bitcast(tf.cast(old, tf.int8), tf.uint8), tf.bitcast(tf.cast(new, tf.int8), tf.uint8))\n    reconstruction_loss *= (256*256*3)\n    \n    return reconstruction_loss\n\ndef l1_loss(old, new):\n    return tf.reduce_mean(tf.cast(old, tf.float32)-new)\n\nimage_reconstruction_64.compile(\n    optimizer=tf.keras.optimizers.Adam(lr=0.01, clipnorm=1.0, clipvalue=.5),\n    #loss=reconstruction_loss_func,\n    loss='mse',\n    metrics=['accuracy']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#y = train_df.pop(\"label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\n\n\n\n#1x1 image_restoration B4 ACTIVATION??? 256->64->256->activation\n#would reduce computation (b/c I have the 3x3 do the restoring)\n\n\n#Conv2DTranspose SUCKS?????\n#Check!!!\n\n\n    \n#Reconstruction just immediately killed gradients???\n#Or Dying Relu problem\n\n#ADDED WAY MORE ACTIVATIONS\n#Too many local minimum, Too many (64*64)-1 examples of 1 pixel correct all others wrong\n#Similar to how I use to have all conv have activations\n\n\n#BAD IDEA TO ADD FILTERS AS THE WIDTH\n#THEY AREN'T RELATED TO HORIZONTAL POSITIONING\n\n\n#SEE IF YOU CAN OVERFIT FOR ONE DATA POINT\n\n#Renormalization!!!!!\n\n\n#TRY 5x5 in bottleneck class!\n\n#Activations IN BOTH increasing and decreasing of feature maps!!!!!\nhistory = image_reconstruction_64.fit(\n    #big_arr.astype('int32')[:, :, :, 0], big_arr.astype('int32')[:, :, :, 0],\n    big_arr, big_arr,\n    batch_size=32, #INCREASE??????????\n    epochs=3000, #MIGHT NEED ALOT OF THIS!!! \n        #Will need to decrease units though\n    \n    #callbacks=[EarlyStopping(monitor='train_loss')],\n    verbose=0\n)\npd.DataFrame(history.history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nMAKE POKEMON FROM NAME!!!\n\nTitle -> Image\n\nRandom Image -> Pokemon\n\nPCA on Encoding, get 2-dim representations to plot!\nColor points for specfic pokemon!! (Squirtle, Wartorle, and Blastoise)\n\nSquare the PCA indices in encoding, then DECODE (then try merging just those 2 VALUES)\n\nDivide dataset in half, give two random pokemon, make one random pokemon from OTHER DATASET\n\nlayer.Embeddings()\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"units = 256\nbottleneck_units = 128\nrestoration_units = 256\nregularizer_weight = 0\ninput_shape = [64, 64, 3]\n\nclass TransposeInto2(keras.layers.Layer):\n    def __init__(self, units, **kwargs):\n        super(TransposeInto2, self).__init__()\n        \n        self.units = units\n        \n        self.convReduceFilter1 = layers.Conv2D(bottleneck_units, (1, 1), activation=layers.LeakyReLU(), padding=\"same\")\n        self.convReduceFilter2 = layers.Conv2D(bottleneck_units, (1, 1), activation=layers.LeakyReLU(), padding=\"same\")\n        self.convReduceFilter3 = layers.Conv2D(bottleneck_units, (1, 1), activation=layers.LeakyReLU(), padding=\"same\")\n        self.convReduceFilter4 = layers.Conv2D(bottleneck_units, (1, 1), activation=layers.LeakyReLU(), padding=\"same\")\n        \n        self.FirstImageWidth = layers.Conv2DTranspose(self.units, (3, 1), strides=(2, 1), padding=\"same\", activation=layers.LeakyReLU())\n        self.FirstImageHeight = layers.Conv2DTranspose(self.units, (1, 3), strides=(1, 2), padding=\"same\", activation=layers.LeakyReLU())\n        \n        self.CheckSampling = layers.Conv2D(self.units, (3, 3), activation=layers.LeakyReLU(), padding='same')\n        self.CheckSampling2 = layers.Conv2D(self.units, (3, 3), activation=layers.LeakyReLU(), padding='same')\n        \n        self.BatchNorm3 = layers.BatchNormalization(renorm=batch_renorm)\n        self.BatchNorm4 = layers.BatchNormalization(renorm=batch_renorm)\n        self.BatchNorm5 = layers.BatchNormalization(renorm=batch_renorm)\n        self.BatchNorm6 = layers.BatchNormalization(renorm=batch_renorm)\n        self.BatchNorm7 = layers.BatchNormalization(renorm=batch_renorm)\n        self.BatchNorm8 = layers.BatchNormalization(renorm=batch_renorm)\n        self.BatchNorm9 = layers.BatchNormalization(renorm=batch_renorm)\n        self.BatchNorm10 = layers.BatchNormalization(renorm=batch_renorm)\n        \n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'units': self.units\n        })\n        return config\n        \n        \n    def call(self, inputs):\n        x = self.convReduceFilter1(inputs)\n        x = self.BatchNorm3(x)\n        \n        x = self.FirstImageWidth(x)\n        x = self.BatchNorm4(x)\n        \n        x = self.convReduceFilter2(x)\n        x = self.BatchNorm5(x)\n        \n        x = self.FirstImageHeight(x)\n        x = self.BatchNorm6(x)\n        \n        x = self.convReduceFilter3(x)\n        x = self.BatchNorm7(x)\n        x = self.CheckSampling(x)\n        x = self.BatchNorm8(x)\n        \n        x = self.convReduceFilter4(x)\n        x = self.BatchNorm9(x)\n        x = self.CheckSampling2(x)\n        x = self.BatchNorm10(x)\n        \n        return x\n    \nclass TransposeInto64(keras.layers.Layer):\n    def __init__(self, **kwargs):\n        super(TransposeInto64, self).__init__()\n        inside_units = units*8\n        self.TransposeInto2a = TransposeInto2(inside_units)\n        self.TransposeInto2b = TransposeInto2(inside_units//2)\n        self.TransposeInto2c = TransposeInto2(inside_units//4)\n        self.TransposeInto2d = TransposeInto2(inside_units//8)\n        self.TransposeInto2e = TransposeInto2(inside_units//16)\n        self.TransposeInto2f = TransposeInto2(inside_units//32)\n        \n        \n    def call(self, inputs):\n        x = self.TransposeInto2a(inputs)\n        x = self.TransposeInto2b(x)\n        x = self.TransposeInto2c(x)\n        x = self.TransposeInto2d(x)\n        x = self.TransposeInto2e(x)\n        x = self.TransposeInto2f(x)\n\n        return x\n\n\nclass BottleneckPriorScale(keras.layers.Layer):\n    def __init__(self, bottleneck_units, units, regularizer_weight, **kwargs):\n        super(BottleneckPriorScale, self).__init__()\n        #self.layer1 = layers.Conv2D(bottleneck_units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding='same') #no activation\n        #self.layer2 = layers.Conv2D(units, kernel_size=(3, 3), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n        #activations after every conv!!!!\n        \n        self.bottleneck_units = bottleneck_units\n        self.units = units\n        self.regularizer_weight = regularizer_weight\n        \n        self.layer5 = layers.Conv2D(self.bottleneck_units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(self.regularizer_weight), padding=\"same\")\n        self.layera = layers.LeakyReLU()\n        self.layerb = layers.Dropout(.3)\n        self.layerc = layers.BatchNormalization(renorm=batch_renorm)\n        \n        self.layer8 = layers.Conv2D(self.units, kernel_size=(3, 3), kernel_regularizer=tf.keras.regularizers.l2(self.regularizer_weight), padding=\"same\")\n        self.layer9 = layers.LeakyReLU()\n        self.layer10 = layers.Dropout(.3)\n        self.layer11 = layers.BatchNormalization(renorm=batch_renorm)\n        self.layer12 = layers.Conv2D(self.bottleneck_units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(self.regularizer_weight), padding='same')\n        \n        self.layerd = layers.LeakyReLU()\n        self.layere = layers.Dropout(.3)\n        self.layerf = layers.BatchNormalization(renorm=batch_renorm)\n        self.layer13 = layers.Conv2D(self.units, kernel_size=(3, 3), kernel_regularizer=tf.keras.regularizers.l2(self.regularizer_weight), padding=\"same\")\n\n        #self.layer16 = layers.Conv2D(bottleneck_units, kernel_size=(1, 3), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n\n        #self.layer19 = layers.Conv2D(units, kernel_size=(1, 1), kernel_regularizer=tf.keras.regularizers.l2(regularizer_weight), padding=\"same\")\n        self.layer20 = layers.LeakyReLU()\n        self.layer21 = layers.Dropout(.3)\n        self.layer22 = layers.BatchNormalization(renorm=batch_renorm)\n        \n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'bottleneck_units': self.bottleneck_units,\n            'units': self.units,\n            'regularizer_weight': self.regularizer_weight\n        })\n        return config\n\n    def call(self, inputs):\n        #x = self.layer1(inputs)\n        #x = self.layer2(x)\n\n        x = self.layer5(inputs)\n        x = self.layera(x)\n        x = self.layerb(x)\n        x = self.layerc(x)\n\n        x = self.layer8(x)\n        x = self.layer9(x)\n        x = self.layer10(x)\n        x = self.layer11(x)\n        x = self.layer12(x)\n        x = self.layerd(x)\n        x = self.layere(x)\n        x = self.layerf(x)\n        \n        x = self.layer13(x)\n\n        #x = self.layer16(x)\n\n        #x = self.layer19(x)\n        x = self.layer20(x)\n        x = self.layer21(x)\n        x = self.layer22(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_renorm = False\n\nimage_reconstruction_64 = keras.models.load_model(\"../input/2epochtpumodel/mymodel64.h5\",\n            custom_objects={'LeakyReLU': layers.LeakyReLU, 'BottleneckPriorScale':BottleneckPriorScale,\n                           'TransposeInto64':TransposeInto64, 'TransposeInto2':TransposeInto2})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Don't do compile! Loses either trajectory or all weights!\n\n#loss didn't change (batch size too high or too late?)\n\nhistory = image_reconstruction_64.fit(\n    big_arr, big_arr,\n    batch_size=32, \n    epochs=1,\n    verbose=0\n)\npd.DataFrame(history.history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Random image of a dog, not trained!\nsize = 64, 64\ninp = Image.open('../input/natural-images/natural_images/cat/cat_0001.jpg').resize(size, Image.ANTIALIAS)\nprint(np.array(inp).shape)\nplt.imshow(inp)\nplt.show()\nout = image_reconstruction_64.predict(np.array(inp).reshape(1, 64, 64, 3))\nplt.imshow(out[0].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\nall_layer_names = [l.name for l in image_reconstruction_64.layers]\n\nlayer_names = ['half', 'decoder_output']\nlayers = [image_reconstruction_64.get_layer(l).output for l in layer_names]\n\n\n\n\n\nencoder = tf.keras.Model(image_reconstruction_64.input, layers[0])\n\n\n\n\n\nidx = all_layer_names.index(\"half\")\ninput_shape = image_reconstruction_64.layers[idx+1].get_input_shape_at(0)[1:] # get the input shape of desired layer\nlayer_input = tf.keras.layers.Input(shape=input_shape) # a new input tensor to be able to feed the desired layer\nprint(input_shape)\n\n# create the new nodes for each layer in the path\nx = layer_input\nfor layer in image_reconstruction_64.layers[idx+1:]:\n    x = layer(x)\n\n# create the model\n\ndecoder = tf.keras.Model(layer_input, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encodings = encoder.predict(big_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"enc_arr = np.array([e[0][0] for e in encodings])\nenc_arr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.datasets import make_regression\nfrom sklearn.tree import DecisionTreeRegressor\nfrom matplotlib import pyplot\n\nX, y = make_regression(n_samples=len(big_arr), n_features=len(enc_arr[0]), n_informative=5, random_state=1)\n\nmodel = DecisionTreeRegressor()\n\nmodel.fit(X, y)\n\nimportance = model.feature_importances_\n\nfor i,v in enumerate(importance):\n\tprint('Feature: %0d, Score: %.5f' % (i,v))\n\npyplot.bar([x for x in range(len(importance))], importance)\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From here: https://stats.stackexchange.com/questions/108743/methods-in-r-or-python-to-perform-feature-selection-in-unsupervised-learning\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom collections import defaultdict\nfrom sklearn.metrics.pairwise import euclidean_distances\nfrom sklearn.preprocessing import StandardScaler\n\nclass PFA(object):\n    def __init__(self, n_features, q=None):\n        self.q = q\n        self.n_features = n_features\n\n    def fit(self, X):\n        if not self.q:\n            self.q = X.shape[1]\n\n        sc = StandardScaler()\n        X = sc.fit_transform(X)\n\n        pca = PCA(n_components=self.q).fit(X)\n        A_q = pca.components_.T\n\n        kmeans = KMeans(n_clusters=self.n_features).fit(A_q)\n        clusters = kmeans.predict(A_q)\n        cluster_centers = kmeans.cluster_centers_\n\n        dists = defaultdict(list)\n        for i, c in enumerate(clusters):\n            dist = euclidean_distances([A_q[i, :]], [cluster_centers[c, :]])[0][0]\n            dists[c].append((i, dist))\n\n        self.indices_ = [sorted(f, key=lambda x: x[1])[0][0] for f in dists.values()]\n        self.features_ = X[:, self.indices_]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\npfa = PFA(n_features=2, q=0.95)\npfa.fit(enc_arr)\n\n# To get the transformed matrix\nX = pfa.features_\n\n# To get the column indices of the kept features\ncolumn_indices = pfa.indices_\ncolumn_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\npfa = PFA(n_features=2)\npfa.fit(enc_arr.T)\n\n# To get the transformed matrix\nX = pfa.features_\n\n# To get the column indices of the kept features\nrow_indices = pfa.indices_\nrow_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Most important pictures\nplt.imshow(big_arr[row_indices[0]].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(big_arr[row_indices[1]].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#K-Means is non deterministics, I MAY NEED TO RUN IT A COUPLE OF TIMES\n\nfor e in enc_arr:\n    e[column_indices[0]] = e[column_indices[0]]**2\n    e[column_indices[1]] = e[column_indices[1]]**2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"changed_images = decoder.predict(enc_arr.reshape(-1, 1, 1, 4096))\nplt.imshow(changed_images[0].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"enc_arr = np.array([e[0][0] for e in encodings])\nenc_arr[0][55:200] = 1000\nenc_arr[0][column_indices[1]] = 77\nchanged_images = decoder.predict(enc_arr[0].reshape(-1, 1, 1, 4096))\nplt.imshow(changed_images[0].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(big_arr[27])\n#plt.show()\n#plt.imshow(big_arr[47])\n#plt.show()\nplt.imshow(big_arr[30])\nplt.show()\nplt.imshow(big_arr[150])\nplt.show()\nencodings = encoder.predict(np.array([big_arr[30], big_arr[150]]))\nprint(encodings.shape)\n#merged = ((encodings[0]+encodings[1]+encodings[2]+encodings[3])/4)\nmerged = ((encodings[0]*5+encodings[1])/6)\npred = decoder.predict(merged.reshape(1, 1, 1, -1))\n\nx_min = pred[0].min(axis=(1, 2), keepdims=True)\nx_max = pred[0].max(axis=(1, 2), keepdims=True)\n\np = (pred[0] - x_min)/(x_max-x_min)\nplt.imshow(p*1.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cubone_and_marok = [6, 7]\neggs_and_tor = [4, 5]\ncolor_arr = []\nplt.imshow(big_arr[4])\nplt.show()\nplt.imshow(big_arr[5])\nplt.show()\nfor i in range(len(dim_reduced)):\n    if i in cubone_and_marok:\n        color_arr.append(\"Orange\")\n    elif i in eggs_and_tor:\n        color_arr.append(\"Green\")\n    else:\n        color_arr.append(\"Blue\")\n        \nplt.scatter(dim_reduced, color=color_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers import LeakyReLU\nimport tensorflow as tf\n\nlatent_dim = 1000\n\ninput_shape = [256, 256, 1]\ninput_layer = layers.Input(input_shape)\nconv_layer1 = layers.Conv2D(1000, kernel_size=(256, 1))(input_layer)\nconv_layer2 = layers.Conv2D(1000, kernel_size=(1, 256), activation=LeakyReLU())(conv_layer1) \nbatch_norm1 = layers.BatchNormalization()(conv_layer2)\nmean_layer = layers.Dense(1000, name=\"mean_output\")(batch_norm1)\nstd_layer = layers.Dense(1000, name=\"std_output\")(batch_norm1)\n\nfrom keras import backend as K\n\ndef sampling(args):\n    z_mean, z_log_sigma = args\n    epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim),\n                              mean=0., stddev=0.1)\n    return z_mean + K.exp(z_log_sigma) * epsilon\n\nz = layers.Lambda(sampling)([mean_layer, std_layer])\n\n#encoder = Model.model(input_layer, [mean_layer, std_layer, z])\n#would need to do encoder[2]\nencoder = Model.model(input_layer, z)\n\n\ninput_layer2 = layers.Input(1000)\ndense_layer = layers.Dense(1000, activation='relu')\ndense_layer = layers.Dense(256*256*1, activation='sigmoid')\noutput = layers.Reshape(input_shape, name=\"decoder_output\")\n\ndecoder = Model.mode(input_layer2, output)\n\n\nwhole_output = decoder(encoder(input_layer))\nvar_autoencoder = Model.model(input_layer, whole_output, name=\"variantional\")\n\nvar_autoencoder.compile(loss=reconstruction_loss_func,\n                   optimizer=tf.keras.optimizers.Adam(lr=0.0005))\n                   #metrics=['accuracy'])\n\n#Do I need validation data????\nhistory = var_autoencoder.fit(\n    big_arr[0].reshape(1, 64, 64, 1), big_arr[0].reshape(1, 64, 64, 1),\n    epochs=100,\n    batch_size = 1,\n    verbose=0\n)\npd.DataFrame(history.history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers import LeakyReLU\nimport tensorflow as tf\n\n#Bad Accuracy BUTTTTTTT\n#TRY OUT THIS LATENT SPACE FIRST!\n\ndef reconstruction_loss_func(old, new):\n    #1 for each wrong pixel, harsh!\n    \n    reconstruction_loss = keras.losses.binary_crossentropy(old, new)\n    reconstruction_loss *= (256*256*3)\n    \n    return reconstruction_loss\n\nlatent_dim = 1024\ninput_shape = [64, 64, 3]\nautoencoder = keras.Sequential([\n    #layers.Flatten(input_shape=input_shape),\n    layers.BatchNormalization(input_shape=input_shape, renorm=True),\n    layers.Conv2D(latent_dim, kernel_size=(64, 1), input_shape=input_shape),\n    layers.Conv2D(latent_dim, kernel_size=(1, 64)),\n    layers.Dense(2048, activation=layers.LeakyReLU(), name=\"encoder_output\"),\n    layers.BatchNormalization(renorm=True),\n    layers.Conv2D(latent_dim, kernel_size=(1, 1)),\n    layers.Dense(64*64*3),\n    layers.LeakyReLU(),\n    layers.Reshape(input_shape, name=\"decoder_output\")\n])\n\n#Is a direct mapping to the SHAPE I want better???\n\nautoencoder.compile(loss='mse',\n                   optimizer=tf.keras.optimizers.Adam(lr=0.005, clipnorm=1.0, clipvalue=.5),\n                   metrics=['accuracy'])\n\n#Do I need validation data????\nhistory = autoencoder.fit(\n    big_arr, big_arr,\n    epochs=1500,\n    batch_size=32,\n    verbose=0\n)\npd.DataFrame(history.history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_reconstruction.layers[30].name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nimport matplotlib.pyplot as plt\n\ndef show_image(arr):\n    w, h = 28, 28\n    img = Image.fromarray(arr, 'L')\n    img.save('my.png')\n    img.show()\n    plt.imshow(arr, cmap='gray')\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ex_image = big_arr[55]\nprediction = autoencoder.predict(np.array([ex_image]).astype('int32'))\n\nprint(ex_image.dtype)\n\nplt.imshow(ex_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.unique(prediction[0]))\n\nplt.imshow(prediction[0].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## WHOA\nimport tensorflow as tf\n\n\n#.predict() returns data\n#.output() RETURNS A LAYER\n\nall_layer_names = [l.name for l in autoencoder.layers]\n\nlayer_names = ['encoder_output', 'decoder_output']\nlayers = [autoencoder.get_layer(l).output for l in layer_names]\n\nencoder = tf.keras.Model(autoencoder.input, layers[0])\n\n\n\n\n\nidx = all_layer_names.index(\"encoder_output\")\ninput_shape = autoencoder.layers[idx].get_input_shape_at(0)[1:] # get the input shape of desired layer\nlayer_input = tf.keras.layers.Input(shape=input_shape) # a new input tensor to be able to feed the desired layer\nprint(input_shape)\n\n# create the new nodes for each layer in the path\nx = layer_input\nfor layer in autoencoder.layers[idx:]:\n    x = layer(x)\n\n# create the model\n\ndecoder = tf.keras.Model(layer_input, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ex_image = big_arr[44]\nprediction = image_reconstruction_64.predict(np.array([ex_image]).astype('int32'))\n\nprint(ex_image.dtype)\n\nplt.imshow(ex_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.unique(prediction[0]))\nplt.imshow(prediction[0].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## get the layer names!\nimage_reconstruction.summary()"},{"metadata":{"trusted":true},"cell_type":"code","source":"## WHOA\nimport tensorflow as tf\n\n\n#.predict() returns data\n#.output() RETURNS A LAYER\n\nall_layer_names = [l.name for l in image_reconstruction_64.layers]\n\nlayer_names = ['half', 'decoder_output']\nlayers = [image_reconstruction_64.get_layer(l).output for l in layer_names]\n\nencoder = tf.keras.Model(image_reconstruction_64.input, layers[0])\n\n\n\n\n\nidx = all_layer_names.index(\"half\")\ninput_shape = image_reconstruction_64.layers[idx+1].get_input_shape_at(0)[1:] # get the input shape of desired layer\nlayer_input = tf.keras.layers.Input(shape=input_shape) # a new input tensor to be able to feed the desired layer\nprint(input_shape)\n\n# create the new nodes for each layer in the path\nx = layer_input\nfor layer in image_reconstruction_64.layers[idx+1:]:\n    x = layer(x)\n\n# create the model\n\ndecoder = tf.keras.Model(layer_input, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ex_image = big_arr[801]\nprediction = image_reconstruction_64.predict(np.array([ex_image]))\nplt.imshow(ex_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(prediction.astype('uint8')[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ex_image = big_arr[432]\nvec1 = encoder.predict(np.array([ex_image]))\nex_image = big_arr[801]\nvec2 = encoder.predict(np.array([ex_image]))\nvec = ((vec1+vec2))/2\n\nprint(vec1==vec2)\nprint(vec1)\nprint(vec2)\n\nchanged_image = decoder.predict(vec)\nplt.imshow(changed_image[0].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#NOW READ!!!\n\nimage_into_df\n.get_layer().output\nmodel.Model(, [outputs_LIST])\noutput = model()\ntf.gradient_tape as tape:\n    compute_loss()\ngrads = tape.gradient(loss, image)\nopt.apply_gradients([grads, init_image])\n    \ndef get_functional_model_and_layers():\n    layers_style = ['block3_conv2']\n    layers_content = ['block6_conv1', 'block7_conv1']\n    \n    vgg = tf.keras.applications.vgg19.VGG19(include_top=False, weights='imagenet')\n    vgg.trainable = False\n    \n    outputs = [vgg.get_layer(ls).output for ls in layers_style]+ \\\n            [vgg.get_layer(lc).output for lc in layers_content]\n\n    return Model.model(vgg.input, outputs)\n    \ndef content_loss(base_content, target)\n    tf.reduce_mean(tf.square(base_content-target))\n    \ndef style_loss(base_style, gram_target):\n    tf.reduce_mean(tf.square(tf.subtract(gram_matrix(base_style), gram_target)))\n\ndef gram_matrix(A):\n    channels = A.shape[-1]\n    a = A.reshape(-1, channels)\n    n = A.shape[0]\n    \n    return tf.matmul(a, a, transpose=True) / n.cast(float32)\n    \n\ndef feature_representation(style_path, content_path):\n    im = load_and_process_img(style_path)\n    im2 = load_and_process_img(content_path)\n    \n    inp = np.concantanate([im, im2])\n    \n    output = model(inp)\n    \n    #one for each layer\n    return [output[0]], [output[1], output[2]]\n    \n    \ndef combine_losses():\n    \n    \ndef compute_grad(cfg):\n    with tf.gradientTape() as tape:\n        loss = combine_losses(**cfg)\n    total_loss = loss[0]\n    better_image = tape.gradient(all_loses, init_image) #!!!!!!!!!\n    \ndef run():\n    \n    \nrun()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TWO INPUTS TO DISCRIMINATOR, So I could train discriminator as an anchor (anchor input too)\n    #Give it two images, it must determine if they are in the same class\n    #Sometimes give it 2 real images, sometimes give it a fake image so discriminator gets\n        #doesn't just start guessing all outputs are fake\n    \n#DISCRIMINATOR INPUT EXAMPLE:\n    #text, image of same class(fake/real)\n    #image, image of same class(fake/real)\n    #image, same image at night(different fake image at night/real)\n    \n#Conditional discriminator makes for: CLASS SPECFIC GENERATOR\n\n#FOR GANS TO WORK, YOU NEED A DETAILED & WIDE DATASET OF \"REAL\" IMAGES\n\nvgg = tf.applications.keras.vgg16()\nmodel = Model.model(vgg.input, vgg.get_layer('block5_conv1').output)\nmodel_discriminator = keras.Model([\n        layers.BatchNormalization(input_shape=input_shape),\n        layers.Dense(1)\n])\ndef gan(samples, batch_size=5):\n    \n    #MSE is high when image is way off, \n            #score of discrimator might not do that\n    #i can't use mse, but MSE correlates best to getting the image right???\n    \n    #MAYBE IF WE DID ENOUGH SCORING DURING EVERY ITERATION IT WOULD EQUAL AN MSE\n    \n    #model_discriminator.compile()\n    \n    for sample in samples.shuffle(batch_size):\n        fake_image = model(sample)\n        pred = model_discriminator.predict(fake_image)\n        gen_loss = 0\n        dis_loss = 0\n        if pred != 0:\n            dis_loss = 1\n        else:\n            gen_loss = 1\n    \n    with tf.gradientTape() as tape:\n        loss = tf.reduce_mean(tf.square(target-model(sample.shuffle(1))))\n    grads = tape.gradient(loss, model.get_weights())\n    #grads = tape.gradient(gen_loss, model.get_weights())\n    #grads = tape.gradient(dis_loss, model_discriminator.get_weights())\n    opt = Adam.optimizer(loss='mse')\n    new_weights = model.get_weights()\n    opt.apply_gradients([grads, new_weights])\n    model.assign_weights(new_weights)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\ntf.enable_eager_execution()\nprint(\"Eager execution: {}\".format(tf.executing_eagerly()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#minimzing distance between intermediate values\n#Optimizing hidden values??","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Content layer where will pull our feature maps\ncontent_layers = ['block5_conv2'] \n\n# Style layer we are interested in\nstyle_layers = ['block1_conv1',\n                'block2_conv1',\n                'block3_conv1', \n                'block4_conv1', \n                'block5_conv1'\n               ]\n\nnum_content_layers = len(content_layers)\nnum_style_layers = len(style_layers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n  \"\"\" Creates our model with access to intermediate layers. \n  \n  This function will load the VGG19 model and access the intermediate layers. \n  These layers will then be used to create a new model that will take input image\n  and return the outputs from these intermediate layers from the VGG model. \n  \n  Returns:\n    returns a keras model that takes image inputs and outputs the style and \n      content intermediate layers. \n  \"\"\"\n  # Load our model. We load pretrained VGG, trained on imagenet data (weights=’imagenet’)\n  vgg = tf.keras.applications.vgg19.VGG19(include_top=False, weights='imagenet')\n  vgg.trainable = False\n  # Get output layers corresponding to style and content layers \n  style_outputs = [vgg.get_layer(name).output for name in style_layers]\n  content_outputs = [vgg.get_layer(name).output for name in content_layers]\n  model_outputs = style_outputs + content_outputs\n  # Build model \n  return models.Model(vgg.input, model_outputs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Do the same thing for:\n    #1. Textual Data\n    #2. My own model\n    #3. Different layers, sharing or not sharing a loss function\n    \n    #vgg.get_layer(name).output\n    #name=\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#simple content loss function, MSE\n\ndef get_content_loss(base_content, target):\n  return tf.reduce_mean(tf.square(base_content - target))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This is a complicated gram_matrix for style loss\n#Apparently it's just a kernel feature map/receptive field\n    #MSE at the end though\n\ndef gram_matrix(input_tensor):\n  # We make the image channels first \n  channels = int(input_tensor.shape[-1])\n  a = tf.reshape(input_tensor, [-1, channels])\n  n = tf.shape(a)[0]\n  gram = tf.matmul(a, a, transpose_a=True)\n  return gram / tf.cast(n, tf.float32)\n\n#gram matrix, a square matrix\n    #matrix reshape to [LxW, Channels]\n    #Squared itself, Channels get dot producted out of existence\n    #divide [LxW, LxW] by LxW\n \ndef get_style_loss(base_style, gram_target):\n  \"\"\"Expects two images of dimension h, w, c\"\"\"\n  # height, width, num filters of each layer\n  height, width, channels = base_style.get_shape().as_list()\n  gram_style = gram_matrix(base_style)\n  \n  return tf.reduce_mean(tf.square(gram_style - gram_target))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_feature_representations(model, content_path, style_path):\n  \"\"\"Helper function to compute our content and style feature representations.\n \n  This function will simply load and preprocess both the content and style \n  images from their path. Then it will feed them through the network to obtain\n  the outputs of the intermediate layers. \n  \n  Arguments:\n    model: The model that we are using.\n    content_path: The path to the content image.\n    style_path: The path to the style image\n    \n  Returns:\n    returns the style features and the content features. \n  \"\"\"\n  # Load our images in \n  content_image = load_and_process_img(content_path)\n  style_image = load_and_process_img(style_path)\n  \n  # batch compute content and style features\n  stack_images = np.concatenate([style_image, content_image], axis=0)\n  model_outputs = model(stack_images)\n  \n  # Get the style and content feature representations from our model  \n  style_features = [style_layer[0] for style_layer in model_outputs[:num_style_layers]]\n  content_features = [content_layer[1] for content_layer in model_outputs[num_style_layers:]]\n  return style_features, content_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_loss(model, loss_weights, init_image, gram_style_features, content_features):\n  \"\"\"This function will compute the loss total loss.\n  \n  Arguments:\n    model: The model that will give us access to the intermediate layers\n    loss_weights: The weights of each contribution of each loss function. \n      (style weight, content weight, and total variation weight)\n    init_image: Our initial base image. This image is what we are updating with \n      our optimization process. We apply the gradients wrt the loss we are \n      calculating to this image.\n    gram_style_features: Precomputed gram matrices corresponding to the \n      defined style layers of interest.\n    content_features: Precomputed outputs from defined content layers of \n      interest.\n      \n  Returns:\n    returns the total loss, style loss, content loss, and total variational loss\n  \"\"\"\n  style_weight, content_weight, total_variation_weight = loss_weights\n  \n  # Feed our init image through our model. This will give us the content and \n  # style representations at our desired layers. Since we're using eager\n  # our model is callable just like any other function!\n  model_outputs = model(init_image)\n  \n  style_output_features = model_outputs[:num_style_layers]\n  content_output_features = model_outputs[num_style_layers:]\n  \n  style_score = 0\n  content_score = 0\n\n  # Accumulate style losses from all layers\n  # Here, we equally weight each contribution of each loss layer\n  weight_per_style_layer = 1.0 / float(num_style_layers)\n  for target_style, comb_style in zip(gram_style_features, style_output_features):\n    style_score += weight_per_style_layer * get_style_loss(comb_style[0], target_style)\n    \n  # Accumulate content losses from all layers \n  weight_per_content_layer = 1.0 / float(num_content_layers)\n  for target_content, comb_content in zip(content_features, content_output_features):\n    content_score += weight_per_content_layer* get_content_loss(comb_content[0], target_content)\n  \n  style_score *= style_weight\n  content_score *= content_weight\n  total_variation_score = total_variation_weight * total_variation_loss(init_image)\n\n  # Get total loss\n  loss = style_score + content_score + total_variation_score \n  return loss, style_score, content_score, total_variation_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_grads(cfg):\n  with tf.GradientTape() as tape: \n    all_loss = compute_loss(**cfg)\n  # Compute gradients wrt input image\n  total_loss = all_loss[0]\n  return tape.gradient(total_loss, cfg['init_image']), all_loss\n\n#our loss function with respect to our input image for the backwards pass.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_style_transfer(content_path, \n                       style_path,\n                       num_iterations=1000,\n                       content_weight=1e3, \n                       style_weight = 1e-2): \n  display_num = 100\n  # We don't need to (or want to) train any layers of our model, so we set their trainability\n  # to false. \n  model = get_model() \n  for layer in model.layers:\n    layer.trainable = False\n  \n  # Get the style and content feature representations (from our specified intermediate layers) \n  style_features, content_features = get_feature_representations(model, content_path, style_path)\n  gram_style_features = [gram_matrix(style_feature) for style_feature in style_features]\n  \n  # Set initial image\n  init_image = load_and_process_img(content_path)\n  init_image = tfe.Variable(init_image, dtype=tf.float32)\n  # Create our optimizer\n  opt = tf.train.AdamOptimizer(learning_rate=10.0)\n\n  # For displaying intermediate images \n  iter_count = 1\n  \n  # Store our best result\n  best_loss, best_img = float('inf'), None\n  \n  # Create a nice config \n  loss_weights = (style_weight, content_weight)\n  cfg = {\n      'model': model,\n      'loss_weights': loss_weights,\n      'init_image': init_image,\n      'gram_style_features': gram_style_features,\n      'content_features': content_features\n  }\n    \n  # For displaying\n  plt.figure(figsize=(15, 15))\n  num_rows = (num_iterations / display_num) // 5\n  start_time = time.time()\n  global_start = time.time()\n  \n  norm_means = np.array([103.939, 116.779, 123.68])\n  min_vals = -norm_means\n  max_vals = 255 - norm_means   \n  for i in range(num_iterations):\n    grads, all_loss = compute_grads(cfg)\n    loss, style_score, content_score = all_loss\n    # grads, _ = tf.clip_by_global_norm(grads, 5.0)\n    opt.apply_gradients([(grads, init_image)])\n    clipped = tf.clip_by_value(init_image, min_vals, max_vals)\n    init_image.assign(clipped)\n    end_time = time.time() \n    \n    if loss < best_loss:\n      # Update best loss and best image from total loss. \n      best_loss = loss\n      best_img = init_image.numpy()\n\n\n      iter_count += 1\n  print('Total time: {:.4f}s'.format(time.time() - global_start))\n      \n  return best_img, best_loss ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#No batch normalization until after!\nfrom keras import backend as K\n\nlatent_dim = 1024\nclass encoding_layer():\n    def __init__():\n        super(encoding_layer, self)\n        self.embeddDense = layers.Dense(latent_dim*2, activation=layers.LeakyReLU())\n        self.MeanNoAct = layers.Dense(latent_dim)\n        self.StdNoAct = layers.Dense(latent_dim)\n        self.Forget = layers.Dense(latent_dim, activation='softmax')\n\n    def call(inputs):\n        x = embeddDense(inputs)\n        m = self.MeanNoAct(x)\n        s = self.StdNoAct(x)\n        f = self.Forget(x)\n        \n        return m, s, f\n\ndef sampling(inputs):\n    mean, std, forget = inputs\n    epilson = K.random.normal(shape=(K.shape(mean)[0]), latent_dim, mean=0, stdev=.1)\n        \n    return (mean + K.exp(std) * eplison) * forget\n    \nclass decoding_layer():\n    def __init__():\n        super(decoding_layer, self)\n        self.lamb = layers.Lambda(sampling)\n        \n    def call(inputs)\n        x = self.lamb(inputs)\n        \n        return x\n    \n   \n\nclass expand:\n    def __init__(self, units):\n        super(expand, self)\n        self.bottle = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\n        self.norm1 = layers.BatchNormalization(renorm=batch_renorm)   \n        self.trans1 = layers.Conv2DTranspose(512, (3, 1), strides=(2, 1), padding='same', activation=layers.LeakyReLU())\n        self.norm2 = layers.BatchNormalization(renorm=batch_renorm)\n        self.trans2 = layers.Conv2DTranspose(512, (1, 3), strides=(1, 2), padding='same', activation=layers.LeakyReLU())\n        self.norm3 = layers.BatchNormalization(renorm=batch_renorm)\n        self.conv1 = layers.Conv2D(512, (3, 1), padding='same', activation=layers.LeakyReLU())\n        self.norm4 = layers.BatchNormalization(renorm=batch_renorm)\n        self.conv2 = layers.Conv2D(512, (1, 3), padding='same', activation=layers.LeakyReLU())\n        self.norm5 = layers.BatchNormalization(renorm=batch_renorm)\n        \n    def call(inputs):\n        x = self.bottle(inputs)\n        x = self.norm1(x)\n        x = self.trans1(x)\n        x = self.norm2(x)\n        x = self.trans2(x)\n        x = self.norm3(x)\n        x = self.conv1(x)\n        x = self.norm4(x)\n        x = self.conv2(x)\n        x = self.norm5(x)\n        \n        return x\n\nx = layers.Input(input_shape=[100])(inputs)\nx = layers.Dense(1024, activation=layers.LeakyReLU())(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nenc_outputs = encoding_layer()(x)\nx = decoding_layer()(enc_outputs)\nx = expand(512)(x)\nx = expand(512)(x)\nx = expand(256)(x)\nx = expand(128)(x)\nx = expand(64)(x)\nx = expand(32)(x)\nx = layers.Conv2D(3, (3, 3), padding=\"same\", activation=layers.LeakyReLU())\ngenerator_model = keras.Model(inputs, x)\n\ninputs = big_arr?????\n\nx = layers.Input(input_shape=[64, 64, 3])(inputs)\nx = layers.Conv2D(64, (3, 3), strides=(2, 2), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (3, 3), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(128, (3, 3), strides=(2, 2), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(128, (3, 3), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(128, (3, 3), strides=(2, 2), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(256, (3, 3), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(256, (3, 3), strides=(2, 2), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(256, (3, 3), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(256, (3, 3), strides=(2, 2), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(512, (3, 3), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(512, (3, 3), strides=(2, 2), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(512, (3, 3), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(512, (3, 3), strides=(2, 2), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(512, (3, 3), activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(64, (1, 1), padding='same', activation=layers.LeakyReLU())\nx = layers.Dropout(.4)(x)\nx = layers.BatchNormalization(renorm=batch_renorm)(x)\nx = layers.Conv2D(1, (1, 1), padding='same', activation='sigmoid')\ndiscrim_model = keras.Model(inputs, x)\n    \ndef reconstruction_loss_func(old, new):\n    #1 for each wrong pixel, harsh!\n    \n    #reconstruction_loss = keras.losses.binary_crossentropy(tf.bitcast(tf.cast(old, tf.int8), tf.uint8), tf.bitcast(tf.cast(new, tf.int8), tf.uint8))\n    mean, sigma, _ = enc_outputs\n    k1_loss = 1 + sigma - K.square(mean) - K.exp(sigma)\n    k1_loss = K.sum(k1_loss, axis=-1)\n    k1_loss *= -.5\n    \n    reconstruction_loss = keras.losses.binary_crossentropy(old, new)\n    reconstruction_loss *= (256*256*3)\n    \n    return reconstruction_loss + k1_loss\n\n\ngenerator_model.compile(\n`#switch between rec loss and discriminator loss!\n    #loss='binary_crossentropy', \n    loss = reconstruction_loss_func,\n    optimizer=keras.optimizer.Adam(lr=0.0002, beta_1=0.5),\n    metrics=['accuracy']\n)\nhistory = image_reconstruction_64.fit(\n    big_arr, big_arr,\n    batch_size=32, \n    epochs=100,\n    verbose=0\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inputs: Point in latent space, e.g. a 100 element vector of Gaussian random numbers."},{"metadata":{"trusted":true},"cell_type":"code","source":"lr=0.0002, beta_1=0.5\n# generate points in latent space as input for the generator\ndef generate_latent_points(latent_dim, n_samples):\n\tx_input = randn(latent_dim * n_samples)\n\tx_input = x_input.reshape(n_samples, latent_dim)\n\treturn x_input\n\ndef generate_fake_samples(g_model, latent_dim, n_samples):\n\tx_input = generate_latent_points(latent_dim, n_samples)\n\tX = g_model.predict(x_input)\n\ty = zeros((n_samples, 1))\n\treturn X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}