{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Variational Autoencoder on Fashion MNIST Dataset\n\nBayesian Inference - model underlying distribution\n\nEncoder Q(mu, lg(variance) | X) - input -> latent space (mean, stddev like vectors)\n\nz = random(mu, lg(variance))    - Sample latent space\n\nDecoder P(X | z)                - latent space of input -> sample\n\n\nhttp://kvfrans.com/variational-autoencoders-explained/\n\nhttps://wiseodd.github.io/techblog/2016/12/10/variational-autoencoder/\n\nhttps://github.com/pytorch/examples/blob/master/vae/main.py"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport sklearn as skl\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import optim\nfrom torch.utils.data import Dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MNIST(Dataset):\n    def __init__(self, filename, batch_size, train=True, shuffle=None):\n        data = pd.read_csv(filename)\n        \n        self.batch_size = batch_size\n\n        if shuffle is None:\n            shuffle = train\n        \n        if shuffle:\n            data = skl.utils.shuffle(data)\n            data.reset_index(inplace=True, drop=True)\n\n        if train:\n            self.images = data.iloc[:, 1:] / 255\n            self.labels = data.iloc[:, 0]\n        else:\n            self.images = data / 255\n            self.labels = np.empty(len(data))\n            \n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        if isinstance(idx, int):\n            idx = list([idx])\n\n        images = torch.from_numpy(self.images.iloc[idx].values).float()\n        \n        labels = torch.from_numpy(np.array(self.labels[idx]))\n        \n        return images, labels\n\n    def __iter__(self):\n        for i in range(0, len(self), self.batch_size):\n            yield self[i + np.arange(self.batch_size)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Read Digits Dataset\ntrain_set = MNIST('../input/digit-recognizer/train.csv', batch_size=4)\ntest_set = MNIST('../input/digit-recognizer/test.csv', batch_size=1, train=False)\n\nlabel_map = list(range(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Read Fashion dataset\ntrain_set = MNIST('/kaggle/input/fashionmnist/fashion-mnist_train.csv', batch_size=16)\ntest_set = MNIST('/kaggle/input/fashionmnist/fashion-mnist_test.csv', batch_size=1, train=True, shuffle=False)  ## Each test image has the expected label, train=True\n\nlabel_map = {\n    0: 'Top',\n    1: 'Trouser',\n    2: 'Pullover',\n    3: 'Dress',\n    4: 'Coat',\n    5: 'Sandals',\n    6: 'Shirt',\n    7: 'Sneaker',\n    8: 'Bag',\n    9: 'Boot',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Encoder(nn.Module):\n    def __init__(self, layer_data):\n        super(Encoder, self).__init__()\n\n        incoming, hidden, out = layer_data\n\n        self.la = nn.Linear(incoming, hidden)\n        \n        self.l_mu = nn.Linear(hidden, out)\n        self.l_logvar = nn.Linear(hidden, out)\n    \n    def forward(self, x):\n        \"\"\"\n        Forward pass through encoder and decoder.\n        \"\"\"\n        x = F.relu(self.la(x))\n\n        mu = self.l_mu(x)\n        logvar = self.l_logvar(x)  ## logvar = log(sigma^2)\n\n        return mu, logvar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Decoder(nn.Module):\n    def __init__(self, layer_data):\n        super(Decoder, self).__init__()\n\n        incoming, hidden, out = layer_data\n\n        self.ly = nn.Linear(out, hidden)\n        self.lz = nn.Linear(hidden, incoming)\n\n    def forward(self, z):\n        \"\"\"\n        Forward pass through encoder and decoder.\n        \"\"\"\n        y = F.relu(self.ly(z))\n        y = torch.sigmoid(self.lz(y))  # Sigmoid bounds image to [-1, 1]?\n\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VAE(nn.Module):\n    def __init__(self, layer_data):\n        super(VAE, self).__init__()\n        \n        self.encoder = Encoder(layer_data)\n        self.decoder = Decoder(layer_data)\n    \n    def forward(self, x):\n        \"\"\"\n        Forward pass through encoder and decoder.\n        \"\"\"\n        mu, logvar = self.encoder.forward(x)\n\n        std = torch.exp(0.5 * logvar)\n        eps = torch.randn_like(std)\n        \n        z = mu + eps*std\n\n        y = self.decoder.forward(z)\n\n        return y, mu, logvar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Initialize\nvae = VAE([784, 512, 2])\n\ndef loss_function(recon_x, x, mu, logvar):\n    \"\"\"\n    Cross entropy + KL Divergence loss\n    \"\"\"\n    BCE = F.binary_cross_entropy(recon_x, x.view((-1, 784)), reduction='sum')\n\n    # https://arxiv.org/abs/1312.6114\n    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n\n    return BCE + KLD\n\noptimizer = optim.Adam(vae.parameters(), lr=1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Train\nvae.train()\n\nfor epoch in range(3):\n    optimizer.zero_grad()\n    \n    total_loss = 0\n    for i, data in enumerate(train_set):\n        inputs, labels = data\n\n        optimizer.zero_grad()\n\n        outputs, mu, logvar = vae(inputs)\n        \n        loss = loss_function(outputs, inputs, mu, logvar)\n        loss.backward()\n        \n        total_loss += loss.item()\n\n        optimizer.step()\n\n        total_loss += loss.item()\n        if i % 2000 == 1999:\n            print(f'{epoch+1}, {i+1} - loss: {total_loss / 2000:.3f}')\n            total_loss = 0.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Generate\nvae.eval()\n\n# predictions = [vae(inputs)[0].data for inputs, labels in test_set]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_map","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Show off\noffset = 0\n\nfor idx in range(10):\n    image, labels = test_set[idx]\n\n    while label_map[int(labels[0])] != 'Boot':        \n        offset += 1\n        \n        image, labels = test_set[idx + offset]\n    \n    plt.imshow(vae(image)[0].data.reshape((28, 28)))\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}