{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nfrom torchvision import transforms\nfrom matplotlib import pyplot as plt\nimport os\nimport time\nimport pandas as pd\nimport subprocess\nimport os\nfrom datetime import datetime,timedelta\nfrom datetime import date\nimport urllib.request\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('----------------------- Libraries are imported ---------------------------')\n \n### Import Data and Create Train and test sets\n \ntorch.manual_seed(42)\ntorch.cuda.manual_seed(42)\n \n## Finds images\ntrain_data_path = '../input/gan-getting-started/'\n \n### Rescaling incoming image to 28 by 28 pixels\n### After Rescaling, convert the image to a tensor\ntransform = transforms.Compose([transforms.Resize((128,128)),transforms.ToTensor()])\ntrain_data = torchvision.datasets.ImageFolder(root=train_data_path,transform=transform)\ntest_data = torchvision.datasets.ImageFolder(root=train_data_path,transform=transform)\nbatch_size = 32\ntrain_loader = torch.utils.data.DataLoader(train_data,batch_size,shuffle=True)\ntest_loader = train_loader\n##### Declare the model architecture\n \nd = 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = 20\n \nclass VAE(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        \n        \n        self.conv1 = nn.Sequential(\n        nn.Conv2d(in_channels=3,out_channels=3,kernel_size=3,padding=1,stride=1),\n            nn.ReLU(),\n        nn.BatchNorm2d(3),\n        nn.Conv2d(in_channels=3,out_channels=2,kernel_size=3,padding=1,stride=1),\n        nn.ReLU(),\n        nn.BatchNorm2d(2)\n    \n            \n        )\n        self.resnet_adder = nn.Sequential(\n            nn.ReLU(),\n        nn.BatchNorm2d(3)\n        )\n        \n        self.fc1 = nn.Sequential(\n            ### Reduce the number of channels to 1 without changing the width and dimensions of the images\n            nn.Linear(128*128*2,128),\n            \n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Linear(128,64),\n            \n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Linear(64,32)\n            \n        )\n \n        self.encoder = nn.Sequential(\n            \n            nn.Linear(32, d ** 2),\n            nn.ReLU(),\n            nn.Linear(d ** 2, d * 2)\n        )\n        \n \n        self.decoder = nn.Sequential(\n            nn.Linear(d, d ** 2),\n            nn.ReLU(),\n            nn.Linear(d ** 2, 32)\n        )\n        self.fc2 = nn.Sequential(\n            nn.Linear(32,64),\n            nn.ReLU(),\n            nn.BatchNorm1d(64),\n            nn.Linear(64,128),\n            nn.ReLU(),\n            nn.BatchNorm1d(128),\n            nn.Linear(128,128*128*2),\n            nn.ReLU(),\n            nn.BatchNorm1d(128*128*2)\n            \n        )\n        self.tconv1 = nn.Sequential(\n            nn.ConvTranspose2d(in_channels=2,out_channels=3,kernel_size=3,stride=1,padding=1),\n            nn.ReLU(),\n            nn.BatchNorm2d(3),\n            nn.ConvTranspose2d(in_channels=3,out_channels=3,kernel_size=3,stride=1,padding=1),\n            nn.Sigmoid()\n        \n        )\n \n    def reparameterise(self, mu, logvar):\n        if self.training:\n            ## Using log variance to ensure that we get a positive std dev\n            ## Converting to std dev in the real space\n            std = logvar.mul(0.5).exp_()\n            ### Create error term which has the same shape as std dev sampled from a N(0,1) distribution\n            eps = std.data.new(std.size()).normal_()\n            #eps = torch.zeros(std.size())\n            ### Add the mean and the std_dev \n            return eps.mul(std).add_(mu)\n        else:\n            return mu\n \n    def forward(self, x):\n        \n        conv1_output = self.conv1(x)\n        #conv1_output = self.resnet_adder(conv1_output + x) \n        fc1_output = self.fc1(conv1_output.view(-1,128*128*2))\n        \n        ### Convert Encoded vector into shape (N,2,d)\n        mu_logvar = self.encoder(fc1_output).view(-1, 2, d)\n        ### First vector for each image is mean of the latent distribution\n        mu = mu_logvar[:, 0, :]\n        ### Second vector for each image is log-variance of the latent distribution\n        logvar = mu_logvar[:, 1, :]\n        ### Create variable Z = mu + error * Std_dev\n        z = self.reparameterise(mu, logvar)\n        ### Get decoder output\n        decoder_output = self.decoder(z)\n        \n        fc2_output = self.fc2(decoder_output)\n        tconv1_output = self.tconv1(fc2_output.view(fc2_output.size(0),2,128,128))\n        ## Resize Decoder Output to Pass it to TransposedConv2d layer to recontruct 3 channeled image\n        #decoder_output = decoder_output.view(decoder_output.size(0),1,28,28) \n        ## Return Reconstructed Output and mean and log-variance\n        return tconv1_output, mu, logvar,z\n    \nmodel = VAE()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate = 3e-3\n \noptimizer = torch.optim.Adam(\n    model.parameters(),\n    lr=learning_rate\n    )\n### Loss function\n \n#Reconstruction + KL divergence losses summed over all elements and batch\n \ndef loss_function(x_hat, x, mu, logvar):\n    MSE = nn.MSELoss(reduction='sum')\n    ## Making sure that distributions do not overlap\n#     loss = nn.functional.binary_cross_entropy(\n#         x_hat, x, reduction='sum'\n#     )\n    loss = 1.0*MSE(x_hat,x)\n    #loss = MSE(x_hat,x.view(x.size(0), -1))\n#     BCE = nn.functional.binary_cross_entropy(\n#         x_hat, x.view(-1, 28*28*3), reduction='sum'\n#     )\n    ### Makes sure that distributions of each image span entire latent space and the range does not explode\n    KLD = 0.5 * torch.sum(logvar.exp() - logvar - 1 + mu.pow(2))\n \n    return loss + KLD\n \n##### Training the model\nprint('----------------------- Training is started ---------------------------')\n \nepochs = 50\ncodes = dict(μ=list(), logσ2=list(), y=list())\n \nloss = {\n    'train_loss':[],\n    'test_loss' : []\n}\nfor epoch in range(0, epochs + 1):\n    # Training\n    if epoch > 0:  # test untrained net first\n        model.train()\n        train_loss = 0\n        for x, _ in train_loader:\n            #x = x.to(device)\n            # ===================forward=====================\n            x_hat, mu, logvar,_ = model(x)\n            loss = loss_function(x_hat, x, mu, logvar)\n            train_loss += loss.item()\n            # ===================backward====================\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n        # ===================log========================\n        \n        #loss['train_loss'].append(train_loss)\n        if epoch % 1 ==0:\n            print(f'====> Epoch: {epoch} Average loss: {train_loss / len(train_loader.dataset):.4f}')\n    \n    # Testing\n    \n    means, logvars, labels = list(), list(), list()\n    with torch.no_grad():\n        model.eval()\n        test_loss = 0\n        for x, _ in test_loader:\n            #x = x.to(device)\n            # ===================forward=====================\n            x_hat, mu, logvar,_ = model(x)\n            test_loss += loss_function(x_hat, x, mu, logvar).item()\n            # =====================log=======================\n            means.append(mu.detach())\n            logvars.append(logvar.detach())\n            #labels.append(y.detach())\n    # ===================log========================\n    #loss['test_loss'].append(test_loss)\n    codes['μ'].append(torch.cat(means))\n    codes['logσ2'].append(torch.cat(logvars))\n    test_loss /= len(test_loader.dataset)\n    if epoch % 1 == 0:\n        print(f'====> Test set loss: {test_loss:.4f}')\n        #display_images(x, x_hat, 1, f'Epoch {epoch}')\n        #plt.show()\n \n#### Extract embeddings\nprint('----------------------- Training has ended ---------------------------')\n \n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Generating Images From Samples\nimport numpy as np\nfrom torch.nn.modules.upsampling import Upsample\nimport matplotlib.pyplot as plt\nfor i in range(10):\n    print(i)\n    N = 1\n    z = torch.randn((N,d))\n    decoder_output = model.decoder(z)        \n    fc2_output = model.fc2(decoder_output)\n    tconv1_output = model.tconv1(fc2_output.view(N,2,128,128))\n    trans = transforms.ToPILImage()\n    trans1 = Upsample(scale_factor = 2, mode='nearest')\n    tconv1_output = trans1(tconv1_output)\n    tconv1_output = tconv1_output.view(3,256,256)\n    img = trans(tconv1_output)\n    plt.imshow(np.asarray(img))\n    plt.pause(0.5)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = {'model': model,\n              'state_dict': model.state_dict(),\n              'optimizer' : optimizer.state_dict()}\n \ntorch.save(checkpoint, 'checkpoint.pth')\n \nprint('----------------------- Save the Model ---------------------------')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}