{"cells":[{"metadata":{},"cell_type":"markdown","source":"I tried CT-GAN.\n\nThis is the first time to open kenel.\n\nIf there are any problems, please comment!\n\n\nreference of code:\n\nhttps://www.kaggle.com/speedwagon/ralsgan-dogs\n\nhttps://github.com/ozanciga/gans-with-pytorch\n\n\npaper:\n\nhttps://arxiv.org/abs/1803.01541"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport time\n\n# Any results you write to the current directory are saved as output.\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_kernel_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from __future__ import print_function\nimport torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.optim as optim\nimport torch.utils.data\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nimport torchvision.utils as vutils\nfrom torch.autograd import Variable\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms\nfrom torchvision.utils import save_image\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nnz = 128\nlr = 0.0007\nbeta1 = 0.5\nepochs = 99999999\n\nreal_label = 0.90\nfake_label = 0.10\n\nn_critic = 5\nlambda_1 = 10\nlambda_2 = 2\nparam_M = 0.2\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random_transforms = [\n    #transforms.ColorJitter(brightness=0.75, contrast=0.75, saturation=0.75, hue=0.51), \n    transforms.RandomRotation(degrees=5)]\ntransform = transforms.Compose([transforms.Resize(64),\n                                transforms.CenterCrop(64),\n                                transforms.RandomHorizontalFlip(p=0.5),\n                                transforms.RandomApply(random_transforms, p=0.3),\n                                transforms.ToTensor(),\n                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\ntrain_data = datasets.ImageFolder('../input/all-dogs/', transform=transform)\ntrain_loader = torch.utils.data.DataLoader(train_data, shuffle=True,\n                                           batch_size=batch_size, num_workers=4)\n                                           \nimgs, label = next(iter(train_loader))\nimgs = imgs.numpy().transpose(0, 2, 3, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Defenition"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Generator(nn.Module):\n    def __init__(self, nz=128, channels=3):\n        super(Generator, self).__init__()\n        \n        self.nz = nz\n        self.channels = channels\n        \n        def convlayer(n_input, n_output, k_size=4, stride=2, padding=0):\n            block = [\n                nn.ConvTranspose2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False),\n                nn.BatchNorm2d(n_output),\n                nn.ReLU(inplace=True),\n            ]\n            return block\n\n        self.model = nn.Sequential(\n            *convlayer(self.nz, 1024, 4, 1, 0), # Fully connected layer via convolution.\n            *convlayer(1024, 512, 4, 2, 1),\n            *convlayer(512, 256, 4, 2, 1),\n            *convlayer(256, 128, 4, 2, 1),\n            *convlayer(128, 64, 4, 2, 1),\n            nn.ConvTranspose2d(64, self.channels, 3, 1, 1),\n            nn.Tanh()\n        )\n\n\n    def forward(self, z):\n        z = z.view(-1, self.nz, 1, 1)\n        img = self.model(z)\n        return img\n\nclass Discriminator(nn.Module):\n    def __init__(self, channels=3):\n        super(Discriminator, self).__init__()\n        \n        self.channels = channels\n\n        def convlayer(n_input, n_output, k_size=4, stride=2, padding=0, bn=False):\n            block = [nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False)]\n            if bn:\n                block.append(nn.BatchNorm2d(n_output))\n            block.append(nn.LeakyReLU(0.2, inplace=True))\n            return block\n    \n        self.model = nn.Sequential(\n            *convlayer(self.channels, 32, 4, 2, 1),\n            *convlayer(32, 64, 4, 2, 1),\n            *convlayer(64, 128, 4, 2, 1, bn=True),\n            *convlayer(128, 256, 4, 2, 1, bn=True),\n        )\n        \n        self.output_layer = nn.Conv2d(256, 1, 4, 1, 0, bias=False)\n\n\n    def forward(self, imgs, dropout=0.0, intermediate_output=False):\n        u1 = self.model(imgs)\n        u2 = self.output_layer(u1)\n        out = torch.sigmoid(u2)\n        if intermediate_output:\n            return out.view(-1, 1), (u1.view(imgs.size(0), 256, -1)).mean(dim=2) # u1 is the D_(.), intermediate layer given in paper.\n    \n        return out.view(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Use some initialization in the future.\ndef init_weights(m):\n    if type(m) == nn.ConvTranspose2d:\n        torch.nn.init.kaiming_normal(m.weight, mode='fan_out', nonlinearity='relu')\n    elif type(m) == nn.Conv2d:\n        torch.nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='leaky_relu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cuda = torch.cuda.is_available()\nTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor\n\ngenerator = Generator().to(device)\ndiscriminator = Discriminator().to(device)\n\noptimizer_D = optim.Adam(discriminator.parameters(), lr=lr, betas=(beta1, 0.999))\noptimizer_G = optim.Adam(generator.parameters(), lr=lr, betas=(beta1, 0.999))\ncriterion = nn.BCELoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(epochs):\n    if ((time.time()-start_kernel_time)/3600>8.0):\n        break\n    print('Epoch {}'.format(epoch))\n    for ii, (real_images, _) in tqdm(enumerate(train_loader), total=len(train_loader)-1):\n        if ((time.time()-start_kernel_time)/3600>8.0):\n            break\n        # == Discriminator update == #\n        for iter in range(n_critic):\n            if ((time.time()-start_kernel_time)/3600>8.0):\n                break\n            # Sample real and fake images, using notation in paper.\n            x = real_images.to(device)\n            noise = torch.randn(real_images.shape[0], nz, 1, 1, device=device)\n\n            x_tilde = Variable(generator(noise), requires_grad=True)\n            epsilon = Variable(Tensor(real_images.shape[0], 1, 1, 1).uniform_(0, 1))\n\n            x_hat = epsilon*x + (1 - epsilon)*x_tilde\n            x_hat = torch.autograd.Variable(x_hat, requires_grad=True)\n\n            # Put the interpolated data through critic.\n            dw_x = discriminator(x_hat)\n            grad_x = torch.autograd.grad(outputs=dw_x, inputs=x_hat,\n                                         grad_outputs=Variable(Tensor(real_images.size(0), 1).fill_(real_label), requires_grad=False),\n                                         create_graph=True, retain_graph=True, only_inputs=True)\n            grad_x = grad_x[0].view(real_images.size(0), -1)\n            grad_x = grad_x.norm(p=2, dim=1)\n\n            # Update critic.\n            optimizer_D.zero_grad()\n\n            # Standard WGAN loss.\n            d_wgan_loss = torch.mean(discriminator(x_tilde)) - torch.mean(discriminator(x))\n\n            # WGAN-GP loss.\n            d_wgp_loss = torch.mean((grad_x - 1)**2)\n\n            ###### Consistency term. ######\n            dw_x1, dw_x1_i = discriminator(x, dropout=0.5, intermediate_output=True) # Perturb the input by applying dropout to hidden layers.\n            dw_x2, dw_x2_i = discriminator(x, dropout=0.5, intermediate_output=True)\n            # Using l2 norm as the distance metric d, referring to the official code (paper ambiguous on d).\n            second_to_last_reg = ((dw_x1_i-dw_x2_i) ** 2).mean(dim=1).unsqueeze_(1).unsqueeze_(2).unsqueeze_(3)\n            d_wct_loss = (dw_x1-dw_x2) ** 2 \\\n                         + 0.1 * second_to_last_reg \\\n                         - param_M\n            d_wct_loss, _ = torch.max(d_wct_loss, 0) # torch.max returns max, and the index of max\n            d_wct_loss = d_wct_loss.sum()\n\n            # Combined loss.\n            d_loss = d_wgan_loss + lambda_1*d_wgp_loss + lambda_2*d_wct_loss\n\n            d_loss.backward()\n            optimizer_D.step()\n\n        # == Generator update == #\n        noise = torch.randn(batch_size, nz, 1, 1, device=device)\n        imgs_fake = generator(noise)\n\n        optimizer_G.zero_grad()\n        \n        labels = torch.full((batch_size, 1), real_label, device=device)\n        output = discriminator(imgs_fake)\n        g_loss = criterion(output, labels)\n\n        g_loss.backward()\n        optimizer_G.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_generated_img():\n    noise = torch.randn(1, nz, 1, 1, device=device)\n    gen_image = generator(noise).to(\"cpu\").clone().detach().squeeze(0)\n    gen_image = gen_image.numpy().transpose(1, 2, 0)\n    plt.imshow(gen_image)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for _ in range(25):\n    show_generated_img()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists('../output_images'):\n    os.mkdir('../output_images')\nim_batch_size = 50\nn_images=10000\nfor i_batch in range(0, n_images, im_batch_size):\n    gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)\n    gen_images = generator(gen_z)\n    images = gen_images.to(\"cpu\").clone().detach()\n    images = images.numpy().transpose(0, 2, 3, 1)\n    for i_image in range(gen_images.size(0)):\n        save_image(gen_images[i_image, :, :, :], os.path.join('../output_images', f'image_{i_batch+i_image:05d}.png'))\n\n\nimport shutil\nshutil.make_archive('images', 'zip', '../output_images')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}