{"cells":[{"metadata":{},"cell_type":"markdown","source":"I have to say, the first setup can not run successfully.  \nBut it seems the author run the second setup and the first is just for reference."},{"metadata":{},"cell_type":"markdown","source":"fork from this [post](https://www.kaggle.com/jesucristo/gan-introduction)"},{"metadata":{},"cell_type":"markdown","source":"v1: looks like only run 10 epochs, but may have run 100 epochs, I don't know. epoch=100  \nv3: same thing, but I still don't know how many epochs it ran. epoch=100  \nv5: crop dog part of the image and then train, epoch=500  \nv6: same as v5, but use LeakyReLU (0.2) in generator (ney, both networks) other than ReLU, epoch=500  \nv7: same as v5, but epoch = 700  \nv8: same as v5, but real label = 1 other than 0.9, epochs = 500  "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# import os\n# print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function\nimport os\nimport time\nimport torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.optim as optim\nimport torchvision.utils as vutils\nfrom torch.autograd import Variable\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms\nfrom torchvision.utils import save_image\nimport matplotlib.image as mpimg\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# some dogs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# PATH = \"../input/all-dogs/all-dogs/\"\n# images = os.listdir(PATH)\n# print(f\"There are {len(images)} pictures of dogs.\")\n\n# fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(12,10))\n\n# for indx, axis in enumerate(axes.flatten()):\n#     rnd_indx = np.random.randint(0, len(images))\n#     img = plt.imread(PATH + images[rnd_indx])\n#     imgplot = axis.imshow(img)\n#     axis.set_title(images[rnd_indx])\n#     axis.set_axis_off()\n# plt.tight_layout(rect=[0, 0.03, 1, 0.95])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np, pandas as pd, os\nimport xml.etree.ElementTree as ET\nimport matplotlib.pyplot as plt, zipfile\nfrom PIL import Image\n\nDogsOnly = True\n\nROOT = \"../input/\"\nIMAGES = os.listdir(ROOT + \"all-dogs/all-dogs/\")\nbreeds = os.listdir(ROOT + \"annotation/Annotation/\")\n\nidxIn = 0; namesIn = []\nimagesIn = np.zeros((25000, 64, 64, 3))\n\nif DogsOnly:\n    for breed in breeds:\n        for dog in os.listdir(ROOT + \"annotation/Annotation/\" + breed):\n            try: img = Image.open(ROOT + \"all-dogs/all-dogs/\" + dog + '.jpg')\n            except: continue\n            tree = ET.parse(ROOT + \"annotation/Annotation/\" + breed + '/' + dog)\n            root = tree.getroot()\n            objects = root.findall('object')\n            for o in objects:\n                bndbox = o.find('bndbox')\n                xmin = int(bndbox.find(\"xmin\").text)\n                ymin = int(bndbox.find(\"ymin\").text)\n                xmax = int(bndbox.find(\"xmax\").text)\n                ymax = int(bndbox.find(\"ymax\").text)\n                w = np.min((xmax-xmin, ymax-ymin))\n                img2 = img.crop((xmin,ymin,xmin+w,ymin+w))\n                img2 = img2.resize((64,64), Image.ANTIALIAS)\n                imagesIn[idxIn, :, :, :] = np.asarray(img2)\n                namesIn.append(breed)\n                idxIn += 1\n    idx = np.arange(idxIn)\n    np.random.shuffle(idx)\n    imagesIn = imagesIn[idx, :, :, :]\n    namesIn = np.array(namesIn)[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/questions/44429199/how-to-load-a-list-of-numpy-arrays-to-pytorch-dataset-loader\n\n\n\nbatch_size = 32\n\nimport torch\nimport numpy as np\nimport torch.utils.data as utils\n\ntensor_x = torch.stack([torch.Tensor(i) for i in imagesIn.transpose(0,3,1,2)]) # transform to torch tensors\nprint(type(tensor_x))\nmy_dataset = utils.TensorDataset(tensor_x) # create your datset\ndataloader = utils.DataLoader(my_dataset, shuffle=True,batch_size=batch_size) # create your dataloader\n\n# imgs = next(iter(dataloader))\n# imgs = torch.stack(imgs)[0].numpy()\n# imgs2 = imgs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch_size = 32\n# image_size = 64\n\n# random_transforms = [transforms.ColorJitter(), transforms.RandomRotation(degrees=20)]\n# transform = transforms.Compose([\n#     transforms.Resize(64), \n#     transforms.CenterCrop(64),\n# #     transforms.RandomHorizontalFlip(p=0.5),\n# #     transforms.RandomApply(random_transforms, p=0.2),\n#     transforms.ToTensor(),\n#     transforms.Normalize((0.5, 0.5, 0.5), (0.5,0.5,0.5) )  \n# ])\n\n# train_data = datasets.ImageFolder(\"../input/all-dogs/\", transform=transform)\n# dataloader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size = batch_size)\n\n# imgs, label = next(iter(dataloader))\n# imgs = imgs.numpy().transpose(0, 2, 3, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"show some read-in images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in range(5):\n#     plt.imshow(imgs[i])\n#     plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"image quality real low!"},{"metadata":{},"cell_type":"markdown","source":"# Weights init"},{"metadata":{"trusted":true},"cell_type":"code","source":"def weights_init(m):\n    \"\"\"\n    Take a neural network m as input, weight_init will initialize all its weights.\n    \"\"\"\n    classname = m.__class__.__name__\n    if classname.find(\"Conv\") != -1:\n        m.weight.data.normal_(0.0, 0.02)\n    elif classname.find(\"BatchNorm\") != -1:\n        m.weight.data.normal_(1.0, 0.02)\n        m.bias.data.fill_(0)    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"# class G(nn.Module):\n#     def __init__(self):\n#         # inherit torch.nn Module\n#         super(G, self).__init__()\n        \n#         self.main = nn.Sequential(\n#             nn.ConvTranspose2d(100, 512, 4, stride=1, padding=0, bias=False),\n#             nn.BatchNorm2d(512),\n#             nn.ReLU(True),\n            \n#             nn.ConvTranspose2d(512, 256, 4, stride=2, padding=1, bias=False),\n#             nn.BatchNorm2d(256),\n#             nn.ReLU(True),\n\n#             nn.ConvTranspose2d(256, 128, 4, stride=2, padding=1, bias=False),\n#             nn.BatchNorm2d(128),\n#             nn.ReLU(True),\n            \n#             nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1, bias=False),\n#             nn.BatchNorm2d(64),\n#             nn.ReLU(True),\n            \n#             nn.ConvTranspose2d(64, 3, 4, stride=2, padding=0, bias=False),\n#             nn.Tanh()\n#             )\n        \n#     def forward(self, input):\n#         output = self.main(input)\n#         return output\n\n# netG = G()\n# netG.apply(weights_init)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Discriminator"},{"metadata":{"trusted":true},"cell_type":"code","source":"# class D(nn.Module):\n#     def __init__(self):\n#         super(D,self).__init__()\n#         self.main = nn.Sequential(\n#             nn.Conv2d(3, 64, 4, stride=2, padding=1, bias=False),\n#             nn.LeakyReLU(negative_slope=0.2, inplace=True),\n            \n#             nn.Conv2d(64, 128, 4, stride=2, padding=1, bias=False),\n#             nn.BatchNorm2d(128),\n#             nn.LeakyReLU(negative_slope=0.2, inplace=True),\n            \n#             nn.Conv2d(128, 256, 4, stride=2, padding=1, bias=False),\n#             nn.BatchNorm2d(256),\n#             nn.LeakyReLU(negative_slope=0.2, inplace=True),\n            \n#             nn.Conv2d(256, 512, 4, stride=2, padding=1, bias=False),\n#             nn.BatchNorm2d(512),\n#             nn.LeakyReLU(negative_slope=0.2, inplace=True),\n            \n#             nn.Conv2d(512, 1, 4, stride=1, padding=0, bias=False),\n#             nn.Sigmoid()\n#             )\n    \n#     def forward(self, input):\n#         output = self.main(input)\n#         # .view(-1) = Flattens the output into 1D instead of 2D\n#         return output.view(-1)\n        \n# netD = D()\n# netD.apply(weights_init)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Another setup (only this works)"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Generator(nn.Module):\n    def __init__(self, nz=128, channels=3):\n        super(Generator, self).__init__()\n        \n        self.nz=nz\n        self.channels=channels\n        \n        def convlayer(n_input, n_output, k_size=4, stride=2, padding=0):\n            block = [nn.ConvTranspose2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False),\n                    nn.BatchNorm2d(n_output),\n                     nn.ReLU(inplace=True)\n                    ]\n            return block\n        \n        self.model = nn.Sequential(\n            *convlayer(self.nz, 1024, 4, 1, 0),\n            *convlayer(1024, 512, 4, 2, 1),\n            *convlayer(512, 256, 4, 2, 1),\n            *convlayer(256, 128, 4, 2, 1),\n            *convlayer(128, 64, 4, 2, 1),\n            nn.ConvTranspose2d(64, self.channels, 3, 1, 1),\n            nn.Tanh()\n            )\n        \n    def forward(self,z):\n        z = z.view(-1, self.nz, 1, 1)\n        img = self.model(z)\n        return img\n    \n    \n    \nclass Discriminator(nn.Module):\n    def __init__(self, channels=3):\n        super(Discriminator, self).__init__()\n\n        self.channels = channels\n        \n        def convlayer(n_input, n_output, k_size = 4, stride = 2, padding = 0, bn=False):\n            block = [nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding,bias=False)]\n            if bn:\n                block.append(nn.BatchNorm2d(n_output))\n            block.append(nn.LeakyReLU(0.2, inplace=True))\n            return block\n        \n        self.model = nn.Sequential(\n            *convlayer(self.channels, 32, 4, 2, 1),\n            *convlayer(32, 64, 4, 2, 1),\n            *convlayer(64, 128, 4, 2, 1, bn=True),\n            *convlayer(128, 256, 4, 2, 1, bn=True),\n            nn.Conv2d(256, 1, 4, 1, 0, bias=False)\n        )\n    \n    def forward(self, imgs):\n        logits = self.model(imgs)\n        out = torch.sigmoid(logits)\n        return out.view(-1,1)\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir results\n!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# EPOCH = 0\n# LR = 0.001\n# criterion = nn.BCELoss()\n# optimizerD = optim.Adam(netD.parameters(), lr=LR, betas = (0.5, 0.999))\n# optimizerG = optim.Adam(netG.parameters(), lr=LR, betas = (0.5, 0.999))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for epoch in range(EPOCH):\n#     for i, data in enumerate(dataloader, 0):\n#         # Updating the weights of the neural network of the discriminator\n#         netD.zero_grad()\n        \n#         # Train discriminator with real image\n#         real,_ = data\n#         input = Variable(real)\n#         target = Variable(torch.ones(input.size()[0]))\n#         output = netD(input)\n#         errD_real = criterion(output, target)\n        \n#         # Train discriminator with fake image\n#         noise = Variable(torch.randn(input.size()[0], 100, 1, 1))\n#         fake = netG(noise)\n#         target = Variable(torch.zeros(input.size()[0]))\n#         output = netD(fake.detach())\n#         errD_fake = criterion(output, target)\n        \n#         # Backpropagate the total error\n#         errD = errD_real + errD_fake\n#         errD.backward()\n#         optimizerD.step()\n        \n#         # Train generator\n#         netG.zero_grad()\n#         target = Variable(torch.ones(input.size()[0]))\n#         output = netD(fake)\n#         errG = criterion(output, target)\n#         errG.backward()\n#         optimizerG.step()\n        \n#         # Print the losses and save the real images and the generated images of the minibatch every 100 steps\n#         print(\"[%d/%d][%d/%d] Loss_D: %.4f; Loss_G: %.4f\" % (epoch, EPOCH, i, len(dataloader), errD.item(), errG.item()))\n#         if i % 100 == 0:\n#             vutils.save_image(real, \"%s/real_samples.png\" % \"./results\", normalize = True)\n#             fake = netG(noise)\n#             vutils.save_image(fake.data, \"%s/fake_samples_epoch_%03d.png\" % (\"./results\", epoch), normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch_size = 32\nLR_G = 0.001\nLR_D = 0.0005\n\nbeta1 = 0.5\nepochs = 500\nreal_label = 1\nfake_label = 0\nnz = 128\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# initialize models and optimizers"},{"metadata":{"trusted":true},"cell_type":"code","source":"# def plot_loss(G_losses, D_losses, epoch):\n#     plt.figure(figsize=(10,5))\n#     plt.title(\"Generator and Discriminator Loss - EPOCH\" + str(epoch))\n#     plt.plot(G_losses,label='G')\n#     plt.plot(D_losses,label='D')\n#     plt.xlabel('iterations')\n#     plt.ylabel('Loss')\n#     plt.legend()\n#     plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_generated_img(n_images = 5):\n    sample = []\n    for _ in range(n_images):\n        noise = torch.randn(1, nz, 1, 1, device = device)\n        gen_images = netG(noise).to(\"cpu\").clone().detach().squeeze(0)\n        gen_images = gen_images.numpy().transpose(1, 2, 0)\n        sample.append(gen_images)\n    \n    figure, axes = plt.subplots(1, len(sample), figsize = (96,96))\n    for index, axis in enumerate(axes):\n        axis.axis('off')\n        image_array = sample[index]\n        axis.imshow(image_array)\n        \n    plt.show()\n    plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"netG = Generator(nz).to(device)\nnetD = Discriminator().to(device)\n\ncriterion = nn.BCELoss()\n\noptimizerD = optim.Adam(netD.parameters(), lr=LR_D, betas=(beta1, 0.999))\noptimizerG = optim.Adam(netG.parameters(), lr=LR_G, betas=(beta1, 0.999))\n\nfixed_noise = torch.randn(25, nz, 1, 1, device=device)\n\nG_losses = []\nD_losses = []\nepoch_time = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor epoch in range(epochs):\n    start = time.time()\n    print(\"Epoch: {}\".format(epoch+1))\n    for ii, real_images in tqdm(enumerate(dataloader), total=len(dataloader)):\n        # update D network\n        # train with real\n        netD.zero_grad()\n        real_images = real_images[0].to(device)\n        \n        batch_size = real_images.size(0)\n        labels = torch.full((batch_size,1), real_label, device=device)\n        \n        output = netD(real_images)\n        errD_real = criterion(output, labels)\n        errD_real.backward()\n        D_x = output.mean().item()\n        \n        # train with fake\n        noise = torch.randn(batch_size, nz, 1, 1, device = device)\n        fake = netG(noise)\n        labels.fill_(fake_label)\n        output = netD(fake.detach())\n        errD_fake = criterion(output, labels)\n        errD_fake.backward()\n        D_G_z1 = output.mean().item()\n        errD = errD_real + errD_fake\n        optimizerD.step()\n        \n        # update G network\n        netG.zero_grad()\n        labels.fill_(real_label)\n        output = netD(fake)\n        errG = criterion(output, labels)\n        errG.backward()\n        D_G_z2 = output.mean().item()\n        optimizerG.step()\n        \n        # save losses for plotting\n#         G_losses.append(errG.item())\n#         D_losses.append(errD.item())\n        \n        if (ii+1) % len(dataloader) == 0:\n            print(\"[%d/%d][%d/%d] Loss_D:%.4f Loss_G:%.4f D(x):%.4f D(G(z)):%.4f / %.4f\"\n                 % (epoch+1, epochs, ii+1, len(dataloader),\n                   errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))\n            \n#     plot_loss(G_losses, D_losses, epoch)\n#     G_losses = []\n#     D_losses = []\n    show_generated_img()\n#     epoch_time.append(time.time()-start)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (\">> average EPOCH duration = \", np.mean(epoch_time))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generate submission images"},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nif not os.path.exists(\"../output_images\"):\n    os.mkdir(\"../output_images\")\n    \nim_batch_size = 50\nn_images = 10000\nfor i_batch in tqdm(range(0, n_images, im_batch_size)):\n    gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)\n    gen_images = netG(gen_z)\n    images = gen_images.to(\"cpu\").clone().detach()\n    images = images.numpy().transpose(0, 2, 3, 1)\n    for i_image in range(gen_images.size(0)):\n        save_image(gen_images[i_image,:,:,:], os.path.join(\"../output_images\", f\"image_{i_batch+i_image:05d}.png\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.make_archive(\"images\", \"zip\", \"../output_images\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}