{"cells":[{"metadata":{},"cell_type":"markdown","source":"## GAN Hacks\n\n\nCurrent kernels uses https://www.kaggle.com/francoisdubois/ralsgan-dogs-improved-parameters/ as base\nand implements a few of GAN hacks from:\n\n- [Luca's post](https://www.kaggle.com/c/generative-dog-images/discussion/98595#latest-573198)\n- [Soumith github](https://github.com/soumith/ganhacks)\n\n\n### Hacks:\n\n- Label smoothing within specified range\n- Random labels flips for discriminator\n- LeakyRELU for both Generator and Discriminator\n- BatchNorm with 0.9 momentum\n- Dropout for both networks\n\n\n---\nPrevious kernel disclaimer:\n\nThis kernel is only an optimization of parameters from @Vladislav\n\nhttps://www.kaggle.com/speedwagon/ralsgan-dogs\nI have also taken some parameters from @nanashi\n\nhttps://www.kaggle.com/jesucristo/gan-introduction\nI have changed some parameters in the GAN process and as well in the data loader.\nI have also added more epochs.\nPlease Upvotes both kernel mentionned here.¶"},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\n\nimport matplotlib.pyplot as plt  # \nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn, optim\nfrom tqdm import tqdm\n\nfrom torchvision import datasets, transforms\nfrom torchvision.utils import save_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nLR_G = 0.0005\nLR_D = 0.0005\n\nbeta1 = 0.5\nepochs = 260\n\nreal_label = 0.5\nfake_label = 0\nnz = 128\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Generator(nn.Module):\n    def __init__(self, nz=128, channels=3, dropout=0.25):\n        super(Generator, self).__init__()\n\n        self.nz = nz\n        self.channels = channels\n        self.dropout = dropout\n\n        def convlayer(n_input, n_output, k_size=4, stride=2, padding=0):\n            block = [\n                nn.ConvTranspose2d(\n                    n_input,\n                    n_output,\n                    kernel_size=k_size,\n                    stride=stride,\n                    padding=padding,\n                    bias=False,\n                ),\n                nn.BatchNorm2d(n_output, momentum=0.9),\n                # use LeakyRELU\n                nn.LeakyReLU(inplace=True),\n                nn.Dropout(self.dropout),\n            ]\n            return block\n\n        self.model = nn.Sequential(\n            *convlayer(\n                self.nz, 1024, 4, 1, 0\n            ),  # Fully connected layer via convolution.\n            *convlayer(1024, 512, 4, 2, 1),\n            *convlayer(512, 256, 4, 2, 1),\n            *convlayer(256, 128, 4, 2, 1),\n            *convlayer(128, 64, 4, 2, 1),\n            nn.ConvTranspose2d(64, self.channels, 3, 1, 1),\n            nn.Tanh(),\n        )\n\n    def forward(self, z):\n        z = z.view(-1, self.nz, 1, 1)\n        img = self.model(z)\n        return img\n\n\nclass Discriminator(nn.Module):\n    def __init__(self, channels=3, dropout=0.25, bn=True):\n        super(Discriminator, self).__init__()\n\n        self.channels = channels\n        self.dropout = dropout\n        # add BN by default\n        self.bn = bn\n\n        def convlayer(n_input, n_output, k_size=4, stride=2, padding=0, bn=self.bn):\n            block = [\n                nn.Conv2d(\n                    n_input,\n                    n_output,\n                    kernel_size=k_size,\n                    stride=stride,\n                    padding=padding,\n                    bias=False,\n                )\n            ]\n            if bn:\n                block.append(nn.BatchNorm2d(n_output, momentum=0.9))\n            # use LeakyRELU\n            block.append(nn.LeakyReLU(0.2, inplace=True))\n            block.append(nn.Dropout(self.dropout)),\n            return block\n\n        self.model = nn.Sequential(\n            *convlayer(self.channels, 32, 4, 2, 1),\n            *convlayer(32, 64, 4, 2, 1),\n            *convlayer(64, 128, 4, 2, 1, bn=True),\n            *convlayer(128, 256, 4, 2, 1, bn=True),\n            nn.Conv2d(256, 1, 4, 1, 0, bias=False),  # FC with Conv.\n        )\n\n    def forward(self, imgs):\n        logits = self.model(imgs)\n        out = torch.sigmoid(logits).view(-1, 1)\n        # do not use sigmoid for usage of BCE with logits loss\n        # out = logits.view(-1, 1)\n        return out\n\n\n# kaiming normal weights init\ndef weights_init(m):\n    if isinstance(m, nn.Conv2d):\n        torch.nn.init.kaiming_normal_(m.weight)\n    if isinstance(m, nn.ConvTranspose2d):\n        torch.nn.init.kaiming_normal_(m.weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nimage_size = 64\n\nrandom_transforms = [transforms.ColorJitter(), transforms.RandomRotation(degrees=0)]\ntransform = transforms.Compose(\n    [\n        transforms.Resize(64),\n        transforms.CenterCrop(64),\n        transforms.RandomHorizontalFlip(p=0.5),\n        transforms.RandomApply(random_transforms, p=0),\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ]\n)\n\ntrain_data = datasets.ImageFolder(\"../input/\", transform=transform)\ntrain_loader = torch.utils.data.DataLoader(\n    train_data, shuffle=True, batch_size=batch_size, num_workers=2, pin_memory=True\n)\n\nimgs, label = next(iter(train_loader))\nimgs = imgs.numpy().transpose(0, 2, 3, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"raw","source":"fig = plt.figure(figsize=(20, 16))\nfor ii, img in enumerate(imgs):\n    ax = fig.add_subplot(4, 8, ii + 1, xticks=[], yticks=[])\n    plt.imshow((img + 1.0) / 2.0)"},{"metadata":{"trusted":true},"cell_type":"code","source":"DROPOUT_RATE = 0.5\n\nnetG = Generator(nz, dropout=DROPOUT_RATE).to(device)\nnetD = Discriminator(dropout=DROPOUT_RATE).to(device)\n\n# apply weights init\nnetG.apply(weights_init)\nnetD.apply(weights_init)\n\n# use more stable version of BCE \n# criterion = nn.BCEWithLogitsLoss()\ncriterion = nn.BCELoss()\n\n# SGD for discriminator, Adam for generator, according to \n# https://github.com/soumith/ganhacks\n# optimizerD = optim.SGD(netD.parameters(), lr=LR_D, momentum=0.9)\noptimizerD = optim.Adam(netD.parameters(), lr=LR_D, betas=(beta1, 0.999))\noptimizerG = optim.Adam(netG.parameters(), lr=LR_G, betas=(beta1, 0.999))\n\nfixed_noise = torch.randn(25, nz, 1, 1, device=device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_generated_img():\n    noise = torch.randn(1, nz, 1, 1, device=device)\n    gen_image = netG(noise).to(\"cpu\").clone().detach().squeeze(0)\n    gen_image = gen_image.numpy().transpose(1, 2, 0)\n    plt.imshow(gen_image)\n    plt.show()\n    \n    \n# https://www.dlology.com/blog/bag-of-tricks-for-image-classification-with-convolutional-neural-networks-in-keras/\ndef smooth_labels(y, smooth_factor):\n    # label smoothing ref: https://www.robots.ox.ac.uk/~vgg/rg/papers/reinception.pdf\n    y *= 1 - smooth_factor\n    y += smooth_factor / y.shape[1]\n    return y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"# parameters specifying whether to smooth and flip labels\nUSE_LABEL_SMOOTHING = True\nUSE_LABEL_FLIP = True\n\n# smoothing factor range for discriminator\nDISC_SMOOTH_FACTOR_REAL = (-20, 20)\nDISC_SMOOTH_FACTOR_FAKE = (0, 30)\n# smoothing factor range for generator\nGEN_SMOOTH_FACTOR = (-20, 20)\n# probability of flipping labels for discriminator\nREAL_FLIP_PROB = 0.01\n# probability of smoothing labels\nSMOOTH_PROB = 0.5\n\n\nfor epoch in range(epochs):\n    for ii, (real_images, train_labels) in tqdm(\n        enumerate(train_loader), total=len(train_loader)\n    ):\n\n        netD.zero_grad()\n        real_images = real_images.to(device)\n        batch_size = real_images.size(0)\n        labels = torch.full((batch_size, 1), real_label, device=device)\n\n        if USE_LABEL_FLIP:\n            # occasional flip of real labels\n            if np.random.random() < REAL_FLIP_PROB:\n                # print(\"real labels flip for discriminator\")\n                labels = torch.full((batch_size, 1), fake_label, device=device)\n\n        if USE_LABEL_SMOOTHING:\n            if np.random.random() < SMOOTH_PROB:\n                # print(\"real label smoothing for discriminator\")\n                disc_smoothing = np.random.randint(\n                    DISC_SMOOTH_FACTOR_REAL[0], DISC_SMOOTH_FACTOR_REAL[1]\n                )\n                disc_smoothing /= 100\n                labels = smooth_labels(labels, disc_smoothing)\n                # print(labels)\n\n        output = netD(real_images)\n        errD_real = criterion(output, labels)\n        errD_real.backward()\n        D_x = output.mean().item()\n\n        # train with fake\n        noise = torch.randn(batch_size, nz, 1, 1, device=device)\n        fake = netG(noise)\n\n        labels.fill_(fake_label)\n        if USE_LABEL_SMOOTHING:\n            if np.random.random() < SMOOTH_PROB:\n                # print(\"fake label smoothing for discriminator\")\n                disc_smoothing = np.random.randint(\n                    DISC_SMOOTH_FACTOR_FAKE[0], DISC_SMOOTH_FACTOR_FAKE[1]\n                )\n                disc_smoothing /= 100\n                labels = smooth_labels(labels, disc_smoothing)\n                # print(labels)\n\n        output = netD(fake.detach())\n        errD_fake = criterion(output, labels)\n        errD_fake.backward()\n        D_G_z1 = output.mean().item()\n        errD = errD_real + errD_fake\n        optimizerD.step()\n\n        ############################\n        # (2) Update G network: maximize log(D(G(z)))\n        ###########################\n        netG.zero_grad()\n        labels.fill_(real_label)  # fake labels are real for generator cost\n        if USE_LABEL_SMOOTHING:\n            if np.random.random() < SMOOTH_PROB:\n                # print(\"label smoothing for generator\")\n                gen_smoothing = np.random.randint(\n                    GEN_SMOOTH_FACTOR[0], GEN_SMOOTH_FACTOR[1]\n                )\n                gen_smoothing /= 100\n                labels = smooth_labels(labels, gen_smoothing)\n                # print(labels)\n\n        output = netD(fake)\n        errG = criterion(output, labels)\n        errG.backward()\n        D_G_z2 = output.mean().item()\n        optimizerG.step()\n\n        if (ii + 1) % (len(train_loader) // 2) == 0:\n            print(\n                \"[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f\"\n                % (\n                    epoch + 1,\n                    epochs,\n                    ii + 1,\n                    len(train_loader),\n                    errD.item(),\n                    errG.item(),\n                    D_x,\n                    D_G_z1,\n                    D_G_z2,\n                )\n            )\n\n#             valid_image = netG(fixed_noise)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generated results "},{"metadata":{"trusted":true},"cell_type":"code","source":"gen_z = torch.randn(32, nz, 1, 1, device=device)\ngen_images = netG(gen_z).to(\"cpu\").clone().detach()\ngen_images = gen_images.numpy().transpose(0, 2, 3, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(25, 16))\nfor ii, img in enumerate(gen_images):\n    ax = fig.add_subplot(4, 8, ii + 1, xticks=[], yticks=[])\n    plt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make predictions and submit"},{"metadata":{"trusted":false},"cell_type":"code","source":"if not os.path.exists(\"../output_images\"):\n    os.mkdir(\"../output_images\")\n\nim_batch_size = 50\nn_images = 10000\n\nfor i_batch in range(0, n_images, im_batch_size):\n    gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)\n    gen_images = (netG(gen_z) + 1.0) / 2.0\n    images = gen_images.to(\"cpu\").clone().detach()\n    images = images.numpy().transpose(0, 2, 3, 1)\n    for i_image in range(gen_images.size(0)):\n        save_image(\n            gen_images[i_image, :, :, :],\n            os.path.join(\"../output_images\", f\"image_{i_batch+i_image:05d}.png\"),\n        )\n\nimport shutil\n\nshutil.make_archive(\"images\", \"zip\", \"../output_images\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}