{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport PIL\nimport torchvision\nimport torchvision.datasets as dset\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torchvision.utils as vutils\nfrom torch.autograd import Variable\nimport xml.etree.ElementTree as ET\nimport numpy as np\n\nimport matplotlib.pyplot as plt\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.optim as optim\nfrom torch.nn.init import xavier_uniform_\n\n\nimport time\nimport torch\nimport torch.nn as nn\n\nimport torch.nn.parallel\nimport torch.optim as optim\nfrom torch.nn.utils import spectral_norm\nimport torch.utils.data\nimport torchvision\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nimport torchvision.utils as vutils\nfrom torch.autograd import Variable\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms\nfrom torchvision.utils import save_image\nimport matplotlib.image as mpimg\n\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\n\n\nimport numpy as np\nimport os\nimport gzip, pickle\nimport tensorflow as tf\nfrom scipy import linalg\nimport pathlib\nimport urllib\nimport warnings\nfrom tqdm import tqdm\nfrom PIL import Image\nimport zipfile\n\nfrom tqdm import tqdm_notebook as tqdm\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kernel_start_time = time.perf_counter()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PixelwiseNorm(nn.Module):\n    def __init__(self):\n        super(PixelwiseNorm, self).__init__()\n\n    def forward(self, x, alpha=1e-8):\n        \"\"\"\n        forward pass of the module\n        :param x: input activations volume\n        :param alpha: small number for numerical stability\n        :return: y => pixel normalized activations\n        \"\"\"\n        y = x.pow(2.).mean(dim=1, keepdim=True).add(alpha).sqrt()  # [N1HW]\n        y = x / y  # normalize the input x volume\n        return y\n    \nclass MinibatchStdDev(nn.Module):\n    def __init__(self):\n        super(MinibatchStdDev, self).__init__()\n\n    def forward(self, x, alpha=1e-8):\n        \"\"\"\n        forward pass of the layer\n        :param x: input activation volume\n        :param alpha: small number for numerical stability\n        :return: y => x appended with standard deviation constant map\n        \"\"\"\n        batch_size, _, height, width = x.shape\n        # [B x C x H x W] Subtract mean over batch.\n        y = x - x.mean(dim=0, keepdim=True)\n        # [1 x C x H x W]  Calc standard deviation over batch\n        y = torch.sqrt(y.pow(2.0).mean(dim=0, keepdim=False) + alpha)\n\n        # [1]  Take average over feature_maps and pixels.\n        y = y.mean().view(1, 1, 1, 1)\n\n        # [B x 1 x H x W]  Replicate over group and pixels.\n        y = y.repeat(batch_size, 1, height, width)\n\n        # [B x C x H x W]  Append as new feature_map.\n        y = torch.cat([x, y], 1)\n        # return the computed values:\n        return y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## GAN\n### G\nconvlution transpose + spectral norm + pixel wise norm\n### D\nconvlution + spectral norm + batch norm + pixel wise norm"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Generator(nn.Module):\n    def __init__(self, nz=128, num_classes=120, channels=3, nfeats=32):\n        super(Generator, self).__init__()\n        self.nz = nz\n        self.num_classes = num_classes\n        self.channels = channels\n        \n        self.label_emb = nn.Embedding(num_classes, nz)\n        self.pixnorm = PixelwiseNorm()\n        self.conv1 = spectral_norm(nn.ConvTranspose2d(2*nz, nfeats * 8, 4, 1, 0, bias=False))\n        self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))\n        self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))\n        self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))\n        self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))\n        self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, channels, 3, 1, 1, bias=False))\n\n    def forward(self, inputs):\n        z, labels = inputs\n        enc = self.label_emb(labels).view((-1, self.nz, 1, 1))\n        enc = F.normalize(enc, p=2, dim=1)\n        x = torch.cat((z, enc), 1)\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.conv2(x))\n        x = self.pixnorm(x)\n        x = F.relu(self.conv3(x))\n        x = self.pixnorm(x)\n        x = F.relu(self.conv4(x))\n        x = self.pixnorm(x)\n        x = F.relu(self.conv5(x))\n        x = self.pixnorm(x)\n        x = torch.tanh(self.conv6(x))\n        return x\n\n    \nclass Discriminator(nn.Module):\n    def __init__(self, num_classes=120, channels=3, nfeats=64):\n        super(Discriminator, self).__init__()\n        self.channels = channels\n        self.num_classes = num_classes\n        self.label_emb = nn.Embedding(num_classes, 64*64)\n        self.conv1 = nn.Conv2d(channels+1, nfeats, 5, 2, 2, bias=False)\n        self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 5, 2, 2, bias=False))\n        self.bn2 = nn.BatchNorm2d(nfeats * 2)\n        self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 5, 2, 2, bias=False))\n        self.bn3 = nn.BatchNorm2d(nfeats * 4)\n        self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 5, 2, 2, bias=False))\n        self.bn4 = nn.MaxPool2d(2)\n        self.batch_discriminator = MinibatchStdDev()\n        self.pixnorm = PixelwiseNorm()\n        self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))\n\n    def forward(self, inputs):\n        imgs, labels = inputs\n        enc = self.label_emb(labels).view((-1, 1, 64, 64))\n        enc = F.normalize(enc, p=2, dim=1)\n        x = torch.cat((imgs, enc), 1)   # 4 input feature maps(3rgb + 1label)\n        x = F.relu(self.conv1(x), 0.2)\n        x = F.relu(self.bn2(self.conv2(x)), 0.2)\n        x = F.relu(self.bn3(self.conv3(x)), 0.2)\n        x = F.relu(self.bn4(self.conv4(x)), 0.2)\n        x = self.batch_discriminator(x)\n        x = torch.sigmoid(self.conv5(x))\n        return x.view(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(Dataset):\n    def __init__(self, directory, transform=None, n_samples=np.inf, crop_dogs=True):\n        self.directory = directory\n        self.transform = transform\n        self.n_samples = n_samples        \n        self.samples, self.labels = self.load_dogs_data(directory, crop_dogs)\n\n    def load_dogs_data(self, directory, crop_dogs):\n        required_transforms = torchvision.transforms.Compose([\n                torchvision.transforms.Resize(64),\n                torchvision.transforms.CenterCrop(64),\n        ])\n\n        imgs = []\n        labels = []\n        paths = []\n        for root, _, fnames in sorted(os.walk(directory)):\n            for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:\n                path = os.path.join(root, fname)\n                paths.append(path)\n\n        for path in paths:\n            # Load image\n            try: img = dset.folder.default_loader(path)\n            except: continue\n            \n            # Get bounding boxes\n            annotation_basename = os.path.splitext(os.path.basename(path))[0]\n            annotation_dirname = next(\n                    dirname for dirname in os.listdir('../input/annotation/Annotation/') if\n                    dirname.startswith(annotation_basename.split('_')[0]))\n                \n            if crop_dogs:\n                tree = ET.parse(os.path.join('../input/annotation/Annotation/',\n                                             annotation_dirname, annotation_basename))\n                root = tree.getroot()\n                objects = root.findall('object')\n                for o in objects:\n                    bndbox = o.find('bndbox')\n                    xmin = int(bndbox.find('xmin').text)\n                    ymin = int(bndbox.find('ymin').text)\n                    xmax = int(bndbox.find('xmax').text)\n                    ymax = int(bndbox.find('ymax').text)\n                    object_img = required_transforms(img.crop((xmin, ymin, xmax, ymax)))\n                    imgs.append(object_img)\n                    labels.append(annotation_dirname.split('-')[1].lower())\n\n            else:\n                object_img = required_transforms(img)\n                imgs.append(object_img)\n                labels.append(annotation_dirname.split('-')[1].lower())\n            \n        return imgs, labels\n    \n    \n    def __getitem__(self, index):\n        sample = self.samples[index]\n        label = self.labels[index]\n        \n        if self.transform is not None: \n            sample = self.transform(sample)\n        return np.asarray(sample), label\n\n    \n    def __len__(self):\n        return len(self.samples)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"database = '../input/all-dogs/all-dogs/'\ncrop_dogs = True\nn_samples = np.inf\nBATCH_SIZE = 32\n\nepochs = 1600\ncriterion = nn.BCELoss()\n\n# use_soft_noisy_labels=True\n\nnz = 128\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\ntransform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5),\n                                transforms.ToTensor(),\n                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\ntrain_data = DataGenerator(database, transform=transform, n_samples=n_samples, crop_dogs=crop_dogs)\n\ndecoded_dog_labels = {i:breed for i, breed in enumerate(sorted(set(train_data.labels)))}\nencoded_dog_labels = {breed:i for i, breed in enumerate(sorted(set(train_data.labels)))}\ntrain_data.labels = [encoded_dog_labels[l] for l in train_data.labels] # encode dog labels in the data generator\n\n\ntrain_loader = torch.utils.data.DataLoader(train_data, shuffle=True,\n                                           batch_size=BATCH_SIZE, num_workers=4)\n\n\nprint(\"Dog breeds loaded:  \", len(encoded_dog_labels))\nprint(\"Data samples loaded:\", len(train_data))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Hyperparm"},{"metadata":{"trusted":true},"cell_type":"code","source":"netG = Generator(nz, num_classes=len(encoded_dog_labels), nfeats=32).to(device)\nnetD = Discriminator(num_classes=len(encoded_dog_labels), nfeats=32).to(device)\nprint(\"Generator parameters:    \", sum(p.numel() for p in netG.parameters() if p.requires_grad))\nprint(\"Discriminator parameters:\", sum(p.numel() for p in netD.parameters() if p.requires_grad))\n\noptimizerG = optim.Adam(netG.parameters(), lr=0.0002, betas=(0.5, 0.999))\noptimizerD = optim.Adam(netD.parameters(), lr=0.0002, betas=(0.5, 0.999))\n\nlr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG, T_0=epochs//200, eta_min=0.00005)\nlr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD, T_0=epochs//200, eta_min=0.00005)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Some show function"},{"metadata":{"trusted":true},"cell_type":"code","source":"### This is to show one sample image for iteration of chosing\ndef show_generated_img():\n    noise = torch.randn(1, nz, 1, 1, device=device)\n    dog_label = torch.randint(0, len(encoded_dog_labels), (1, ), device=device)\n    gen_image = netG((noise, dog_label)).to(\"cpu\").clone().detach().squeeze(0)\n    #gen_image = netG(noise).to(\"cpu\").clone().detach().squeeze(0)\n    gen_image = gen_image.numpy().transpose(1, 2, 0)\n    gen_image = ((gen_image+1.0)/2.0)\n    plt.imshow(gen_image)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## GAN training (SGAN)\n1. Basic SGAN training\n2. 1600 epochs\n3. BCEloss\n4. True label is 0.7 + random(-0.1~0.1) and Fake label is 0.0 + random(0.0~0.2)"},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(epochs):\n    epoch_time = time.perf_counter()\n    if time.perf_counter() - kernel_start_time > 30000:\n            print(\"Time limit reached! Stopping kernel!\"); break\n\n    for ii, (real_images, dog_labels) in tqdm(enumerate(train_loader),total=len(train_loader)):\n        if real_images.shape[0]!= BATCH_SIZE: continue\n            \n        # smooth label\n        real_labels = torch.squeeze(torch.empty((BATCH_SIZE, 1), device=device).uniform_(0.60, 0.8))\n        fake_labels = torch.squeeze(torch.empty((BATCH_SIZE, 1), device=device).uniform_(0.00, 0.2))\n#         for p in np.random.choice(BATCH_SIZE, size=np.random.randint((BATCH_SIZE//8)), replace=False):\n#             real_labels[p], fake_labels[p] = fake_labels[p], real_labels[p] # swap labels\n        \n        \n        ############################\n        # (1) Update D network\n        ###########################\n        # Update real images to D\n        netD.zero_grad()\n        dog_labels = torch.tensor(dog_labels, device=device)\n        real_images = real_images.to(device)\n        output = netD((real_images, dog_labels))\n        errD_real = criterion(output, real_labels)\n        errD_real.backward()\n        D_x = output.mean().item()\n        \n        # Update fake images to D\n        noise = torch.randn(BATCH_SIZE, nz, 1, 1, device=device)\n        fake_images = netG((noise, dog_labels))\n        output = netD((fake_images.detach(), dog_labels)) \n        errD_fake = criterion(output, fake_labels)\n        errD_fake.backward()\n        D_G_z1 = output.mean().item()\n        errD = errD_real + errD_fake\n        optimizerD.step()\n        \n        ############################\n        # (2) Update G network\n        ###########################\n        netG.zero_grad()\n        output = netD((fake_images, dog_labels))\n        errG = criterion(output, real_labels)\n        errG.backward()\n        D_G_z2 = output.mean().item()\n        optimizerG.step()\n        \n        lr_schedulerG.step(epoch)\n        lr_schedulerD.step(epoch)\n\n    \n    print('%.2fs [%d/%d] Loss_D: %.4f Loss_G: %.4f' % (\n          time.perf_counter()-epoch_time, epoch+1, epochs, errD.item(), errG.item()))\n    print('D(x): ',D_x,', D(G(z)): ',D_G_z1)\n    show_generated_img()\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submit the best breed"},{"metadata":{"trusted":true},"cell_type":"code","source":"def mse(imageA, imageB):\n        err = np.sum((imageA.astype(\"float\") - imageB.astype(\"float\")) ** 2)\n        err /= float(imageA.shape[0] * imageA.shape[1])\n        return err\n\ndef analyse_generated_by_class(n_images=5):\n    good_breeds = []\n    for l in range(len(decoded_dog_labels)):\n        sample = []\n        for _ in range(n_images):\n            noise = torch.randn(1, nz, 1, 1, device=device)\n            dog_label = torch.full((1,) , l, device=device, dtype=torch.long)\n            gen_image = netG((noise, dog_label)).to(\"cpu\").clone().detach().squeeze(0)\n            gen_image = gen_image.numpy().transpose(1, 2, 0)\n            sample.append(gen_image)\n        \n        # if mse for sample k to k-1 is too different , discard the sample\n        d = np.round(np.sum([mse(sample[k], sample[k+1]) for k in range(len(sample)-1)])/n_images, 1)\n        \n        if d < 1.0: continue  # had mode colapse(discard)\n            \n        print(f\"Generated breed({d}): \", decoded_dog_labels[l])\n        figure, axes = plt.subplots(1, len(sample), figsize=(64, 64))\n        for index, axis in enumerate(axes):\n            axis.axis('off')\n            image_array = (sample[index] + 1.) / 2.\n            axis.imshow(image_array)\n        plt.show()\n        \n        good_breeds.append(l)\n    return good_breeds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_submit(good_breeds):\n    print(\"Creating submit\")\n    os.makedirs('../output_images', exist_ok=True)\n    im_batch_size = 100\n    n_images = 10000\n    \n    all_dog_labels = np.random.choice(good_breeds, size=n_images, replace=True)\n    for i_batch in range(0, n_images, im_batch_size):\n        noise = torch.randn(im_batch_size, nz, 1, 1, device=device)\n        dog_labels = torch.from_numpy(all_dog_labels[i_batch: (i_batch+im_batch_size)]).to(device)\n        gen_images = netG((noise, dog_labels))\n        gen_images = (gen_images.to(\"cpu\").clone().detach() + 1) / 2\n        for ii, img in enumerate(gen_images):\n            save_image(gen_images[ii, :, :, :], os.path.join('../output_images', f'image_{i_batch + ii:05d}.png'))\n            \n    import shutil\n    shutil.make_archive('images', 'zip', '../output_images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"good_breeds = analyse_generated_by_class(6)\ncreate_submit(good_breeds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}