{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytorch Rals-C-SAGAN\n* Ra - Relativistic Average;\n* Ls - Least Squares;\n* C - Conditional;\n* SA - Self-Attention;\n* DCGAN - Deep Convolutional Generative Adversarial Network\n\n<br>\nReferences:\n* https://www.kaggle.com/speedwagon/ralsgan-dogs\n* https://www.kaggle.com/cdeotte/dog-breed-cgan\n* https://github.com/eriklindernoren/PyTorch-GAN/blob/master/implementations/cgan/cgan.py\n* https://github.com/voletiv/self-attention-GAN-pytorch/blob/master/sagan_models.py"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport PIL\nimport torchvision\nimport torchvision.datasets as dset\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torchvision.utils as vutils\nfrom torch.autograd import Variable\nimport xml.etree.ElementTree as ET\nimport numpy as np\n\nimport matplotlib.pyplot as plt\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.optim as optim\nfrom torch.nn.init import xavier_uniform_\n\n\nimport time\nimport torch\nimport torch.nn as nn\n\nimport torch.nn.parallel\nimport torch.optim as optim\nfrom torch.nn.utils import spectral_norm\nimport torch.utils.data\nimport torchvision\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nimport torchvision.utils as vutils\nfrom torch.autograd import Variable\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms\nfrom torchvision.utils import save_image\nimport matplotlib.image as mpimg\n\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\n\n\nimport numpy as np\nimport os\nimport gzip, pickle\nimport tensorflow as tf\nfrom scipy import linalg\nimport pathlib\nimport urllib\nimport warnings\nfrom tqdm import tqdm\nfrom PIL import Image\nimport zipfile\n\nfrom tqdm import tqdm_notebook as tqdm\n\n\nkernel_start_time = time.perf_counter()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Blocks"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def snconv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True):\n    return spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\n                                   stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias))\n\ndef snlinear(in_features, out_features):\n    return spectral_norm(nn.Linear(in_features=in_features, out_features=out_features))\n\n\ndef sn_embedding(num_embeddings, embedding_dim):\n    return spectral_norm(nn.Embedding(num_embeddings=num_embeddings, embedding_dim=embedding_dim))\n\n\nclass PixelwiseNorm(nn.Module):\n    def __init__(self):\n        super(PixelwiseNorm, self).__init__()\n\n    def forward(self, x, alpha=1e-8):\n        \"\"\"\n        forward pass of the module\n        :param x: input activations volume\n        :param alpha: small number for numerical stability\n        :return: y => pixel normalized activations\n        \"\"\"\n        y = x.pow(2.).mean(dim=1, keepdim=True).add(alpha).sqrt()  # [N1HW]\n        y = x / y  # normalize the input x volume\n        return y\n    \nclass MinibatchStdDev(nn.Module):\n    def __init__(self):\n        super(MinibatchStdDev, self).__init__()\n\n    def forward(self, x, alpha=1e-8):\n        \"\"\"\n        forward pass of the layer\n        :param x: input activation volume\n        :param alpha: small number for numerical stability\n        :return: y => x appended with standard deviation constant map\n        \"\"\"\n        batch_size, _, height, width = x.shape\n        # [B x C x H x W] Subtract mean over batch.\n        y = x - x.mean(dim=0, keepdim=True)\n        # [1 x C x H x W]  Calc standard deviation over batch\n        y = torch.sqrt(y.pow(2.0).mean(dim=0, keepdim=False) + alpha)\n\n        # [1]  Take average over feature_maps and pixels.\n        y = y.mean().view(1, 1, 1, 1)\n\n        # [B x 1 x H x W]  Replicate over group and pixels.\n        y = y.repeat(batch_size, 1, height, width)\n\n        # [B x C x H x W]  Append as new feature_map.\n        y = torch.cat([x, y], 1)\n        # return the computed values:\n        return y\n    \n    \nclass Self_Attn(nn.Module):\n    \"\"\" Self attention Layer\"\"\"\n    def __init__(self, in_channels):\n        super(Self_Attn, self).__init__()\n        self.in_channels = in_channels\n        self.snconv1x1_theta = snconv2d(in_channels=in_channels, out_channels=in_channels//8, kernel_size=1, stride=1, padding=0)\n        self.snconv1x1_phi = snconv2d(in_channels=in_channels, out_channels=in_channels//8, kernel_size=1, stride=1, padding=0)\n        self.snconv1x1_g = snconv2d(in_channels=in_channels, out_channels=in_channels//2, kernel_size=1, stride=1, padding=0)\n        self.snconv1x1_attn = snconv2d(in_channels=in_channels//2, out_channels=in_channels, kernel_size=1, stride=1, padding=0)\n        self.maxpool = nn.MaxPool2d(2, stride=2, padding=0)\n        self.softmax  = nn.Softmax(dim=-1)\n        self.sigma = nn.Parameter(torch.zeros(1))\n\n    def forward(self, x):\n        _, ch, h, w = x.size()\n        # Theta path\n        theta = self.snconv1x1_theta(x)\n        theta = theta.view(-1, ch//8, h*w)\n        # Phi path\n        phi = self.snconv1x1_phi(x)\n        phi = self.maxpool(phi)\n        phi = phi.view(-1, ch//8, h*w//4)\n        # Attn map\n        attn = torch.bmm(theta.permute(0, 2, 1), phi)\n        attn = self.softmax(attn)\n        # g path\n        g = self.snconv1x1_g(x)\n        g = self.maxpool(g)\n        g = g.view(-1, ch//2, h*w//4)\n        # Attn_g\n        attn_g = torch.bmm(g, attn.permute(0, 2, 1))\n        attn_g = attn_g.view(-1, ch//2, h, w)\n        attn_g = self.snconv1x1_attn(attn_g)\n        # Out\n        out = x + self.sigma * attn_g\n        return out\n\n    \nclass ConditionalBatchNorm2d(nn.Module):\n    def __init__(self, num_features, num_classes):\n        super().__init__()\n        self.num_features = num_features\n        self.bn = nn.BatchNorm2d(num_features)\n        self.embed = nn.Embedding(num_classes, num_features * 2)\n        self.embed.weight.data[:, :num_features].fill_(1.)  # Initialize scale to 1\n        self.embed.weight.data[:, num_features:].zero_()    # Initialize bias at 0\n\n    def forward(self, inputs):\n        x, y = inputs\n        \n        out = self.bn(x)\n        gamma, beta = self.embed(y).chunk(2, 1)\n        out = gamma.view(-1, self.num_features, 1, 1) * out + beta.view(-1, self.num_features, 1, 1)\n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generator and Discriminator"},{"metadata":{"trusted":true},"cell_type":"code","source":"class UpConvBlock(nn.Module):\n    def __init__(self, n_input, n_output, num_classes, k_size=4, stride=2, padding=0,\n                 bias=False, dropout_p=0.0, norm=None):\n        super(UpConvBlock, self).__init__()\n        self.dropout_p = dropout_p\n        self.norm = norm\n        self.upconv = spectral_norm(nn.ConvTranspose2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=bias))\n        if norm == \"cbn\": self.normt = ConditionalBatchNorm2d(n_output, num_classes)\n        elif norm == \"pixnorm\": self.normt = PixelwiseNorm()\n        elif norm == \"bn\": self.normt = nn.BatchNorm2d(n_output)\n        self.activ = nn.LeakyReLU(0.05, inplace=True)\n        self.dropout = nn.Dropout2d(p=dropout_p)\n    \n    def forward(self, inputs):\n        x0, labels = inputs\n\n        x = self.upconv(x0)\n        if self.norm is not None:\n            if self.norm == \"cbn\":\n                x = self.activ(self.normt((x, labels)))\n            else:\n                x = self.activ(self.normt(x))\n        if self.dropout_p > 0.0:\n            x = self.dropout(x)\n        return x\n\n\nclass Generator(nn.Module):\n    def __init__(self, nz=128, num_classes=120, channels=3, nfilt=64):\n        super(Generator, self).__init__()\n        self.nz = nz\n        self.num_classes = num_classes\n        self.channels = channels\n        \n        self.label_emb = nn.Embedding(num_classes, nz)\n        self.pixelnorm = PixelwiseNorm()\n        self.upconv1 = UpConvBlock(2*nz, nfilt*16, num_classes, k_size=4, stride=1, padding=0, norm=\"cbn\", dropout_p=0.15)\n        self.upconv2 = UpConvBlock(nfilt*16, nfilt*8, num_classes, k_size=4, stride=2, padding=1, norm=\"cbn\", dropout_p=0.10)\n        self.upconv3 = UpConvBlock(nfilt*8, nfilt*4, num_classes, k_size=4, stride=2, padding=1, norm=\"cbn\", dropout_p=0.05)\n        self.upconv4 = UpConvBlock(nfilt*4, nfilt*2, num_classes, k_size=4, stride=2, padding=1, norm=\"cbn\", dropout_p=0.05)\n        self.upconv5 = UpConvBlock(nfilt*2, nfilt, num_classes, k_size=4, stride=2, padding=1, norm=\"cbn\", dropout_p=0.05)\n        self.self_attn = Self_Attn(nfilt)\n        self.upconv6 = UpConvBlock(nfilt, 3, num_classes, k_size=3, stride=1, padding=1, norm=\"cbn\")\n        self.out_conv = spectral_norm(nn.Conv2d(3, 3, 3, 1, 1, bias=False))  \n        self.out_activ = nn.Tanh()\n        \n    def forward(self, inputs):\n        z, labels = inputs\n        \n        enc = self.label_emb(labels).view((-1, self.nz, 1, 1))\n        enc = F.normalize(enc, p=2, dim=1)\n        x = torch.cat((z, enc), 1)\n        \n        x = self.upconv1((x, labels))\n        x = self.upconv2((x, labels))\n        x = self.upconv3((x, labels))\n        x = self.upconv4((x, labels))\n        x = self.upconv5((x, labels))\n        x = self.self_attn(x)\n        x = self.upconv6((x, labels))\n        x = self.out_conv(x)\n        img = self.out_activ(x)              \n        return img\n    \n    \nclass Discriminator(nn.Module):\n    def __init__(self, num_classes=120, channels=3, nfilt=64):\n        super(Discriminator, self).__init__()\n        self.channels = channels\n        self.num_classes = num_classes\n\n        def down_convlayer(n_input, n_output, k_size=4, stride=2, padding=0, dropout_p=0.0):\n            block = [spectral_norm(nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False)),\n                     nn.BatchNorm2d(n_output),\n                     nn.LeakyReLU(0.2, inplace=True),\n                    ]\n            if dropout_p > 0.0: block.append(nn.Dropout(p=dropout_p))\n            return block\n        \n        self.label_emb = nn.Embedding(num_classes, 64*64)\n        self.model = nn.Sequential(\n            *down_convlayer(self.channels + 1, nfilt, 4, 2, 1),\n            Self_Attn(nfilt),\n            \n            *down_convlayer(nfilt, nfilt*2, 4, 2, 1, dropout_p=0.10),\n            *down_convlayer(nfilt*2, nfilt*4, 4, 2, 1, dropout_p=0.15),\n            *down_convlayer(nfilt*4, nfilt*8, 4, 2, 1, dropout_p=0.25),\n            \n            MinibatchStdDev(),\n            spectral_norm(nn.Conv2d(nfilt*8 + 1, 1, 4, 1, 0, bias=False)),\n        )\n\n    def forward(self, inputs):\n        imgs, labels = inputs\n\n        enc = self.label_emb(labels).view((-1, 1, 64, 64))\n        enc = F.normalize(enc, p=2, dim=1)\n        x = torch.cat((imgs, enc), 1)   # 4 input feature maps(3rgb + 1label)\n        \n        out = self.model(x)\n        return out.view(-1)\n\n    \ndef weights_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        nn.init.normal_(m.weight.data, 0.0, 0.02)        \n    elif classname.find('BatchNorm') != -1:\n        nn.init.normal_(m.weight.data, 1.0, 0.02)\n        nn.init.constant_(m.bias.data, 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data loader"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"class DataGenerator(Dataset):\n    def __init__(self, directory, transform=None, n_samples=np.inf, crop_dogs=True):\n        self.directory = directory\n        self.transform = transform\n        self.n_samples = n_samples        \n        self.samples, self.labels = self.load_dogs_data(directory, crop_dogs)\n\n    def load_dogs_data(self, directory, crop_dogs):\n        required_transforms = torchvision.transforms.Compose([\n                torchvision.transforms.Resize(64),\n                torchvision.transforms.CenterCrop(64),\n        ])\n\n        imgs = []\n        labels = []\n        paths = []\n        for root, _, fnames in sorted(os.walk(directory)):\n            for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:\n                path = os.path.join(root, fname)\n                paths.append(path)\n\n        for path in paths:\n            # Load image\n            try: img = dset.folder.default_loader(path)\n            except: continue\n            \n            # Get bounding boxes\n            annotation_basename = os.path.splitext(os.path.basename(path))[0]\n            annotation_dirname = next(\n                    dirname for dirname in os.listdir('../input/annotation/Annotation/') if\n                    dirname.startswith(annotation_basename.split('_')[0]))\n                \n            if crop_dogs:\n                tree = ET.parse(os.path.join('../input/annotation/Annotation/',\n                                             annotation_dirname, annotation_basename))\n                root = tree.getroot()\n                objects = root.findall('object')\n                for o in objects:\n                    bndbox = o.find('bndbox')\n                    xmin = int(bndbox.find('xmin').text)\n                    ymin = int(bndbox.find('ymin').text)\n                    xmax = int(bndbox.find('xmax').text)\n                    ymax = int(bndbox.find('ymax').text)\n                    object_img = required_transforms(img.crop((xmin, ymin, xmax, ymax)))\n                    imgs.append(object_img)\n                    labels.append(annotation_dirname.split('-')[1].lower())\n\n            else:\n                object_img = required_transforms(img)\n                imgs.append(object_img)\n                labels.append(annotation_dirname.split('-')[1].lower())\n            \n        return imgs, labels\n    \n    \n    def __getitem__(self, index):\n        sample = self.samples[index]\n        label = self.labels[index]\n        \n        if self.transform is not None: \n            sample = self.transform(sample)\n        return np.asarray(sample), label\n\n    \n    def __len__(self):\n        return len(self.samples)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Parameters"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"database = '../input/all-dogs/all-dogs/'\ncrop_dogs = True\nn_samples = np.inf\nBATCH_SIZE = 32\n\nepochs = 401\n\nuse_soft_noisy_labels=True\n\nnz = 128\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\ntransform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5),\n                                transforms.ToTensor(),\n                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\ntrain_data = DataGenerator(database, transform=transform, n_samples=n_samples, crop_dogs=crop_dogs)\n\ndecoded_dog_labels = {i:breed for i, breed in enumerate(sorted(set(train_data.labels)))}\nencoded_dog_labels = {breed:i for i, breed in enumerate(sorted(set(train_data.labels)))}\ntrain_data.labels = [encoded_dog_labels[l] for l in train_data.labels] # encode dog labels in the data generator\n\n\ntrain_loader = torch.utils.data.DataLoader(train_data, shuffle=True,\n                                           batch_size=BATCH_SIZE, num_workers=4)\n\n\nprint(\"Dog breeds loaded:  \", len(encoded_dog_labels))\nprint(\"Data samples loaded:\", len(train_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"netG = Generator(nz, num_classes=len(encoded_dog_labels), nfilt=64).to(device)\nnetD = Discriminator(num_classes=len(encoded_dog_labels), nfilt=64).to(device)\nweights_init(netG)\nweights_init(netD)\nprint(\"Generator parameters:    \", sum(p.numel() for p in netG.parameters() if p.requires_grad))\nprint(\"Discriminator parameters:\", sum(p.numel() for p in netD.parameters() if p.requires_grad))\n\noptimizerG = optim.Adam(netG.parameters(), lr=0.00025, betas=(0.1, 0.99))\noptimizerD = optim.Adam(netD.parameters(), lr=0.00100, betas=(0.1, 0.99))\n\nlr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG, T_0=epochs//20, eta_min=0.00001)\nlr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD, T_0=epochs//20, eta_min=0.00004)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mse(imageA, imageB):\n        err = np.sum((imageA.astype(\"float\") - imageB.astype(\"float\")) ** 2)\n        err /= float(imageA.shape[0] * imageA.shape[1])\n        return err\n\ndef show_generated_img(n_images=5, nz=128):\n    sample = []\n    for _ in range(n_images):\n        noise = torch.randn(1, nz, 1, 1, device=device)\n        dog_label = torch.randint(0, len(encoded_dog_labels), (1, ), device=device)\n        gen_image = netG((noise, dog_label)).to(\"cpu\").clone().detach().squeeze(0)\n        gen_image = gen_image.numpy().transpose(1, 2, 0)\n        sample.append(gen_image)\n        \n    figure, axes = plt.subplots(1, len(sample), figsize=(64, 64))\n    for index, axis in enumerate(axes):\n        axis.axis('off')\n        image_array = (sample[index] + 1.) / 2.\n        axis.imshow(image_array)\n    plt.show()\n\n    \ndef analyse_generated_by_class(n_images=5):\n    good_breeds = []\n    for l in range(len(decoded_dog_labels)):\n        sample = []\n        for _ in range(n_images):\n            noise = torch.randn(1, nz, 1, 1, device=device)\n            dog_label = torch.full((1,) , l, device=device, dtype=torch.long)\n            gen_image = netG((noise, dog_label)).to(\"cpu\").clone().detach().squeeze(0)\n            gen_image = gen_image.numpy().transpose(1, 2, 0)\n            sample.append(gen_image)\n        \n        d = np.round(np.sum([mse(sample[k], sample[k+1]) for k in range(len(sample)-1)])/n_images, 1)\n        if d < 1.0: continue  # had mode colapse(discard)\n            \n        print(f\"Generated breed({d}): \", decoded_dog_labels[l])\n        figure, axes = plt.subplots(1, len(sample), figsize=(64, 64))\n        for index, axis in enumerate(axes):\n            axis.axis('off')\n            image_array = (sample[index] + 1.) / 2.\n            axis.imshow(image_array)\n        plt.show()\n        \n        good_breeds.append(l)\n    return good_breeds\n\n\ndef create_submit(good_breeds):\n    print(\"Creating submit\")\n    os.makedirs('../output_images', exist_ok=True)\n    im_batch_size = 100\n    n_images = 10000\n    \n    all_dog_labels = np.random.choice(good_breeds, size=n_images, replace=True)\n    for i_batch in range(0, n_images, im_batch_size):\n        noise = torch.randn(im_batch_size, nz, 1, 1, device=device)\n        dog_labels = torch.from_numpy(all_dog_labels[i_batch: (i_batch+im_batch_size)]).to(device)\n        gen_images = netG((noise, dog_labels))\n        gen_images = (gen_images.to(\"cpu\").clone().detach() + 1) / 2\n        for ii, img in enumerate(gen_images):\n            save_image(gen_images[ii, :, :, :], os.path.join('../output_images', f'image_{i_batch + ii:05d}.png'))\n            \n    import shutil\n    shutil.make_archive('images', 'zip', '../output_images')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(epochs):\n    epoch_time = time.perf_counter()\n    if time.perf_counter() - kernel_start_time > 31000:\n            print(\"Time limit reached! Stopping kernel!\"); break\n\n    for ii, (real_images, dog_labels) in enumerate(train_loader):\n        if real_images.shape[0]!= BATCH_SIZE: continue\n        \n        if use_soft_noisy_labels:\n            real_labels = torch.squeeze(torch.empty((BATCH_SIZE, 1), device=device).uniform_(0.70, 0.95))\n            fake_labels = torch.squeeze(torch.empty((BATCH_SIZE, 1), device=device).uniform_(0.05, 0.15))\n            for p in np.random.choice(BATCH_SIZE, size=np.random.randint((BATCH_SIZE//8)), replace=False):\n                real_labels[p], fake_labels[p] = fake_labels[p], real_labels[p] # swap labels\n        else:\n            real_labels = torch.full((BATCH_SIZE, 1), 1.0, device=device)\n            fake_labels = torch.full((BATCH_SIZE, 1), 0.0, device=device)\n        \n        ############################\n        # (1) Update D network\n        ###########################\n        netD.zero_grad()\n\n        dog_labels = torch.tensor(dog_labels, device=device)\n        real_images = real_images.to(device)\n        noise = torch.randn(BATCH_SIZE, nz, 1, 1, device=device)\n        \n        outputR = netD((real_images, dog_labels))\n        fake_images = netG((noise, dog_labels))\n\n        outputF = netD((fake_images.detach(), dog_labels))\n        errD = (torch.mean((outputR - torch.mean(outputF) - real_labels) ** 2) + \n                torch.mean((outputF - torch.mean(outputR) + real_labels) ** 2))/2\n        errD.backward(retain_graph=True)\n        optimizerD.step()\n\n        ############################\n        # (2) Update G network\n        ###########################\n        netG.zero_grad()\n        \n        outputF = netD((fake_images, dog_labels))\n        errG = (torch.mean((outputR - torch.mean(outputF) + real_labels) ** 2) +\n                torch.mean((outputF - torch.mean(outputR) - real_labels) ** 2))/2\n        errG.backward()\n        optimizerG.step()\n        \n        lr_schedulerG.step(epoch)\n        lr_schedulerD.step(epoch)\n\n    if epoch % 10 == 0:\n        print('%.2fs [%d/%d] Loss_D: %.4f Loss_G: %.4f' % (\n              time.perf_counter()-epoch_time, epoch+1, epochs, errD.item(), errG.item()))\n        show_generated_img(6)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualise generated results by label and submit"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"good_breeds = analyse_generated_by_class(6)\ncreate_submit(good_breeds)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}