{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nBased on StyleGAN\nhttps://github.com/rosinality/style-based-gan-pytorch\n\nAuthor: Mark Peng\n\"\"\"\n\n%load_ext autoreload\n%autoreload 2\n\n# https://docs.fast.ai/dev/gpu.html#gpu-memory-notes\n# !pip3 install nvidia-ml-py3\n\nimport os\nimport random, math\nfrom math import sqrt\nimport time\nimport copy\nimport torch\nimport torch.nn as nn\nfrom torch.nn import init\nimport torch.nn.parallel\nimport torch.backends.cudnn as cudnn\nimport torch.optim as optim\nimport torch.utils.data\nimport torch.nn.functional as F\nfrom torch import Tensor\nfrom torch.nn import Parameter\nfrom torch.nn.utils import spectral_norm\nfrom torch.autograd import Function, Variable, grad\n\nfrom torchvision import datasets, transforms, utils\nfrom torchvision.utils import save_image, make_grid\n\nfrom scipy.stats import truncnorm\n\nimport xml.etree.ElementTree as ET\nfrom tqdm import tqdm\nimport numpy as np\nimport shutil\n\nimport matplotlib.pyplot as plt\nimport matplotlib.animation as animation\n%matplotlib inline\nfrom IPython.display import HTML\n\nimport gc\ngc.enable()\n\ntorch.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# kernel_mode = False\nkernel_mode = True\n\nif kernel_mode:\n    show_metric = False\n    show_animation = False\n    save_middle_model = False\n    \n    if show_metric:\n        dataset_folder = \"../input/generative-dog-images\"\n        image_folder = \"../input/generative-dog-images/all-dogs/all-dogs\"\n        annotation_folder = \"../input/generative-dog-images/annotation\"\n    else:\n        dataset_folder = \"../input\"\n        image_folder = \"../input/all-dogs\"\n        annotation_folder = \"../input/annotation\"\n\nelse:\n    show_metric = True\n    # show_metric = False\n    show_animation = True\n    save_middle_model = False\n    dataset_folder = \".\"\n    image_folder = \"all-dogs\"\n    annotation_folder = \".\"\n\n# os.listdir(dataset_folder)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\n\nmax_training_time = 32_300  # keep 100 seconds for submission\n# max_training_time = 32_100  # keep 5 minutes for submission\n# max_training_time = 31_800  # keep 10 minutes for submission, approx. iteration: 103_509 (P100)\n\nif kernel_mode:\n    max_iterations = 3_000_000\nelse:\n    max_iterations = 112_000\n\ncode_size = 128\nn_mlp = 8\nn_critic = 1  # accumualte iterations\n\n\nclass Args:\n    base_lr = 0.0015\n    lr = {8: 0.002, 16: 0.004, 32: 0.006, 64: 0.008} # markpeng - faster learing rate\n# lr = {8: 0.0015, 16: 0.003, 32: 0.004, 64: 0.006} # markpeng - faster learing rate\n#     lr = {8: 0.0015, 16: 0.002, 32: 0.003, 64: 0.005} # markpeng - faster learing rate\n    batch = {8: 128, 16: 64, 32: 32, 64: 32}\n    phase = {8: 800_000, 16: 400_000, 32: 400_000, 64: 400_000}\n    init_size = 8\n    max_size = 64\n    mixing = True\n    loss = 'r1'  # or 'wgan-gp'\n\n\nargs = Args()\n\n# Beta hyperparam for Adam optimizers\nbeta1 = 0.0\nbeta2 = 0.99\n# beta1 = 0.5\n# beta2 = 0.999\n\ntransform = transforms.Compose([\n    transforms.RandomHorizontalFlip(p=0.3),  # previous: 0.1\n    # transforms.RandomHorizontalFlip(p=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n])\n\nif kernel_mode:\n    show_animation_iters = 2000\n    state_print_iters = 1000\nelse:\n    show_animation_iters = 2000\n    state_print_iters = 100\n\nmifid_check_iters = 2000\nmifid_check_images = 500\nearly_stopping_patience = 3\n\ntruncnorm_threshold = 1\n\nif show_metric:\n    predict_batch_size = 25\nelse:\n    predict_batch_size = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# Dataset Loader"},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"def load_dataset_images(root, n_samples=25000, image_size=64):\n    IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif',\n                      '.tiff', '.webp')\n\n    def is_valid_file(x):\n        return datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)\n\n    required_transforms = transforms.Compose([\n        transforms.Resize(image_size),\n        transforms.CenterCrop(image_size),\n    ])\n\n    imgs = []\n    paths = []\n    for root, _, fnames in sorted(os.walk(root)):\n        for fname in sorted(fnames)[:min(n_samples, 999999999999999)]:\n            path = os.path.join(root, fname)\n            paths.append(path)\n\n    for path in paths:\n        if is_valid_file(path):\n            # Load image\n            img = datasets.folder.default_loader(path)\n\n            # Get bounding boxes\n            annotation_basename = os.path.splitext(os.path.basename(path))[0]\n            annotation_dirname = next(\n                dirname\n                for dirname in os.listdir(f'{annotation_folder}/Annotation/')\n                if dirname.startswith(annotation_basename.split('_')[0]))\n            annotation_filename = os.path.join(\n                f'{annotation_folder}/Annotation/', annotation_dirname,\n                annotation_basename)\n            tree = ET.parse(annotation_filename)\n            root = tree.getroot()\n            objects = root.findall('object')\n            for o in objects:\n                bndbox = o.find('bndbox')\n                xmin = int(bndbox.find('xmin').text)\n                ymin = int(bndbox.find('ymin').text)\n                xmax = int(bndbox.find('xmax').text)\n                ymax = int(bndbox.find('ymax').text)\n\n                w = np.min((xmax - xmin, ymax - ymin))\n                bbox = (xmin, ymin, xmin + w, ymin + w)\n\n                object_img = required_transforms(img.crop(bbox))\n\n                imgs.append(object_img)\n                \n    print(f\"Number of cropped dog images: {len(imgs)}\")\n\n    return imgs\n\n# Load all images into memory once for reuse\nin_memory_images = load_dataset_images(image_folder)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"class DataGenerator(torch.utils.data.Dataset):\n    def __init__(self,\n                 dataset_images=None,\n                 transform=None,\n                 target_image_size=64):\n        self.dataset_images = dataset_images\n        self.transform = transform\n        self.target_image_size = target_image_size\n        self.samples = []\n\n        if self.target_image_size < 64:\n            print(f\"Resizing images for DataGenerator to {target_image_size}x{target_image_size} ......\")\n            resize_start_time = time.time()\n            required_transforms = transforms.Compose(\n                [transforms.Resize(self.target_image_size)])\n            for img in dataset_images:\n                self.samples.append(required_transforms(img))\n\n            print(f\"Time spent on resizing images: \" + \\\n                  f\"{(time.time() - resize_start_time):.2f} seconds\")\n        else:\n            self.samples = self.dataset_images\n\n    def __getitem__(self, index):\n        sample = self.samples[index]\n\n        if self.transform is not None:\n            sample = self.transform(sample)\n\n        return np.asarray(sample)\n\n    def __len__(self):\n        return len(self.samples)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"def sample_data(batch_size, image_size=8):\n    train_data = DataGenerator(\n        in_memory_images, transform=transform, target_image_size=image_size)\n    train_loader = torch.utils.data.DataLoader(\n        train_data, shuffle=True, batch_size=batch_size, num_workers=4)\n\n    return train_loader","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"# train_loader = sample_data(batch_size, image_size=8)\n# train_loader = sample_data(batch_size, image_size=16)\n# train_loader = sample_data(batch_size, image_size=32)\n# train_loader = sample_data(batch_size, image_size=64)\n# next(iter(train_loader)).shape","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"# Resizing images for DataGenerator to 8x8 ......\n# Time spent on resizing images: 0.45 seconds\n# Resizing images for DataGenerator to 16x16 ......\n# Time spent on resizing images: 0.50 seconds\n# Resizing images for DataGenerator to 32x32 ......\n# Time spent on resizing images: 0.63 seconds\n\n# torch.Size([32, 3, 64, 64])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GAN Model Definition"},{"metadata":{"trusted":true},"cell_type":"code","source":"def requires_grad(model, flag=True):\n    for p in model.parameters():\n        p.requires_grad = flag\n\n\ndef accumulate(model1, model2, decay=0.999):\n    par1 = dict(model1.named_parameters())\n    par2 = dict(model2.named_parameters())\n\n    for k in par1.keys():\n        par1[k].data.mul_(decay).add_(1 - decay, par2[k].data)\n\n\ndef adjust_lr(optimizer, lr):\n    for group in optimizer.param_groups:\n        mult = group.get('mult', 1)\n        group['lr'] = lr * mult\n\n\ndef init_linear(linear):\n    init.xavier_normal(linear.weight)\n    linear.bias.data.zero_()\n\n\ndef init_conv(conv, glu=True):\n    init.kaiming_normal(conv.weight)\n    if conv.bias is not None:\n        conv.bias.data.zero_()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EqualLR:\n    def __init__(self, name):\n        self.name = name\n\n    def compute_weight(self, module):\n        weight = getattr(module, self.name + '_orig')\n        fan_in = weight.data.size(1) * weight.data[0][0].numel()\n\n        return weight * sqrt(2 / fan_in)\n\n    @staticmethod\n    def apply(module, name):\n        fn = EqualLR(name)\n\n        weight = getattr(module, name)\n        del module._parameters[name]\n        module.register_parameter(name + '_orig', nn.Parameter(weight.data))\n        module.register_forward_pre_hook(fn)\n\n        return fn\n\n    def __call__(self, module, input):\n        weight = self.compute_weight(module)\n        setattr(module, self.name, weight)\n\n\ndef equal_lr(module, name='weight'):\n    EqualLR.apply(module, name)\n\n    return module\n\n\nclass FusedUpsample(nn.Module):\n    def __init__(self, in_channel, out_channel, kernel_size, padding=0):\n        super().__init__()\n\n        weight = torch.randn(in_channel, out_channel, kernel_size, kernel_size)\n        bias = torch.zeros(out_channel)\n\n        fan_in = in_channel * kernel_size * kernel_size\n        self.multiplier = sqrt(2 / fan_in)\n\n        self.weight = nn.Parameter(weight)\n        self.bias = nn.Parameter(bias)\n\n        self.pad = padding\n\n    def forward(self, input):\n        weight = F.pad(self.weight * self.multiplier, [1, 1, 1, 1])\n        weight = (weight[:, :, 1:, 1:] + weight[:, :, :-1, 1:] +\n                  weight[:, :, 1:, :-1] + weight[:, :, :-1, :-1]) / 4\n\n        out = F.conv_transpose2d(input,\n                               weight,\n                               self.bias,\n                               stride=2,\n                               padding=self.pad)\n\n        return out\n\n\nclass FusedDownsample(nn.Module):\n    def __init__(self, in_channel, out_channel, kernel_size, padding=0):\n        super().__init__()\n\n        weight = torch.randn(out_channel, in_channel, kernel_size, kernel_size)\n        bias = torch.zeros(out_channel)\n\n        fan_in = in_channel * kernel_size * kernel_size\n        self.multiplier = sqrt(2 / fan_in)\n\n        self.weight = nn.Parameter(weight)\n        self.bias = nn.Parameter(bias)\n\n        self.pad = padding\n\n    def forward(self, input):\n        weight = F.pad(self.weight * self.multiplier, [1, 1, 1, 1])\n        weight = (weight[:, :, 1:, 1:] + weight[:, :, :-1, 1:] +\n                  weight[:, :, 1:, :-1] + weight[:, :, :-1, :-1]) / 4\n\n        out = F.conv2d(input, weight, self.bias, stride=2, padding=self.pad)\n\n        return out\n\n\nclass PixelNorm(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input):\n        return input / torch.sqrt(\n            torch.mean(input**2, dim=1, keepdim=True) + 1e-8)\n\n\nclass BlurFunctionBackward(Function):\n    @staticmethod\n    def forward(ctx, grad_output, kernel, kernel_flip):\n        ctx.save_for_backward(kernel, kernel_flip)\n\n        grad_input = F.conv2d(grad_output,\n                              kernel_flip,\n                              padding=1,\n                              groups=grad_output.shape[1])\n\n        return grad_input\n\n    @staticmethod\n    def backward(ctx, gradgrad_output):\n        kernel, kernel_flip = ctx.saved_tensors\n\n        grad_input = F.conv2d(gradgrad_output,\n                              kernel,\n                              padding=1,\n                              groups=gradgrad_output.shape[1])\n\n        return grad_input, None, None\n\n\nclass BlurFunction(Function):\n    @staticmethod\n    def forward(ctx, input, kernel, kernel_flip):\n        ctx.save_for_backward(kernel, kernel_flip)\n\n        output = F.conv2d(input, kernel, padding=1, groups=input.shape[1])\n\n        return output\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        kernel, kernel_flip = ctx.saved_tensors\n\n        grad_input = BlurFunctionBackward.apply(grad_output, kernel,\n                                                kernel_flip)\n\n        return grad_input, None, None\n\n\nblur = BlurFunction.apply\n\n\nclass Blur(nn.Module):\n    def __init__(self, channel):\n        super().__init__()\n\n        weight = torch.tensor([[1, 2, 1], [2, 4, 2], [1, 2, 1]],\n                              dtype=torch.float32)\n        weight = weight.view(1, 1, 3, 3)\n        weight = weight / weight.sum()\n        weight_flip = torch.flip(weight, [2, 3])\n\n        self.register_buffer('weight', weight.repeat(channel, 1, 1, 1))\n        self.register_buffer('weight_flip',\n                             weight_flip.repeat(channel, 1, 1, 1))\n\n    def forward(self, input):\n        return blur(input, self.weight, self.weight_flip)\n        # return F.conv2d(input, self.weight, padding=1, groups=input.shape[1])\n\n\nclass EqualConv2d(nn.Module):\n    def __init__(self, *args, **kwargs):\n        super().__init__()\n\n        conv = nn.Conv2d(*args, **kwargs)\n        conv.weight.data.normal_()\n        conv.bias.data.zero_()\n        conv = conv\n        self.conv = equal_lr(conv)\n\n    def forward(self, input):\n        return self.conv(input)\n\n\nclass EqualLinear(nn.Module):\n    def __init__(self, in_dim, out_dim):\n        super().__init__()\n\n        linear = nn.Linear(in_dim, out_dim)\n        linear.weight.data.normal_()\n        linear.bias.data.zero_()\n\n        self.linear = equal_lr(linear)\n\n    def forward(self, input):\n        return self.linear(input)\n\n\nclass ConvBlock(nn.Module):\n    def __init__(\n            self,\n            in_channel,\n            out_channel,\n            kernel_size,\n            padding,\n            kernel_size2=None,\n            padding2=None,\n            downsample=False,\n            fused=False,\n    ):\n        super().__init__()\n\n        pad1 = padding\n        pad2 = padding\n        if padding2 is not None:\n            pad2 = padding2\n\n        kernel1 = kernel_size\n        kernel2 = kernel_size\n        if kernel_size2 is not None:\n            kernel2 = kernel_size2\n\n        self.conv1 = nn.Sequential(\n            EqualConv2d(in_channel, out_channel, kernel1, padding=pad1),\n            nn.LeakyReLU(0.2),\n        )\n\n        if downsample:\n            if fused:\n                self.conv2 = nn.Sequential(\n                    Blur(out_channel),\n                    FusedDownsample(out_channel,\n                                    out_channel,\n                                    kernel2,\n                                    padding=pad2),\n                    nn.LeakyReLU(0.2),\n                )\n\n            else:\n                self.conv2 = nn.Sequential(\n                    Blur(out_channel),\n                    EqualConv2d(out_channel,\n                                out_channel,\n                                kernel2,\n                                padding=pad2),\n                    nn.AvgPool2d(2),\n                    nn.LeakyReLU(0.2),\n                )\n\n        else:\n            self.conv2 = nn.Sequential(\n                EqualConv2d(out_channel, out_channel, kernel2, padding=pad2),\n                nn.LeakyReLU(0.2),\n            )\n\n    def forward(self, input):\n        out = self.conv1(input)\n        out = self.conv2(out)\n\n        return out\n\n\nclass AdaptiveInstanceNorm(nn.Module):\n    def __init__(self, in_channel, style_dim):\n        super().__init__()\n\n        self.norm = nn.InstanceNorm2d(in_channel)\n        self.style = EqualLinear(style_dim, in_channel * 2)\n\n        self.style.linear.bias.data[:\n                                    in_channel] = 1  # set bias to 1 for style associated\n        self.style.linear.bias.data[in_channel:] = 0\n\n    def forward(self, input, style):\n        style = self.style(style).unsqueeze(2).unsqueeze(3)\n        gamma, beta = style.chunk(2, 1)\n\n        out = self.norm(input)\n        out = gamma * out + beta\n\n        return out\n\n\nclass NoiseInjection(nn.Module):\n    def __init__(self, channel):\n        super().__init__()\n\n        self.weight = nn.Parameter(torch.zeros(1, channel, 1, 1))\n\n    def forward(self, image, noise):\n        return image + self.weight * noise\n\n\nclass ConstantInput(nn.Module):\n    def __init__(self, channel, size=4):\n        super().__init__()\n\n        self.input = nn.Parameter(torch.randn(1, channel, size, size))\n\n    def forward(self, input):\n        batch = input.shape[0]\n        out = self.input.repeat(batch, 1, 1, 1)\n\n        return out\n\n\nclass StyledConvBlock(nn.Module):\n    def __init__(\n            self,\n            in_channel,\n            out_channel,\n            kernel_size=3,\n            padding=1,\n            style_dim=code_size,\n            initial=False,\n            upsample=False,\n            fused=False,\n    ):\n        super().__init__()\n\n        if initial:\n            self.conv1 = ConstantInput(in_channel)\n\n        else:\n            if upsample:\n                if fused:\n                    self.conv1 = nn.Sequential(\n                        FusedUpsample(in_channel,\n                                      out_channel,\n                                      kernel_size,\n                                      padding=padding),\n                        Blur(out_channel),\n                    )\n\n                else:\n                    self.conv1 = nn.Sequential(\n                        nn.Upsample(scale_factor=2, mode='nearest'),\n                        EqualConv2d(in_channel,\n                                    out_channel,\n                                    kernel_size,\n                                    padding=padding),\n                        Blur(out_channel),\n                    )\n\n            else:\n                self.conv1 = EqualConv2d(in_channel,\n                                         out_channel,\n                                         kernel_size,\n                                         padding=padding)\n\n        self.noise1 = equal_lr(NoiseInjection(out_channel))\n        self.adain1 = AdaptiveInstanceNorm(out_channel, style_dim)\n        self.lrelu1 = nn.LeakyReLU(0.2)\n\n        self.conv2 = EqualConv2d(out_channel,\n                                 out_channel,\n                                 kernel_size,\n                                 padding=padding)\n        self.noise2 = equal_lr(NoiseInjection(out_channel))\n        self.adain2 = AdaptiveInstanceNorm(out_channel, style_dim)\n        self.lrelu2 = nn.LeakyReLU(0.2)\n\n    def forward(self, input, style, noise):\n        out = self.conv1(input)\n        out = self.noise1(out, noise)\n        out = self.lrelu1(out)\n        out = self.adain1(out, style)\n\n        out = self.conv2(out)\n        out = self.noise2(out, noise)\n        out = self.lrelu2(out)\n        out = self.adain2(out, style)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Generator(nn.Module):\n    def __init__(self, code_dim, fused=True):\n        super().__init__()\n\n        self.progression = nn.ModuleList([\n            StyledConvBlock(128, 128, 3, 1, initial=True),  # 4\n            StyledConvBlock(128, 128, 3, 1, upsample=True),  # 8\n            StyledConvBlock(128, 64, 3, 1, upsample=True),  # 16\n            StyledConvBlock(64, 32, 3, 1, upsample=True, fused=fused),  # 32\n            StyledConvBlock(32, 16, 3, 1, upsample=True, fused=fused),  # 64\n\n            #             StyledConvBlock(512, 512, 3, 1, initial=True),  # 4\n            #             StyledConvBlock(512, 512, 3, 1, upsample=True),  # 8\n            #             StyledConvBlock(512, 512, 3, 1, upsample=True),  # 16\n            #             StyledConvBlock(512, 512, 3, 1, upsample=True),  # 32\n            #             StyledConvBlock(512, 256, 3, 1, upsample=True),  # 64\n            #             StyledConvBlock(256, 128, 3, 1, upsample=True, fused=fused),  # 128\n            #             StyledConvBlock(128, 64, 3, 1, upsample=True, fused=fused),  # 256\n            #             StyledConvBlock(64, 32, 3, 1, upsample=True, fused=fused),  # 512\n            #             StyledConvBlock(32, 16, 3, 1, upsample=True, fused=fused),  # 1024\n        ])\n\n        self.to_rgb = nn.ModuleList([\n            EqualConv2d(128, 3, 1),\n            EqualConv2d(128, 3, 1),\n            EqualConv2d(64, 3, 1),\n            EqualConv2d(32, 3, 1),\n            EqualConv2d(16, 3, 1),\n\n            #             EqualConv2d(512, 3, 1),\n            #             EqualConv2d(512, 3, 1),\n            #             EqualConv2d(512, 3, 1),\n            #             EqualConv2d(512, 3, 1),\n            #             EqualConv2d(256, 3, 1),\n            #             EqualConv2d(128, 3, 1),\n            #             EqualConv2d(64, 3, 1),\n            #             EqualConv2d(32, 3, 1),\n            #             EqualConv2d(16, 3, 1),\n        ])\n\n        # self.blur = Blur()\n\n    def forward(self, style, noise, step=0, alpha=-1, mixing_range=(-1, -1)):\n        out = noise[0]\n\n        if len(style) < 2:\n            inject_index = [len(self.progression) + 1]\n\n        else:\n            inject_index = random.sample(list(range(step)), len(style) - 1)\n\n        crossover = 0\n\n        for i, (conv, to_rgb) in enumerate(zip(self.progression, self.to_rgb)):\n            if mixing_range == (-1, -1):\n                if crossover < len(\n                        inject_index) and i > inject_index[crossover]:\n                    crossover = min(crossover + 1, len(style))\n\n                style_step = style[crossover]\n\n            else:\n                if mixing_range[0] <= i <= mixing_range[1]:\n                    style_step = style[1]\n\n                else:\n                    style_step = style[0]\n\n            if i > 0 and step > 0:\n                out_prev = out\n\n                out = conv(out, style_step, noise[i])\n\n            else:\n                out = conv(out, style_step, noise[i])\n\n            if i == step:\n                out = to_rgb(out)\n\n                if i > 0 and 0 <= alpha < 1:\n                    skip_rgb = self.to_rgb[i - 1](out_prev)\n                    skip_rgb = F.interpolate(skip_rgb,\n                                             scale_factor=2,\n                                             mode='nearest')\n                    out = (1 - alpha) * skip_rgb + alpha * out\n\n                break\n\n        return out\n\n\nclass StyledGenerator(nn.Module):\n    def __init__(self, code_dim=128, n_mlp=8):\n        super().__init__()\n\n        self.generator = Generator(code_dim)\n\n        layers = [PixelNorm()]\n        for i in range(n_mlp):\n            layers.append(EqualLinear(code_dim, code_dim))\n            layers.append(nn.LeakyReLU(0.2))\n\n        self.style = nn.Sequential(*layers)\n\n    def forward(\n            self,\n            input,\n            noise=None,\n            step=0,\n            alpha=-1,\n            mean_style=None,\n            style_weight=0,\n            mixing_range=(-1, -1),\n    ):\n        styles = []\n        if type(input) not in (list, tuple):\n            input = [input]\n\n        for i in input:\n            styles.append(self.style(i))\n\n        batch = input[0].shape[0]\n\n        if noise is None:\n            noise = []\n\n            for i in range(step + 1):\n                size = 4 * 2**i\n                noise.append(\n                    torch.randn(batch, 1, size, size, device=input[0].device))\n\n        if mean_style is not None:\n            styles_norm = []\n\n            for style in styles:\n                styles_norm.append(mean_style + style_weight *\n                                   (style - mean_style))\n\n            styles = styles_norm\n\n        return self.generator(styles,\n                              noise,\n                              step,\n                              alpha,\n                              mixing_range=mixing_range)\n\n    def mean_style(self, input):\n        style = self.style(input).mean(0, keepdim=True)\n\n        return style","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Discriminator(nn.Module):\n    def __init__(self, fused=True):\n        super().__init__()\n\n        self.progression = nn.ModuleList([\n            ConvBlock(16, 32, 3, 1, downsample=True, fused=fused),  # 64\n            ConvBlock(32, 64, 3, 1, downsample=True, fused=fused),  # 32\n            ConvBlock(64, 128, 3, 1, downsample=True),  # 14\n            ConvBlock(128, 128, 3, 1, downsample=True),  # 8\n            ConvBlock(129, 128, 3, 1, 4, 0),  # 4\n\n            #             ConvBlock(16, 32, 3, 1, downsample=True, fused=fused),  # 512\n            #             ConvBlock(32, 64, 3, 1, downsample=True, fused=fused),  # 256\n            #             ConvBlock(64, 128, 3, 1, downsample=True, fused=fused),  # 128\n            #             ConvBlock(128, 256, 3, 1, downsample=True, fused=fused),  # 64\n            #             ConvBlock(256, 512, 3, 1, downsample=True),  # 32\n            #             ConvBlock(512, 512, 3, 1, downsample=True),  # 16\n            #             ConvBlock(512, 512, 3, 1, downsample=True),  # 8\n            #             ConvBlock(512, 512, 3, 1, downsample=True),  # 4\n            #             ConvBlock(513, 512, 3, 1, 4, 0),\n        ])\n\n        self.from_rgb = nn.ModuleList([\n            EqualConv2d(3, 16, 1),\n            EqualConv2d(3, 32, 1),\n            EqualConv2d(3, 64, 1),\n            EqualConv2d(3, 128, 1),\n            EqualConv2d(3, 128, 1),\n\n            #             EqualConv2d(3, 16, 1),\n            #             EqualConv2d(3, 32, 1),\n            #             EqualConv2d(3, 64, 1),\n            #             EqualConv2d(3, 128, 1),\n            #             EqualConv2d(3, 256, 1),\n            #             EqualConv2d(3, 512, 1),\n            #             EqualConv2d(3, 512, 1),\n            #             EqualConv2d(3, 512, 1),\n            #             EqualConv2d(3, 512, 1),\n        ])\n\n        # self.blur = Blur()\n\n        self.n_layer = len(self.progression)\n\n        self.linear = EqualLinear(code_size, 1)\n\n\n#         self.linear = EqualLinear(512, 1)\n\n    def forward(self, input, step=0, alpha=-1):\n        for i in range(step, -1, -1):\n            index = self.n_layer - i - 1\n\n            if i == step:\n                out = self.from_rgb[index](input)\n\n            if i == 0:\n                # Minibatch stddev\n                out_std = torch.sqrt(out.var(0, unbiased=False) + 1e-8)\n                mean_std = out_std.mean()\n                mean_std = mean_std.expand(out.size(0), 1, 4, 4)\n                out = torch.cat([out, mean_std], 1)\n\n            out = self.progression[index](out)\n\n            if i > 0:\n                if i == step and 0 <= alpha < 1:\n                    skip_rgb = F.avg_pool2d(input, 2)\n                    skip_rgb = self.from_rgb[index + 1](skip_rgb)\n\n                    out = (1 - alpha) * skip_rgb + alpha * out\n\n        out = out.squeeze(2).squeeze(2)\n        # print(input.size(), out.size(), step)\n        out = self.linear(out)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# MiFID Inference Model"},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport gzip, pickle\nimport tensorflow as tf\nfrom scipy import linalg\nimport pathlib\nimport urllib\nimport warnings\nfrom tqdm import tqdm\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"class KernelEvalException(Exception):\n    pass\n\ndef create_model_graph(pth):\n    \"\"\"Creates a graph from saved GraphDef file.\"\"\"\n    # Creates graph from saved graph_def.pb.\n    with tf.gfile.GFile(pth, 'rb') as f:\n        graph_def = tf.GraphDef()\n        graph_def.ParseFromString(f.read())\n        _ = tf.import_graph_def(graph_def, name='Pretrained_Net')\n\n\ndef _get_model_layer(sess, model_name):\n    # layername = 'Pretrained_Net/final_layer/Mean:0'\n    layername = model_params[model_name]['output_layer']\n    layer = sess.graph.get_tensor_by_name(layername)\n    ops = layer.graph.get_operations()\n    for op_idx, op in enumerate(ops):\n        for o in op.outputs:\n            shape = o.get_shape()\n            if shape._dims != []:\n                shape = [s.value for s in shape]\n                new_shape = []\n                for j, s in enumerate(shape):\n                    if s == 1 and j == 0:\n                        new_shape.append(None)\n                    else:\n                        new_shape.append(s)\n                o.__dict__['_shape_val'] = tf.TensorShape(new_shape)\n    return layer\n\n\ndef get_activations(images, sess, model_name, batch_size=32, verbose=False):\n    \"\"\"Calculates the activations of the pool_3 layer for all images.\n\n    Params:\n    -- images      : Numpy array of dimension (n_images, hi, wi, 3). The values\n                     must lie between 0 and 256.\n    -- sess        : current session\n    -- batch_size  : the images numpy array is split into batches with batch size\n                     batch_size. A reasonable batch size depends on the disposable hardware.\n    -- verbose    : If set to True and parameter out_step is given, the number of calculated\n                     batches is reported.\n    Returns:\n    -- A numpy array of dimension (num images, 2048) that contains the\n       activations of the given tensor when feeding inception with the query tensor.\n    \"\"\"\n    inception_layer = _get_model_layer(sess, model_name)\n    n_images = images.shape[0]\n    if batch_size > n_images:\n        print(\n            \"warning: batch size is bigger than the data size. setting batch size to data size\"\n        )\n        batch_size = n_images\n    n_batches = n_images // batch_size + 1\n    pred_arr = np.empty((n_images, model_params[model_name]['output_shape']))\n    for i in range(n_batches):\n    # for i in tqdm(range(n_batches)):\n        if verbose:\n            print(\"\\rPropagating batch %d/%d\" % (i + 1, n_batches),\n                  end=\"\",\n                  flush=True)\n        start = i * batch_size\n        if start + batch_size < n_images:\n            end = start + batch_size\n        else:\n            end = n_images\n\n        batch = images[start:end]\n        pred = sess.run(inception_layer,\n                        {model_params[model_name]['input_layer']: batch})\n        pred_arr[start:end] = pred.reshape(\n            -1, model_params[model_name]['output_shape'])\n    return pred_arr\n\n\ndef normalize_rows(x: np.ndarray):\n    \"\"\"\n    function that normalizes each row of the matrix x to have unit length.\n\n    Args:\n     ``x``: A numpy matrix of shape (n, m)\n\n    Returns:\n     ``x``: The normalized (by row) numpy matrix.\n    \"\"\"\n    return np.nan_to_num(x / np.linalg.norm(x, ord=2, axis=1, keepdims=True))\n\n\ndef cosine_distance(features1, features2):\n    features1_nozero = features1[np.sum(features1, axis=1) != 0]\n    features2_nozero = features2[np.sum(features2, axis=1) != 0]\n    norm_f1 = normalize_rows(features1_nozero)\n    norm_f2 = normalize_rows(features2_nozero)\n\n    d = 1.0 - np.abs(np.matmul(norm_f1, norm_f2.T))\n    mean_min_d = np.mean(np.min(d, axis=1))\n    return mean_min_d\n\n\ndef distance_thresholding(d, eps):\n    if d < eps:\n        return d\n    else:\n        return 1\n\n\ndef calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):\n    \"\"\"Numpy implementation of the Frechet Distance.\n    The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)\n    and X_2 ~ N(mu_2, C_2) is\n            d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).\n            \n    Stable version by Dougal J. Sutherland.\n\n    Params:\n    -- mu1 : Numpy array containing the activations of the pool_3 layer of the\n             inception net ( like returned by the function 'get_predictions')\n             for generated samples.\n    -- mu2   : The sample mean over activations of the pool_3 layer, precalcualted\n               on an representive data set.\n    -- sigma1: The covariance matrix over activations of the pool_3 layer for\n               generated samples.\n    -- sigma2: The covariance matrix over activations of the pool_3 layer,\n               precalcualted on an representive data set.\n\n    Returns:\n    --   : The Frechet Distance.\n    \"\"\"\n\n    mu1 = np.atleast_1d(mu1)\n    mu2 = np.atleast_1d(mu2)\n\n    sigma1 = np.atleast_2d(sigma1)\n    sigma2 = np.atleast_2d(sigma2)\n\n    assert mu1.shape == mu2.shape, \"Training and test mean vectors have different lengths\"\n    assert sigma1.shape == sigma2.shape, \"Training and test covariances have different dimensions\"\n\n    diff = mu1 - mu2\n\n    # product might be almost singular\n    covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)\n    if not np.isfinite(covmean).all():\n        msg = \"fid calculation produces singular product; adding %s to diagonal of cov estimates\" % eps\n        warnings.warn(msg)\n        offset = np.eye(sigma1.shape[0]) * eps\n        covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))\n\n    # numerical error might give slight imaginary component\n    if np.iscomplexobj(covmean):\n        if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):\n            m = np.max(np.abs(covmean.imag))\n            raise ValueError(\"Imaginary component {}\".format(m))\n        covmean = covmean.real\n\n    tr_covmean = np.trace(covmean)\n    return diff.dot(diff) + np.trace(sigma1) + np.trace(\n        sigma2) - 2 * tr_covmean\n\n\n#-------------------------------------------------------------------------------\n\n\ndef calculate_activation_statistics(images,\n                                    sess,\n                                    model_name,\n                                    batch_size=32,\n                                    verbose=False):\n    \"\"\"Calculation of the statistics used by the FID.\n    Params:\n    -- images      : Numpy array of dimension (n_images, hi, wi, 3). The values\n                     must lie between 0 and 255.\n    -- sess        : current session\n    -- batch_size  : the images numpy array is split into batches with batch size\n                     batch_size. A reasonable batch size depends on the available hardware.\n    -- verbose     : If set to True and parameter out_step is given, the number of calculated\n                     batches is reported.\n    Returns:\n    -- mu    : The mean over samples of the activations of the pool_3 layer of\n               the incption model.\n    -- sigma : The covariance matrix of the activations of the pool_3 layer of\n               the incption model.\n    \"\"\"\n    act = get_activations(images, sess, model_name, batch_size, verbose)\n    mu = np.mean(act, axis=0)\n    sigma = np.cov(act, rowvar=False)\n    return mu, sigma, act\n\n\ndef _handle_path_memorization(path, sess, model_name, n_images,\n                              is_checksize, is_check_png):\n    path = pathlib.Path(path)\n    files = list(path.glob('*.jpg')) + list(path.glob('*.png'))\n\n    seed_everything()\n    np.random.shuffle(files)\n    files = files[:n_images]\n    \n    imsize = model_params[model_name]['imsize']\n\n    # In production we don't resize input images. This is just for demo purpose.\n    x = np.array([\n        np.array(\n            img_read_checks(fn, imsize, is_checksize, imsize, is_check_png))\n        for fn in files\n    ])\n    m, s, features = calculate_activation_statistics(x, sess, model_name)\n    del x  #clean up memory\n    gc.collect()\n    return m, s, features\n\n\n# check for image size\ndef img_read_checks(filename,\n                    resize_to,\n                    is_checksize=False,\n                    check_imsize=64,\n                    is_check_png=False):\n    im = Image.open(str(filename))\n    if is_checksize and im.size != (check_imsize, check_imsize):\n        raise KernelEvalException('The images are not of size ' +\n                                  str(check_imsize))\n\n    if is_check_png and im.format != 'PNG':\n        raise KernelEvalException('Only PNG images should be submitted.')\n\n    if resize_to is None:\n        return im\n    else:\n        return im.resize((resize_to, resize_to), Image.ANTIALIAS)\n\n\ndef calculate_kid_given_paths(paths, model_name, model_path,\n                              n_images=1000,\n                              feature_path=None):\n    ''' Calculates the KID of two paths. '''\n    tf.reset_default_graph()\n    create_model_graph(str(model_path))\n    with tf.Session() as sess:\n        sess.run(tf.global_variables_initializer())\n        m1, s1, features1 = _handle_path_memorization(paths[0],\n                                                      sess,\n                                                      model_name,\n                                                      n_images=n_images,\n                                                      is_checksize=True,\n                                                      is_check_png=True)\n        if feature_path is None:\n            m2, s2, features2 = _handle_path_memorization(paths[1],\n                                                          sess,\n                                                          model_name,\n                                                          n_images=n_images,\n                                                          is_checksize=False,\n                                                          is_check_png=False)\n        else:\n            with np.load(feature_path) as f:\n                m2, s2, features2 = f['m'], f['s'], f['features']\n\n        fid_value = calculate_frechet_distance(m1, s1, m2, s2)\n        distance = cosine_distance(features1, features2)\n        return fid_value, distance","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"model_params = {\n    'Inception': {\n        'name': 'Inception',\n        'imsize': 64,\n        'output_layer': 'Pretrained_Net/pool_3:0',\n        'input_layer': 'Pretrained_Net/ExpandDims:0',\n        'output_shape': 2048,\n        'cosine_distance_eps': 0.1\n    }\n}\n\nuser_images_unzipped_path = '../output_images'\nimages_path = [user_images_unzipped_path, image_folder]\nif kernel_mode:\n    public_path = '../input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'\nelse:\n    public_path = 'classify_image_graph_def.pb'\nfid_epsilon = 10e-15","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"def compute_MiFID(n_images=1000):\n    seed_everything()\n    tf.set_random_seed(42)\n    fid_value_public, distance_public = calculate_kid_given_paths(\n        images_path, 'Inception', public_path, n_images)\n    distance_public = distance_thresholding(\n        distance_public, model_params['Inception']['cosine_distance_eps'])\n    miFID = fid_value_public / (distance_public + fid_epsilon)\n    print(\n        f\"FID_public: {fid_value_public}, distance_public: {distance_public} \"\n        + f\"multiplied_public: {miFID}\")\n    return miFID","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Style-GAN"},{"metadata":{"trusted":true},"cell_type":"code","source":"# From https://www.kaggle.com/sakami/ralsgan-dogs-cropping-random?scriptVersionId=17526139\n# Large Scale GAN Training for High Fidelity Natural Image Synthesis\ndef truncated_normal(size, threshold=1):\n    values = truncnorm.rvs(-threshold, threshold, size=size)\n    return values\n\n\ndef show_generated_img_all(num_samples=64, figsize=(20, 20)):\n    with torch.no_grad():\n        gen_images = g_running(torch.randn(num_samples, code_size).cuda(),\n                               step=step,\n                               alpha=alpha).data.cpu()\n        # gen_images = ((gen_images + 1.0) / 2.0)\n\n    # Plot the real images\n    plt.figure(figsize=figsize)\n    plt.subplot(1, 2, 1)\n    plt.axis(\"off\")\n    plt.imshow(\n        np.transpose(\n            make_grid(gen_images, padding=2, normalize=True,\n                      range=(-1, 1)).cpu(), (1, 2, 0)))\n\n    # plt.savefig(filename)\n\n\n### This is to show one sample image for iteration of chosing\ndef show_generated_img(iteration):\n    with torch.no_grad():\n        gen_images = g_running(torch.randn(4, code_size).cuda(),\n                               step=step,\n                               alpha=alpha).data.cpu()\n        # gen_images = ((gen_images + 1.0) / 2.0)\n\n    # Plot the real images\n    plt.figure(figsize=(15, 8))\n    plt.subplot(1, 4, 1)\n    plt.title(f\"Iteration {iteration+1}\")\n    plt.axis(\"off\")\n    plt.imshow(\n        np.transpose(\n            make_grid(gen_images, padding=2, normalize=True,\n                      range=(-1, 1)).cpu(), (1, 2, 0)))\n\n    plt.show()\n\n\ndef generate_validation_images(threshold=1, n_images=1000):\n    if not os.path.exists('../output_images'):\n        os.mkdir('../output_images')\n    else:\n        shutil.rmtree('../output_images', ignore_errors=True)\n        os.mkdir('../output_images')\n\n    im_batch_size = predict_batch_size\n    for i_batch in range(0, n_images, im_batch_size):\n        z = truncated_normal((im_batch_size, code_size), threshold=threshold)\n        gen_z = torch.from_numpy(z).float().to(device)\n\n        with torch.no_grad():\n            gen_images = g_running(gen_z, step=step, alpha=alpha).data.cpu()\n\n        images = gen_images.to(\"cpu\").clone().detach()\n        images = images.numpy().transpose(0, 2, 3, 1)\n\n        for i_image in range(gen_images.size(0)):\n            # Suggestion by Chris\n            # https://www.kaggle.com/wendykan/gan-dogs-starter/notebook#573699\n            # new_image = (gen_images[i_image, :, :, :] + 1.0) / 2.0\n\n            new_image = gen_images[i_image, :, :, :]\n            save_image(new_image,\n                       os.path.join('../output_images',\n                                    f'image_{i_batch+i_image:05d}.png'),\n                       normalize=True,\n                       range=(-1, 1))\n\n    generated_image_count = len(\n        [name for name in os.listdir('../output_images')])\n    print(f\"Number of generated images: {generated_image_count}\")\n\n\nclass EarlyStoppingCriterion(object):\n    def __init__(self, patience, mode, min_delta=0.0):\n        assert patience >= 0 and mode in {'min', 'max'} and min_delta >= 0.0\n        self.patience, self.mode, self.min_delta = patience, mode, min_delta\n        self._count, self._best_score, self.is_improved = 0, None, None\n\n    def step(self, cur_score):\n        if self._best_score is None:\n            self._best_score = cur_score\n            return True\n        else:\n            if self.mode == 'max':\n                self.is_improved = (cur_score >=\n                                    self._best_score + self.min_delta)\n            else:\n                self.is_improved = (cur_score <=\n                                    self._best_score - self.min_delta)\n\n            if self.is_improved:\n                self._count = 0\n                self._best_score = cur_score\n            else:\n                self._count += 1\n            return self._count <= self.patience\n\n    def state_dict(self):\n        return self.__dict__\n\n    def load_state_dict(self, state_dict):\n        self.__dict__.update(state_dict)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"generator = StyledGenerator(code_size, n_mlp).cuda()\ndiscriminator = Discriminator().cuda()\ng_running = StyledGenerator(code_size, n_mlp).cuda()\ng_running.train(False)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"print(discriminator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g_optimizer = optim.Adam(generator.generator.parameters(),\n                         lr=args.base_lr,\n                         betas=(beta1, beta2))\ng_optimizer.add_param_group({\n    'params': generator.style.parameters(),\n    'lr': args.base_lr * 0.01,\n    'mult': 0.01,\n})\nd_optimizer = optim.Adam(discriminator.parameters(),\n                         lr=args.base_lr,\n                         betas=(beta1, beta2))\n\naccumulate(g_running, generator, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://discuss.pytorch.org/t/how-do-i-check-the-number-of-parameters-of-a-model/4325/13\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\n\ndef save_models(model_name):\n    torch.save(\n        {\n            'generator': generator.state_dict(),\n            'discriminator': discriminator.state_dict(),\n            'g_optimizer': g_optimizer.state_dict(),\n            'd_optimizer': d_optimizer.state_dict(),\n            'g_running': g_running.state_dict()\n        },\n        f'{model_name}.model',\n    )\n\n\nprint(\n    f\"Generator Model Size: {count_parameters(generator):,} trainable parameters\"\n)\nprint(\n    f\"Discriminator Model Size: {count_parameters(discriminator):,} trainable parameters\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Redirect outputs to console\nimport sys\nif not kernel_mode:\n    jupyter_console = sys.stdout\n    sys.stdout = open('/dev/stdout', 'w')\n    # Append to log file\n    # sys.stdout = open(f\"stdout.log\", 'a')\n    # sys.stdout = jupyter_console","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","scrolled":true,"trusted":true},"cell_type":"code","source":"early_stopper = EarlyStoppingCriterion(patience=early_stopping_patience,\n                                       mode='min',\n                                       min_delta=1e-4)\n\nG_losses = []\nD_losses = []\ndisc_loss_val = 0\ngen_loss_val = 0\ngrad_loss_val = 0\n\ngc.collect()\n\nprint(\"Starting Training Loop...\")\n\ntrain_start_time = time.time()\n\nalpha = 0\nused_sample = 0\n\nstep = int(math.log2(args.init_size)) - 2\nresolution = 4 * 2**step\nphase = args.phase.get(resolution, 400_000)\n\nloader = sample_data(args.batch.get(resolution, batch_size),\n                     image_size=resolution)\ndata_loader = iter(loader)\n\nmax_step = int(math.log2(args.max_size)) - 2  # 4\nfinal_progress = False\n\nadjust_lr(g_optimizer, args.lr.get(resolution, args.base_lr))\nadjust_lr(d_optimizer, args.lr.get(resolution, args.base_lr))\nrequires_grad(generator, False)\nrequires_grad(discriminator, True)\n\nprint(f\"Training with resolution {resolution}x{resolution} (\" + \\\n      f\"step: {step}, batch_size: {args.batch.get(resolution, batch_size)}, \" + \\\n      f\"Generator LR: {g_optimizer.state_dict()['param_groups'][0]['lr']}, \" + \\\n      f\"Style LR: {g_optimizer.state_dict()['param_groups'][1]['lr']}, \" + \\\n      f\"Discriminator LR: {d_optimizer.state_dict()['param_groups'][0]['lr']})\")\n\ngo_ahead = True\nfor iters in range(max_iterations):\n    if not go_ahead:\n        break\n\n    gc.collect()\n\n    iter_start_time = time.time()\n\n    end = time.time()\n    if (end - start) > max_training_time:\n        print(\n            f\"Trained over {max_training_time:,} seconds, stopped at iteration {iters+1}\"\n        )\n        go_ahead = False\n        break\n\n    discriminator.zero_grad()\n\n    alpha = min(1, 1 / phase * (used_sample + 1))\n\n    if resolution == args.init_size or final_progress:\n        alpha = 1\n\n    if used_sample > phase * 2:\n        used_sample = 0\n        step += 1\n\n        if step > max_step:\n            step = max_step\n            final_progress = True\n        else:\n            alpha = 0\n\n        resolution = 4 * 2**step\n        phase = args.phase.get(resolution, 400_000)\n\n        del loader\n        gc.collect()\n        loader = sample_data(args.batch.get(resolution, batch_size),\n                             image_size=resolution)\n        data_loader = iter(loader)\n\n        # if save_middle_model:\n        #     save_models(f'stylegan_step-{step}')\n\n        adjust_lr(g_optimizer, args.lr.get(resolution, args.base_lr))\n        adjust_lr(d_optimizer, args.lr.get(resolution, args.base_lr))\n\n        print(f\"Training with resolution {resolution}x{resolution} (\" + \\\n              f\"step: {step}, batch_size: {args.batch.get(resolution, batch_size)}, \" + \\\n              f\"Generator LR: {g_optimizer.state_dict()['param_groups'][0]['lr']}, \" + \\\n              f\"Style LR: {g_optimizer.state_dict()['param_groups'][1]['lr']}, \" + \\\n              f\"Discriminator LR: {d_optimizer.state_dict()['param_groups'][0]['lr']})\")\n\n    try:\n        real_image = next(data_loader)\n\n    except (OSError, StopIteration):\n        data_loader = iter(loader)\n        real_image = next(data_loader)\n\n    used_sample += real_image.shape[0]\n\n    b_size = real_image.size(0)\n    real_image = real_image.cuda()\n\n    if args.loss == 'wgan-gp':\n        real_predict = discriminator(real_image, step=step, alpha=alpha)\n        real_predict = real_predict.mean() - 0.001 * (real_predict**2).mean()\n        (-real_predict).backward()\n\n    elif args.loss == 'r1':\n        real_image.requires_grad = True\n        real_predict = discriminator(real_image, step=step, alpha=alpha)\n        real_predict = F.softplus(-real_predict).mean()\n        real_predict.backward(retain_graph=True)\n\n        grad_real = grad(outputs=real_predict.sum(),\n                         inputs=real_image,\n                         create_graph=True)[0]\n        grad_penalty = (grad_real.view(grad_real.size(0),\n                                       -1).norm(2, dim=1)**2).mean()\n        grad_penalty = 10 / 2 * grad_penalty\n        grad_penalty.backward()\n        grad_loss_val = grad_penalty.item()\n\n    if args.mixing and random.random() < 0.9:\n        gen_in11, gen_in12, gen_in21, gen_in22 = torch.randn(\n            4, b_size, code_size, device='cuda').chunk(4, 0)\n        gen_in1 = [gen_in11.squeeze(0), gen_in12.squeeze(0)]\n        gen_in2 = [gen_in21.squeeze(0), gen_in22.squeeze(0)]\n\n    else:\n        gen_in1, gen_in2 = torch.randn(2, b_size, code_size,\n                                       device='cuda').chunk(2, 0)\n        gen_in1 = gen_in1.squeeze(0)\n        gen_in2 = gen_in2.squeeze(0)\n\n    fake_image = generator(gen_in1, step=step, alpha=alpha)\n    fake_predict = discriminator(fake_image, step=step, alpha=alpha)\n\n    if args.loss == 'wgan-gp':\n        fake_predict = fake_predict.mean()\n        fake_predict.backward()\n\n        eps = torch.rand(b_size, 1, 1, 1).cuda()\n        x_hat = eps * real_image.data + (1 - eps) * fake_image.data\n        x_hat.requires_grad = True\n        hat_predict = discriminator(x_hat, step=step, alpha=alpha)\n        grad_x_hat = grad(outputs=hat_predict.sum(),\n                          inputs=x_hat,\n                          create_graph=True)[0]\n        grad_penalty = (\n            (grad_x_hat.view(grad_x_hat.size(0), -1).norm(2, dim=1) -\n             1)**2).mean()\n        grad_penalty = 10 * grad_penalty\n        grad_penalty.backward()\n        grad_loss_val = grad_penalty.item()\n        disc_loss_val = (real_predict - fake_predict).item()\n\n    elif args.loss == 'r1':\n        fake_predict = F.softplus(fake_predict).mean()\n        fake_predict.backward()\n        disc_loss_val = (real_predict + fake_predict).item()\n\n    d_optimizer.step()\n\n    if (iters + 1) % n_critic == 0:\n        generator.zero_grad()\n\n        requires_grad(generator, True)\n        requires_grad(discriminator, False)\n\n        fake_image = generator(gen_in2, step=step, alpha=alpha)\n\n        predict = discriminator(fake_image, step=step, alpha=alpha)\n\n        if args.loss == 'wgan-gp':\n            loss = -predict.mean()\n\n        elif args.loss == 'r1':\n            loss = F.softplus(-predict).mean()\n\n        gen_loss_val = loss.item()\n\n        loss.backward()\n        g_optimizer.step()\n        accumulate(g_running, generator)\n\n        requires_grad(generator, False)\n        requires_grad(discriminator, True)\n\n    if show_animation and (iters + 1) % show_animation_iters == 0:\n        show_generated_img(iters)\n\n    if (iters + 1) % state_print_iters == 0:\n        print(f\"[{iters+1}/{max_iterations}] Phase Size:{phase}, \" + \\\n              f\"Resolution: {4 * 2 ** step}, \" + \\\n              f\"Loss_G: {gen_loss_val:.4f}, \" + \\\n              f\"Loss_D: {disc_loss_val:.4f}, \" + \\\n              f\"Grad: {grad_loss_val:.4f}, Alpha: {alpha:.5f}, Used Samples: {used_sample:,}\")\n        print(f\"Time spent on iteration {iters+1}: \" + \\\n              f\"{(time.time() - iter_start_time):.2f} seconds\")\n\n        # Save Losses for plotting later\n        G_losses.append(gen_loss_val)\n        D_losses.append(disc_loss_val)\n\n        if show_metric and (\n            (iters + 1) %\n                mifid_check_iters) == 0 and resolution == args.max_size:\n            generate_validation_images(threshold=truncnorm_threshold,\n                                       n_images=mifid_check_images)\n            val_mifid = compute_MiFID(mifid_check_images)\n\n            if early_stopper.step(val_mifid):\n                if early_stopper.is_improved:\n                    best_score = val_mifid\n                    best_iters = iters\n                    print(\n                        f'MiFID Improved: {val_mifid:.4f}, saving model ......'\n                    )\n\n                    # save_models(f'stylegan_{iters+1}iters')\n                else:\n                    print('No improvements in this iteration')\n                    gc.collect()\n            else:\n                print(\n                    f'No more improvement....(Best MiFID: {best_score:.6f}, {best_iters+1} iterations)'\n                )\n                gc.collect()\n                break\n\nif show_metric:\n    print(f'Best MiFID: {best_score:.6f}, {best_iters+1} iterations')\nelse:\n    best_iters = iters\nprint(f\"Total time spent on training: \" + \\\n      f\"{(time.time() - train_start_time)/60:.2f} minutes \" + \\\n      f\"({best_iters + 1} iterations, step: {step}, alpha: {alpha})\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# [219300/3000000] Phase:400000, Resolution: 64, Loss_G: 7.3933, Loss_D: 0.0653, Grad: 0.0087, Alpha: 1.00000, Used Samples: 710,848\n# Time spent on iteration 219300: 0.10 seconds\n# Trained over 30,000 seconds, stopped at iteration 219363\n# Total time spent on training: 499.20 minutes (219363 iterations)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not show_metric:\n    final_model = f'stylegan_{best_iters+1}iters'\n    save_models(final_model)\n    print(f\"{final_model}.model saved.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss versus training iteration"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nplt.title(\"Generator and Discriminator Loss During Training\")\nplt.plot(G_losses, label=\"G\")\nplt.plot(D_losses, label=\"D\")\nplt.xlabel(f\"Iterations (Total: {best_iters+1:,})\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generate Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# models = torch.load(f\"stylegan_{best_iters+1}iters.model\")\n# generator.load_state_dict(models[\"generator\"])\n# discriminator.load_state_dict(models[\"discriminator\"])\n# g_running.load_state_dict(models[\"g_running\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# From https://www.kaggle.com/sakami/ralsgan-dogs-cropping-random?scriptVersionId=17526139\n# Large Scale GAN Training for High Fidelity Natural Image Synthesis\ndef truncated_normal(size, threshold=1):\n    values = truncnorm.rvs(-threshold, threshold, size=size)\n    return values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nif not os.path.exists('../output_images'):\n    os.mkdir('../output_images')\nelse:\n    shutil.rmtree('../output_images', ignore_errors=True)\n    os.mkdir('../output_images')\n\n# im_batch_size = 25\nim_batch_size = 50\nn_images = 10000\n\nfirst_batch = []\nfor i_batch in range(0, n_images, im_batch_size):\n    z = truncated_normal((im_batch_size, code_size),\n                         threshold=truncnorm_threshold)\n    gen_z = torch.from_numpy(z).float().to(device)\n\n    with torch.no_grad():\n        gen_images = g_running(gen_z, step=step, alpha=alpha).data.cpu()\n        # Suggestion by Chris\n        # https://www.kaggle.com/wendykan/gan-dogs-starter/notebook#573699\n        # gen_images = ((gen_images + 1.0) / 2.0)\n\n    for i_image in range(gen_images.size(0)):\n        new_image = gen_images[i_image, :, :, :]\n\n        if i_batch == 0:\n            first_batch.append(new_image)\n\n        save_image(new_image,\n                   os.path.join('../output_images',\n                                f'image_{i_batch+i_image:05d}.png'),\n                   normalize=True,\n                   range=(-1, 1))\n\ngenerated_image_count = len([name for name in os.listdir('../output_images')])\nprint(f\"Number of generated images: {generated_image_count}\")\n\nshutil.make_archive('images', 'zip', '../output_images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the saved images\n# plt.figure(figsize=(20, 20))\n# plt.subplot(1, 2, 1)\n# plt.axis(\"off\")\n# plt.title(\"Submission Images\")\n# plt.imshow(\n#     np.transpose(\n#         make_grid(first_batch, padding=2, normalize=True, range=(-1, 1)).cpu(),\n#         (1, 2, 0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nshow_generated_img_all(64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"03fbf842146a4ed8bb86f3e093928a7e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"066cc89966c34979bcc15995638b3742":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"07bbd6e19e6c42c991bc60572604fdc6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a8c494574ed441afbf58e17d3daecda7","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_6dbd62d02079475ab4a1286b495c8e5f","value":692}},"09cd31509a094883a2bf8b6b6242b88d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_8cb8bdebd0a2465d80185fa72bec5399","IPY_MODEL_8acdfd5ce03d41dba5e76b8b1c61ea12"],"layout":"IPY_MODEL_7e07916f4f8c433c93659bb2e476c78c"}},"0bdfce1939314391bfba3a2c273159fa":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c5276e5290154b2a8fc16f220a04fa19","placeholder":"​","style":"IPY_MODEL_4777be7511fd4d36b029541830bd3b14","value":"100% 692/692 [00:34&lt;00:00, 21.70it/s]"}},"129e125b6f514df6b18710cc12e0417b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_cc57ed78cc494670afc471bc1f8d8f8a","placeholder":"​","style":"IPY_MODEL_b149c5c99f904d1b9c230dbac8bbbce9","value":"100% 692/692 [00:34&lt;00:00, 22.05it/s]"}},"139d9e27b0bb4a05bb6b19b22a9d6c4c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_75aec803fe1344f396a5b480370640f4","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2ff69fba0f8d45429313762df3b329d3","value":692}},"1843bf4479b8486f91409bd1f5bcdc2d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"19eaa5438f4c40128dcd46cb8a5eeaaa":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"22b298db91764115ac714897f92a9f52":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"25ff35274c254f3e9de2ca98c80ab2c0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_578fbab606a447f9b09a82c71c56c5b4","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5feea5e5918444ceb156be86ff7adf85","value":692}},"29aae73ccda544648ebac551ebfbe20f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2a359dbbbdbd4a4a8c7baca11a15a8b9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2db1dfd55ab848f38cb449dd34881bce":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2ff69fba0f8d45429313762df3b329d3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"31d88db94ddd4ebd9858ed6ce87996c3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"334fd22534c14825b9c60875ab1af27e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"36093696cf7e40e888e78a7d7f066422":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3d5209c342b24b9d92b514a493ccbc8f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3d8f028141a44ad1b6cbf60435e8f3fa":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3ec3efc280844fd5ae0b67093fb61633":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"3ff8632f16544d2ca73d10bb8f499c19":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c873c6e6c9094bb9b3e5b48fd1eda1c1","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_646d47008c814368a09528a7fe0fc425","value":692}},"43a107fe61cb40deac8d005c952c6f85":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"446202aaea08462ab6ef2e52b6564b4f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b6921caceaf14927904a58c92578d6f3","placeholder":"​","style":"IPY_MODEL_3ec3efc280844fd5ae0b67093fb61633","value":"100% 692/692 [00:35&lt;00:00, 21.55it/s]"}},"44e0c3f20e0b46028c5d46784d46c363":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"4777be7511fd4d36b029541830bd3b14":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"4a918047ee554d569b953ffb493670ad":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_abcd5adb47444dbba19f924f153c0009","placeholder":"​","style":"IPY_MODEL_ecc152dec5fd444fb4f0151c29f8a201","value":"100% 692/692 [00:35&lt;00:00, 19.67it/s]"}},"4b882a1991604f3e8a8d42a342cf7576":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_1843bf4479b8486f91409bd1f5bcdc2d","placeholder":"​","style":"IPY_MODEL_846679e06490491a88a2ccbc1dcf0a94","value":"100% 692/692 [00:34&lt;00:00, 19.81it/s]"}},"4d36652f8f2a46488995c21fc208ba80":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"511c01ac4c1947428e83cbf482a28e61":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ba203068b21f482187226c9b79d789d3","placeholder":"​","style":"IPY_MODEL_03fbf842146a4ed8bb86f3e093928a7e","value":"100% 692/692 [00:35&lt;00:00, 21.98it/s]"}},"527e72c17d7e4619853e71deaa19459e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_31d88db94ddd4ebd9858ed6ce87996c3","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2a359dbbbdbd4a4a8c7baca11a15a8b9","value":692}},"578fbab606a447f9b09a82c71c56c5b4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5feea5e5918444ceb156be86ff7adf85":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"646d47008c814368a09528a7fe0fc425":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"648a4a5d596b4e24b22a96c623a24243":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_a8127a42d6804a1889e4f509a69056d2","IPY_MODEL_511c01ac4c1947428e83cbf482a28e61"],"layout":"IPY_MODEL_e5baa67ec2dc4641a636cdf891ec9e7b"}},"65f62a43ce19499db967c9a469c31744":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"67e8cfd1cfbb4ed9b1f145ee5b9ffb8b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"68ba4fa4ade7471db70f9d76bd906016":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f2b31974f058484899ecbb913299c90c","IPY_MODEL_6e61ea06261c4de383fcb2cefac56c13"],"layout":"IPY_MODEL_ce51ca891c9c4d56ab4a9246971e62f2"}},"6a4c32ede9664b779a2ab60785176e77":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6dbd62d02079475ab4a1286b495c8e5f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"6e61ea06261c4de383fcb2cefac56c13":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_43a107fe61cb40deac8d005c952c6f85","placeholder":"​","style":"IPY_MODEL_d44f989356d04f95b827cede0999782a","value":"100% 692/692 [00:34&lt;00:00, 21.98it/s]"}},"6e63475167344c1ca8ab41314cba536c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_07bbd6e19e6c42c991bc60572604fdc6","IPY_MODEL_129e125b6f514df6b18710cc12e0417b"],"layout":"IPY_MODEL_334fd22534c14825b9c60875ab1af27e"}},"7038cb89e3984690a56ea4994afc9b36":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_36093696cf7e40e888e78a7d7f066422","placeholder":"​","style":"IPY_MODEL_de4bbf648c8646b5a7b8e3601e74a2da","value":"100% 692/692 [00:34&lt;00:00, 19.83it/s]"}},"75aec803fe1344f396a5b480370640f4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7a8fd8860f9543258d0bb5d6f646e978":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7c69c7e9cd274edcaf2b7fcb27242181":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9e5e60d7ab474511a3764fb619aff684","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_bddd5236892443a590e8aa98d042c435","value":692}},"7cc0002137284d8bbe4fe49cc76a63fc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7e07916f4f8c433c93659bb2e476c78c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7f9f34010d6142b6bc2dbb72be2a3174":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d18e469da7434966bd4ee2327b6a2a2f","IPY_MODEL_7038cb89e3984690a56ea4994afc9b36"],"layout":"IPY_MODEL_e894145d502a434a9f5a1821a6cd214f"}},"81ed769dd49b4376bc3ad84fee325dc6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_527e72c17d7e4619853e71deaa19459e","IPY_MODEL_4b882a1991604f3e8a8d42a342cf7576"],"layout":"IPY_MODEL_3d8f028141a44ad1b6cbf60435e8f3fa"}},"846679e06490491a88a2ccbc1dcf0a94":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"8acdfd5ce03d41dba5e76b8b1c61ea12":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ada1dc784ff943068074219c8f3cfeee","placeholder":"​","style":"IPY_MODEL_be339cd0fe754c8cb3e588768e5683da","value":" 97% 673/692 [00:33&lt;00:00, 19.55it/s]"}},"8cb8bdebd0a2465d80185fa72bec5399":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_066cc89966c34979bcc15995638b3742","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7a8fd8860f9543258d0bb5d6f646e978","value":676}},"8e4ca65d13e745c7bfddecef100fac6c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4d36652f8f2a46488995c21fc208ba80","placeholder":"​","style":"IPY_MODEL_65f62a43ce19499db967c9a469c31744","value":"100% 692/692 [00:34&lt;00:00, 22.08it/s]"}},"8fefacf1303f40e58aa9cd88230176e9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"970c1bd16a8f42f5b840c8723c9f6df8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"9a2613ae2a6c4468ad4f09e1752c627e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9c0137b9bbff43f69cdc064389126e2e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7c69c7e9cd274edcaf2b7fcb27242181","IPY_MODEL_446202aaea08462ab6ef2e52b6564b4f"],"layout":"IPY_MODEL_2db1dfd55ab848f38cb449dd34881bce"}},"9e5e60d7ab474511a3764fb619aff684":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a18db1f223b249ef8f2fc31d4ee18050":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_c48c941228714b809e6a93a419dcaa3d","IPY_MODEL_0bdfce1939314391bfba3a2c273159fa"],"layout":"IPY_MODEL_9a2613ae2a6c4468ad4f09e1752c627e"}},"a8127a42d6804a1889e4f509a69056d2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_d7f5b07a36a34c49aa7c22b6a4acdee1","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_44e0c3f20e0b46028c5d46784d46c363","value":692}},"a8c494574ed441afbf58e17d3daecda7":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a9489c9d655040979137b019893ba323":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"abcd5adb47444dbba19f924f153c0009":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ada1dc784ff943068074219c8f3cfeee":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b149c5c99f904d1b9c230dbac8bbbce9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"b64966cce8a84edaa55dba0610ea1769":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_25ff35274c254f3e9de2ca98c80ab2c0","IPY_MODEL_4a918047ee554d569b953ffb493670ad"],"layout":"IPY_MODEL_7cc0002137284d8bbe4fe49cc76a63fc"}},"b6921caceaf14927904a58c92578d6f3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ba203068b21f482187226c9b79d789d3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bddd5236892443a590e8aa98d042c435":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"be339cd0fe754c8cb3e588768e5683da":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"c48c941228714b809e6a93a419dcaa3d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_29aae73ccda544648ebac551ebfbe20f","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8fefacf1303f40e58aa9cd88230176e9","value":692}},"c5276e5290154b2a8fc16f220a04fa19":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c873c6e6c9094bb9b3e5b48fd1eda1c1":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cc57ed78cc494670afc471bc1f8d8f8a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ce51ca891c9c4d56ab4a9246971e62f2":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d17e6539e7254a9bb59f718fd0d1f1a6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3ff8632f16544d2ca73d10bb8f499c19","IPY_MODEL_8e4ca65d13e745c7bfddecef100fac6c"],"layout":"IPY_MODEL_19eaa5438f4c40128dcd46cb8a5eeaaa"}},"d18e469da7434966bd4ee2327b6a2a2f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_3d5209c342b24b9d92b514a493ccbc8f","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_970c1bd16a8f42f5b840c8723c9f6df8","value":692}},"d44f989356d04f95b827cede0999782a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"d7f5b07a36a34c49aa7c22b6a4acdee1":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dd720cf1d6714ca8a7a1d2ae7a6b689e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"de4bbf648c8646b5a7b8e3601e74a2da":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"e5baa67ec2dc4641a636cdf891ec9e7b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e894145d502a434a9f5a1821a6cd214f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"eb76459022ea431bade839b8a9e046b5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_22b298db91764115ac714897f92a9f52","placeholder":"​","style":"IPY_MODEL_a9489c9d655040979137b019893ba323","value":"100% 692/692 [00:34&lt;00:00, 21.95it/s]"}},"ecc152dec5fd444fb4f0151c29f8a201":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"f2b31974f058484899ecbb913299c90c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_dd720cf1d6714ca8a7a1d2ae7a6b689e","max":692,"min":0,"orientation":"horizontal","style":"IPY_MODEL_67e8cfd1cfbb4ed9b1f145ee5b9ffb8b","value":692}},"fd44ccacff1c41e4838ba904e64be2ef":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_139d9e27b0bb4a05bb6b19b22a9d6c4c","IPY_MODEL_eb76459022ea431bade839b8a9e046b5"],"layout":"IPY_MODEL_6a4c32ede9664b779a2ab60785176e77"}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}