{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install ftfy regex tqdm\n! pip install git+https://github.com/openai/CLIP.git\n!pip install scikit-image\n!pip install pytorch-fid","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:41:46.801897Z","iopub.execute_input":"2022-08-17T16:41:46.802612Z","iopub.status.idle":"2022-08-17T16:42:38.249383Z","shell.execute_reply.started":"2022-08-17T16:41:46.802575Z","shell.execute_reply":"2022-08-17T16:42:38.248093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torchvision import transforms\nimport clip\nimport pytorch_lightning as pl\nimport torchvision\nfrom torch import nn,Tensor\nfrom torch.nn import functional as F\nimport numpy as np\nfrom PIL import Image\nimport torch\nimport torchvision.transforms as T\nimport clip\nimport tqdm\nimport glob\nimport os\nimport tqdm\nimport wandb\nfrom pytorch_fid import fid_score, inception\nimport shutil\nfrom kaggle_secrets import UserSecretsClient\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:38.251728Z","iopub.execute_input":"2022-08-17T16:42:38.253315Z","iopub.status.idle":"2022-08-17T16:42:41.568341Z","shell.execute_reply.started":"2022-08-17T16:42:38.253261Z","shell.execute_reply":"2022-08-17T16:42:41.567080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb_key = UserSecretsClient().get_secret(\"wandb_key\")\nwandb.login(key=wandb_key)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:41.573694Z","iopub.execute_input":"2022-08-17T16:42:41.578023Z","iopub.status.idle":"2022-08-17T16:42:43.797667Z","shell.execute_reply.started":"2022-08-17T16:42:41.577975Z","shell.execute_reply":"2022-08-17T16:42:43.796540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using clip to get loss","metadata":{}},{"cell_type":"code","source":"def get_clip_loss(tensor,clip,preprocess,text_features):\n    loss = 0\n    transform = T.ToPILImage()\n    with torch.no_grad():\n        for img_num in range(tensor.shape[0]):\n            Images = []\n            image_input = transform(tensor[img_num]).convert('RGB')\n            Images.append(preprocess(image_input))\n            image_input = torch.tensor(np.stack(Images)).cuda()\n            image_features = clip.encode_image(image_input).float().cuda()\n            loss += text_features.cpu().numpy() @ image_features.cpu().numpy().T\n    loss = loss / tensor.shape[0]\n    return torch.tensor(1/loss, requires_grad=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:43.801358Z","iopub.execute_input":"2022-08-17T16:42:43.801636Z","iopub.status.idle":"2022-08-17T16:42:43.811281Z","shell.execute_reply.started":"2022-08-17T16:42:43.801611Z","shell.execute_reply":"2022-08-17T16:42:43.810274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\nclass ImageDataset(Dataset):\n    def __init__(self, data_dir, transforms=None):\n        monet_dir = os.path.join(data_dir, 'monet_jpg')\n        photo_dir = os.path.join(data_dir, 'photo_jpg')\n\n        self.files_monet = [os.path.join(monet_dir, name) for name in sorted(os.listdir(monet_dir))]\n        self.files_photo = [os.path.join(photo_dir, name) for name in sorted(os.listdir(photo_dir))]\n\n        self.transforms = transforms\n\n    def __len__(self):\n        # we know that len(files_monet) = 300 < 7038 = len(files_photo)\n        return len(self.files_monet)\n\n    def __getitem__(self, index):\n        # we will use only 300 (=len(files_monet)) photos during training\n        # randomly picking them from the first 300 photos\n        random_index = np.random.randint(0, len(self.files_monet))\n        file_monet = self.files_monet[index]\n        file_photo = self.files_photo[random_index]\n\n        image_monet = Image.open(file_monet)\n        image_photo = Image.open(file_photo)\n\n        if self.transforms is not None:\n            image_monet = self.transforms(image_monet)\n            image_photo = self.transforms(image_photo)\n\n        return image_monet, image_photo\n\ndata_dir = '../input/gan-getting-started'\nbatch_size = 4\ntransforms_ = transforms.Compose([\n    #transforms.Resize((256, 256)), # photos already have the same size\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomVerticalFlip(p=0.3),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\n])\n\ndataloader = DataLoader(\n    ImageDataset(data_dir, transforms=transforms_),\n    batch_size=batch_size,\n    shuffle=True,\n    num_workers=0,\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:43.812431Z","iopub.execute_input":"2022-08-17T16:42:43.812722Z","iopub.status.idle":"2022-08-17T16:42:44.344849Z","shell.execute_reply.started":"2022-08-17T16:42:43.812696Z","shell.execute_reply":"2022-08-17T16:42:44.343640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion_GAN = nn.MSELoss()\ncriterion_cycle = nn.L1Loss()\ncriterion_identity = nn.L1Loss()\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=0,\n                 transpose=False, use_leaky=True, use_dropout=False, normalize=True):\n\n        super(ConvBlock, self).__init__()\n        self.block = []\n\n        if transpose:\n            self.block += [nn.ConvTranspose2d(in_channels, out_channels, kernel_size,\n                                              stride, padding, output_padding=1)]\n        else:\n            self.block += [nn.Conv2d(in_channels, out_channels, kernel_size,\n                                     stride, padding, bias=True)]\n        if normalize:\n            self.block += [nn.InstanceNorm2d(out_channels)]\n\n        if use_dropout:\n            self.block += [nn.Dropout(0.5)]\n\n        if use_leaky:\n            self.block += [nn.LeakyReLU(negative_slope=0.2, inplace=True)]\n        else:\n            self.block += [nn.ReLU(inplace=True)]\n\n        self.block = nn.Sequential(*self.block)\n\n    def forward(self, x):\n        return self.block(x)\n\n\nclass ResidualBlock(nn.Module):\n    def __init__(self, channels):\n        super(ResidualBlock, self).__init__()\n        self.block = nn.Sequential(\n            nn.ReflectionPad2d(1),\n            ConvBlock(in_channels=channels, out_channels=channels,\n                      kernel_size=3, use_leaky=False, use_dropout=True),\n            nn.ReflectionPad2d(1),\n            nn.Conv2d(in_channels=channels, out_channels=channels, kernel_size=3),\n            nn.InstanceNorm2d(channels)\n        )\n\n    def forward(self, x):\n        return x + self.block(x)\n\n\nclass Generator(nn.Module):\n    def __init__(self, in_channels, out_channels, num_residual_blocks=9):\n        super(Generator, self).__init__()\n\n        ''' Encoder '''\n        # Inital layer:  3*256*256 -> 64*256*256\n        self.initial = [\n            nn.ReflectionPad2d(in_channels),\n            ConvBlock(in_channels=in_channels, out_channels=64,\n                      kernel_size=2 * in_channels + 1, use_leaky=False),\n        ]\n        self.initial = nn.Sequential(*self.initial)\n\n        # Downsampling:  64*256*256 -> 128*128*128 -> 256*64*64\n        self.down = [\n            ConvBlock(in_channels=64, out_channels=128, kernel_size=3,\n                      stride=2, padding=1, use_leaky=False),\n            ConvBlock(in_channels=128, out_channels=256, kernel_size=3,\n                      stride=2, padding=1, use_leaky=False),\n        ]\n        self.down = nn.Sequential(*self.down)\n\n        \"\"\" Transformer \"\"\"\n        # ResNet:  256*64*64 -> 256*64*64\n        self.transform = [ResidualBlock(256) for _ in range(num_residual_blocks)]\n        self.transform = nn.Sequential(*self.transform)\n\n        \"\"\" Decoder \"\"\"\n        # Upsampling:  256*64*64 -> 128*128*128 -> 64*256*256\n        self.up = [\n            ConvBlock(in_channels=256, out_channels=128, kernel_size=3, stride=2,\n                      padding=1, transpose=True, use_leaky=False),\n            ConvBlock(in_channels=128, out_channels=64, kernel_size=3, stride=2,\n                      padding=1, transpose=True, use_leaky=False),\n        ]\n        self.up = nn.Sequential(*self.up)\n\n        # Out layer:  64*256*256 -> 3*256*256\n        self.out = nn.Sequential(\n            nn.ReflectionPad2d(out_channels),\n            nn.Conv2d(in_channels=64, out_channels=out_channels, kernel_size=2 * out_channels + 1),\n            nn.Tanh()\n        )\n\n    def forward(self, x):\n        x = self.down(self.initial(x))\n        x = self.transform(x)\n        x = self.out(self.up(x))\n        return x\n\n\nclass Discriminator(nn.Module):\n    def __init__(self, in_channels):\n        super(Discriminator, self).__init__()\n\n        self.model = nn.Sequential(\n            # 3*256*256 -> 64*128*128\n            ConvBlock(in_channels=in_channels, out_channels=64, kernel_size=4,\n                      stride=2, padding=1, normalize=False),\n\n            # 64*128*128 -> 128*64*64\n            ConvBlock(in_channels=64, out_channels=128, kernel_size=4, stride=2, padding=1),\n\n            # 128*64*64 -> 256*32*32\n            ConvBlock(in_channels=128, out_channels=256, kernel_size=4, stride=2, padding=1),\n\n            # 256*32*32 -> 512*31*31\n            ConvBlock(in_channels=256, out_channels=512, kernel_size=4, stride=1, padding=1),\n\n            # 512*31*31 -> 1*30*30\n            nn.Conv2d(in_channels=512, out_channels=1, kernel_size=4, stride=1, padding=1),\n        )\n\n    def forward(self, x):\n        return self.model(x)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:44.346729Z","iopub.execute_input":"2022-08-17T16:42:44.347470Z","iopub.status.idle":"2022-08-17T16:42:44.369090Z","shell.execute_reply.started":"2022-08-17T16:42:44.347429Z","shell.execute_reply":"2022-08-17T16:42:44.368062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda'\n\ngenerator_monet2photo = Generator(in_channels=3, out_channels=3, num_residual_blocks=9).to(device)\ngenerator_photo2monet = Generator(in_channels=3, out_channels=3, num_residual_blocks=9).to(device)\n\ndiscriminator_monet = Discriminator(in_channels=3).to(device)\ndiscriminator_photo = Discriminator(in_channels=3).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:44.370682Z","iopub.execute_input":"2022-08-17T16:42:44.371231Z","iopub.status.idle":"2022-08-17T16:42:47.555776Z","shell.execute_reply.started":"2022-08-17T16:42:44.371195Z","shell.execute_reply":"2022-08-17T16:42:47.554828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class train_cycle_gan(pl.LightningModule):\n    def __init__(self, generator_monet2photo,generator_photo2monet,discriminator_monet,discriminator_photo,\n                 clip_model = None,preprocess = None,use_clip = False,text = [],biases = []):\n        super().__init__()\n        self.generator_monet2photo = generator_monet2photo\n        self.generator_photo2monet = generator_photo2monet\n        self.discriminator_monet = discriminator_monet\n        self.discriminator_photo = discriminator_photo\n        self.clip, self.preprocess = clip_model,preprocess\n        self.use_clip = use_clip\n        self.text = text\n        if self.clip != None:\n            texts_features = []\n            for i in text:\n                text_tokens = clip.tokenize(i).cuda()\n                with torch.no_grad():\n                    text_features = self.clip_model.encode_text(text_tokens).float()\n                    text_features /= text_features.norm(dim=-1, keepdim=True)\n                    texts_features.append(text_features)\n            self.texts_features = texts_features\n\n\n    def training_step(self, input, count, **kwargs):\n        real_monet,real_photo = input[0],input[1]\n        fake_photo = self.generator_monet2photo(real_monet)\n        fake_monet = self.generator_photo2monet(real_photo)\n        cycle_photo = self.generator_monet2photo(fake_monet)\n        cycle_monet = self.generator_photo2monet(fake_photo)\n        identity_photo = self.generator_monet2photo(real_photo)\n        identity_monet = self.generator_photo2monet(real_monet)\n        discriminator_outputs_photo = self.discriminator_photo(fake_photo)\n        discriminator_outputs_monet = self.discriminator_monet(fake_monet)\n\n        labels_real = torch.ones(discriminator_outputs_monet.size()).to(self.device)\n        labels_fake = torch.zeros(discriminator_outputs_monet.size()).to(self.device)\n\n        loss_GAN_monet2photo = criterion_GAN(discriminator_outputs_photo, labels_real)\n        loss_GAN_photo2monet = criterion_GAN(discriminator_outputs_monet, labels_real)\n        if  self.clip != None:\n            for i,t in enumerate(self.texts_features):\n                r = get_clip_loss(output,self.clip_model,self.preprocess,t).cuda() * self.biases[i]\n                if i == 0:\n                    clip_loss = r\n                else:\n                    clip_loss +=r\n                self.log('clip_loss:' + self.text[i],r)\n        loss_GAN = (loss_GAN_monet2photo + loss_GAN_photo2monet) / 2\n        if self.use_clip:\n            self.log('clip_loss',clip_loss)\n            loss_GAN+= 2*clip_loss\n        self.log('loss_gan',loss_GAN)\n\n        \n        loss_cycle_photo = criterion_cycle(cycle_photo, real_photo)\n        loss_cycle_monet = criterion_cycle(cycle_monet, real_monet)\n        loss_cycle = (loss_cycle_photo + loss_cycle_monet) / 2\n        self.log('loss_cycle',loss_cycle)\n\n\n        loss_identity_photo = criterion_identity(identity_photo, real_photo)\n        loss_identity_monet = criterion_identity(identity_monet, real_monet)\n        loss_identity = (loss_identity_photo + loss_identity_monet) / 2\n        self.log('loss_identity',loss_identity)\n        loss_generators_total = loss_GAN + 10 * loss_cycle + 5 * loss_identity\n        self.log('loss_generators_total',loss_generators_total)\n\n        #loss_generators_total.backward()\n\n        noisy_labels_real = (torch.rand(discriminator_outputs_monet.size()) < 1).float().to(device)\n\n        # forward-pass + losses\n        loss_real_photo = criterion_GAN(self.discriminator_photo(real_photo), noisy_labels_real)\n        loss_fake_photo = criterion_GAN(self.discriminator_photo(fake_photo.detach()), labels_fake)\n        loss_photo = (loss_real_photo + loss_fake_photo) / 2\n\n        loss_real_monet = criterion_GAN(discriminator_monet(real_monet), noisy_labels_real)\n        loss_fake_monet = criterion_GAN(discriminator_monet(fake_monet.detach()), labels_fake)\n        loss_monet = (loss_real_monet + loss_fake_monet) / 2\n\n        loss_discriminators_total = loss_monet + loss_photo\n\n        # backward-pass\n        #loss_discriminators_total.backward()\n        self.log('loss_discriminators_total',loss_discriminators_total)\n        \n        \n        if count % 50 == 0:\n            self.logger.log_image(key=\"samples\", images=[fake_monet[0:3],real_photo[0:3]])\n        \n        total_loss = loss_discriminators_total + loss_generators_total\n        self.log('total_loss',total_loss)\n        return loss_discriminators_total + loss_generators_total\n\n\n    def configure_optimizers(self):\n        lr = 2e-4\n        b1 = 0.5\n        b2 = 0.999\n        optim_generators = torch.optim.Adam(\n            list(self.generator_monet2photo.parameters()) + list(self.generator_photo2monet.parameters()),\n            lr=lr, betas=(b1, b2)\n        )\n\n        optim_discriminators = torch.optim.Adam(\n            list(self.discriminator_monet.parameters()) + list(self.discriminator_photo.parameters()),\n            lr=lr, betas=(b1, b2)\n        )\n        return  optim_generators\n\n#train_cycle = train_cycle_gan(generator_monet2photo,generator_photo2monet,discriminator_monet,discriminator_photo,clip_model.cuda(),preprocess)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.616767Z","iopub.execute_input":"2022-08-17T03:14:31.617113Z","iopub.status.idle":"2022-08-17T03:14:31.636263Z","shell.execute_reply.started":"2022-08-17T03:14:31.617078Z","shell.execute_reply":"2022-08-17T03:14:31.635291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = {\"learning_rate\": 1e-2,\n       \"batch_size\": 4,\n       \"epochs\": 10}","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.637862Z","iopub.execute_input":"2022-08-17T03:14:31.638550Z","iopub.status.idle":"2022-08-17T03:14:31.652012Z","shell.execute_reply.started":"2022-08-17T03:14:31.638511Z","shell.execute_reply":"2022-08-17T03:14:31.650821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#with wandb.init(project='monet paintnig', job_type=\"train\", config=opt) as run:\n#    logger = pl.loggers.WandbLogger(experiment=run)\n#    trainer = pl.Trainer(max_epochs=10,logger=logger,accelerator='gpu')\n#    trainer.fit(train_cycle, dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.655481Z","iopub.execute_input":"2022-08-17T03:14:31.656438Z","iopub.status.idle":"2022-08-17T03:14:31.663484Z","shell.execute_reply.started":"2022-08-17T03:14:31.656402Z","shell.execute_reply":"2022-08-17T03:14:31.662341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get_images('images/',train_cycle.generator_photo2monet,4000)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.664947Z","iopub.execute_input":"2022-08-17T03:14:31.665996Z","iopub.status.idle":"2022-08-17T03:14:31.671967Z","shell.execute_reply.started":"2022-08-17T03:14:31.665918Z","shell.execute_reply":"2022-08-17T03:14:31.671052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#len(glob.glob('images/*'))","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.673796Z","iopub.execute_input":"2022-08-17T03:14:31.675632Z","iopub.status.idle":"2022-08-17T03:14:31.682147Z","shell.execute_reply.started":"2022-08-17T03:14:31.675456Z","shell.execute_reply":"2022-08-17T03:14:31.681243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fid_value = fid_score.calculate_fid_given_paths(['images','../input/gan-getting-started/monet_jpg'],\n #                                               batch_size=16,\n  #                                              device=device,\n   #                                             dims=64,\n    #                                            num_workers=0)\n#fid_value","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.683703Z","iopub.execute_input":"2022-08-17T03:14:31.684346Z","iopub.status.idle":"2022-08-17T03:14:31.691812Z","shell.execute_reply.started":"2022-08-17T03:14:31.684308Z","shell.execute_reply":"2022-08-17T03:14:31.690869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('images')","metadata":{"execution":{"iopub.status.busy":"2022-08-17T17:36:59.095800Z","iopub.execute_input":"2022-08-17T17:36:59.096801Z","iopub.status.idle":"2022-08-17T17:36:59.103287Z","shell.execute_reply.started":"2022-08-17T17:36:59.096754Z","shell.execute_reply":"2022-08-17T17:36:59.101974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unnorm(img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]):\n    for t, m, s in zip(img, mean, std):\n        t.mul_(s).add_(s)\n        \n    return img\n\ndef get_images(path,model,num):\n    q=0\n    generate_transforms = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ])\n    images = glob.glob('../input/completion-data/1/*')[0:num]\n    with torch.no_grad():\n        for i in images:\n            input_img = generate_transforms(Image.open(i)).cuda()\n            output = model.cuda()(input_img)\n            output = unnorm(output)\n            img = transforms.ToPILImage()(output).convert(\"RGB\")\n            img.save(path + str(q) +'.jpg')\n            q+=1\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:43:00.809444Z","iopub.execute_input":"2022-08-17T16:43:00.809755Z","iopub.status.idle":"2022-08-17T16:43:00.818104Z","shell.execute_reply.started":"2022-08-17T16:43:00.809727Z","shell.execute_reply":"2022-08-17T16:43:00.816991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get_images('images/',train_cycle.generator_photo2monet,1000)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.719814Z","iopub.execute_input":"2022-08-17T03:14:31.720623Z","iopub.status.idle":"2022-08-17T03:14:31.728330Z","shell.execute_reply.started":"2022-08-17T03:14:31.720589Z","shell.execute_reply":"2022-08-17T03:14:31.727404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fid_value = fid_score.calculate_fid_given_paths(['images','../input/gan-getting-started/monet_jpg'],\n  #                                              batch_size=16,\n #                                               device=device,\n   #                                             dims=64,\n    #                                            num_workers=0)\n#fid_value","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:31.731512Z","iopub.execute_input":"2022-08-17T03:14:31.732082Z","iopub.status.idle":"2022-08-17T03:14:31.738681Z","shell.execute_reply.started":"2022-08-17T03:14:31.732053Z","shell.execute_reply":"2022-08-17T03:14:31.737685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#folder_path = '../input/completion-data/'\n\n\n#train_image_folder = torchvision.datasets.ImageFolder(root=folder_path,transform=transforms_)\n#train_data_loader = torch.utils.data.DataLoader(train_image_folder,batch_size = 16,shuffle=True,num_workers=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:47.557314Z","iopub.execute_input":"2022-08-17T16:42:47.557674Z","iopub.status.idle":"2022-08-17T16:42:54.198427Z","shell.execute_reply.started":"2022-08-17T16:42:47.557638Z","shell.execute_reply":"2022-08-17T16:42:54.197430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clip_model,preprocess = clip.load('ViT-B/32')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-17T16:42:54.203792Z","iopub.execute_input":"2022-08-17T16:42:54.206007Z","iopub.status.idle":"2022-08-17T16:43:00.804743Z","shell.execute_reply.started":"2022-08-17T16:42:54.205969Z","shell.execute_reply":"2022-08-17T16:43:00.803687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class unsp_train(pl.LightningModule):\n    def __init__(self, generator_photo2monet,clip_model,preprocess,text = [],biases = []):\n        super().__init__()\n        self.model = generator_photo2monet\n        self.clip_model = clip_model\n        self.preprocess = preprocess\n        self.text = text\n        self.biases = biases\n        texts_features = []\n        for i in text:\n            text_tokens = clip.tokenize(i).cuda()\n            with torch.no_grad():\n                text_features = self.clip_model.encode_text(text_tokens).float()\n                text_features /= text_features.norm(dim=-1, keepdim=True)\n                texts_features.append(text_features)\n        self.texts_features = texts_features\n\n    def training_step(self, input, count, **kwargs):\n        images = input[0]\n        output = self.model(images)\n        \n        for i,t in enumerate(self.texts_features):\n            r = get_clip_loss(output,self.clip_model,self.preprocess,t).cuda() * self.biases[i]\n            if i == 0:\n                clip_loss = r\n            else:\n                clip_loss +=r\n            self.log('clip_loss:' + self.text[i],r)\n        l1 = criterion_GAN(images,output)\n        total_loss = clip_loss + l1\n        self.log('mse',l1)\n        self.log('total_loss',total_loss)\n        if count%100 ==0:\n            self.logger.log_image(key=\"samples\", images=[output[0:3],images[0:3]])\n        return total_loss\n\n    def configure_optimizers(self):\n        lr = 1e-4\n        b1 = 0.5\n        b2 = 0.999\n        optim_generators = torch.optim.Adam(\n            list(self.model.parameters()),\n            lr=lr, betas=(b1, b2)\n        )\n\n        return  optim_generators\n\n#texts =['painting','Claude Monet painting.','beautiful nature']\n#unsep_train_model = unsp_train(generator_photo2monet.cuda(),clip_model.cuda(),preprocess,texts,[5,3,1])\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-17T17:56:02.517673Z","iopub.execute_input":"2022-08-17T17:56:02.518129Z","iopub.status.idle":"2022-08-17T17:56:02.662479Z","shell.execute_reply.started":"2022-08-17T17:56:02.518093Z","shell.execute_reply":"2022-08-17T17:56:02.661563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#opt = {\"learning_rate\": 1e-4,\n#       \"batch_size\": 12,\n#       \"epochs\": 9}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"training model","metadata":{}},{"cell_type":"code","source":"#with wandb.init(project='monet paintnig', job_type=\"train\", config=opt) as run:\n#    logger = pl.loggers.WandbLogger(experiment=run, log_model=True, save_dir='wandb_logger_savedir',)\n#    trainer = pl.Trainer(max_epochs=5,accelerator='gpu',logger=logger)\n#    trainer.fit(unsep_train_model, train_data_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T18:40:32.891763Z","iopub.execute_input":"2022-08-17T18:40:32.892151Z","iopub.status.idle":"2022-08-17T19:24:04.119171Z","shell.execute_reply.started":"2022-08-17T18:40:32.892119Z","shell.execute_reply":"2022-08-17T19:24:04.118088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get_images('images/',unsep_train_model.model,800)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T19:35:04.700291Z","iopub.execute_input":"2022-08-17T19:35:04.701346Z","iopub.status.idle":"2022-08-17T19:35:22.223097Z","shell.execute_reply.started":"2022-08-17T19:35:04.701296Z","shell.execute_reply":"2022-08-17T19:35:22.222141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fid_value = fid_score.calculate_fid_given_paths(['images','../input/gan-getting-started/monet_jpg'],\n#                                                batch_size=16,\n#                                               device=device,\n#                                                dims=64,\n#                                                num_workers=0)\n#fid_value","metadata":{"execution":{"iopub.status.busy":"2022-08-17T19:35:25.718862Z","iopub.execute_input":"2022-08-17T19:35:25.719247Z","iopub.status.idle":"2022-08-17T19:35:29.374708Z","shell.execute_reply.started":"2022-08-17T19:35:25.719215Z","shell.execute_reply":"2022-08-17T19:35:29.373727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"get images","metadata":{}},{"cell_type":"code","source":"run = wandb.init()\nartifact = run.use_artifact('amit1541541/monet paintnig/model-3rni20q8:v0', type='model')\nartifact_dir = artifact.download()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:14:49.699832Z","iopub.execute_input":"2022-08-17T03:14:49.700506Z","iopub.status.idle":"2022-08-17T03:15:02.069829Z","shell.execute_reply.started":"2022-08-17T03:14:49.700472Z","shell.execute_reply":"2022-08-17T03:15:02.068814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb_model = unsp_train(generator_photo2monet,clip_model.cuda(),preprocess)\ncheckpoint = torch.load('./artifacts/model-3rni20q8:v0/model.ckpt')","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:15:15.355754Z","iopub.execute_input":"2022-08-17T03:15:15.356128Z","iopub.status.idle":"2022-08-17T03:15:16.726864Z","shell.execute_reply.started":"2022-08-17T03:15:15.356095Z","shell.execute_reply":"2022-08-17T03:15:16.725858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb_model.load_state_dict(checkpoint['state_dict'])","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:15:19.560067Z","iopub.execute_input":"2022-08-17T03:15:19.560639Z","iopub.status.idle":"2022-08-17T03:15:19.646178Z","shell.execute_reply.started":"2022-08-17T03:15:19.560587Z","shell.execute_reply":"2022-08-17T03:15:19.645343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_images('images/',wandb_model.model,7500)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:15:02.300549Z","iopub.status.idle":"2022-08-17T03:15:02.301026Z","shell.execute_reply.started":"2022-08-17T03:15:02.300782Z","shell.execute_reply":"2022-08-17T03:15:02.300805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"create ","metadata":{}},{"cell_type":"code","source":"shutil.make_archive('images', 'zip', 'images')","metadata":{"execution":{"iopub.status.busy":"2022-08-17T03:15:02.303100Z","iopub.status.idle":"2022-08-17T03:15:02.304055Z","shell.execute_reply.started":"2022-08-17T03:15:02.303797Z","shell.execute_reply":"2022-08-17T03:15:02.303825Z"},"trusted":true},"execution_count":null,"outputs":[]}]}