{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install --upgrade torch","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:36:44.752455Z","iopub.execute_input":"2022-05-26T14:36:44.752838Z","iopub.status.idle":"2022-05-26T14:36:57.088188Z","shell.execute_reply.started":"2022-05-26T14:36:44.752752Z","shell.execute_reply":"2022-05-26T14:36:57.087286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:36:57.090373Z","iopub.execute_input":"2022-05-26T14:36:57.09063Z","iopub.status.idle":"2022-05-26T14:36:57.132859Z","shell.execute_reply.started":"2022-05-26T14:36:57.090594Z","shell.execute_reply":"2022-05-26T14:36:57.132243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nimport itertools\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport PIL\nfrom PIL import Image\nimport random\nimport shutil\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_curve\nfrom sklearn import metrics\nimport time\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.nn.init as init\nfrom torch.utils.data import Dataset, random_split, DataLoader\n\nimport torchvision.models as models\nimport torchvision.transforms as transforms\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-26T14:36:57.134011Z","iopub.execute_input":"2022-05-26T14:36:57.134317Z","iopub.status.idle":"2022-05-26T14:36:59.637178Z","shell.execute_reply.started":"2022-05-26T14:36:57.13428Z","shell.execute_reply":"2022-05-26T14:36:59.636512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:36:59.63928Z","iopub.execute_input":"2022-05-26T14:36:59.639692Z","iopub.status.idle":"2022-05-26T14:36:59.675987Z","shell.execute_reply.started":"2022-05-26T14:36:59.639653Z","shell.execute_reply":"2022-05-26T14:36:59.675392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:36:59.677251Z","iopub.execute_input":"2022-05-26T14:36:59.677642Z","iopub.status.idle":"2022-05-26T14:36:59.716562Z","shell.execute_reply.started":"2022-05-26T14:36:59.677604Z","shell.execute_reply":"2022-05-26T14:36:59.715887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self, monet_dir, photo_dir, size=(256, 256), normalize=True):\n        super().__init__()\n        self.monet_dir = monet_dir\n        self.photo_dir = photo_dir\n        self.monet_idx = dict()\n        self.photo_idx = dict()\n        self.transform = transforms.Compose([transforms.Resize(size),transforms.ToTensor(),\n                                             transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n        for i, monet in enumerate(os.listdir(self.monet_dir)):\n            self.monet_idx[i] = monet\n        for i, photo in enumerate(os.listdir(self.photo_dir)):\n            self.photo_idx[i] = photo\n            \n    def __len__(self):\n        return min(len(self.monet_idx.keys()), len(self.photo_idx.keys()))\n    \n    def __getitem__(self, idx):\n        rand_idx = int(np.random.uniform(0, len(self.monet_idx.keys())))\n        photo_path = os.path.join(self.photo_dir, self.photo_idx[rand_idx])\n        monet_path = os.path.join(self.monet_dir, self.monet_idx[idx])\n        photo_img = Image.open(photo_path)\n        photo_img = self.transform(photo_img)\n        monet_img = Image.open(monet_path)\n        monet_img = self.transform(monet_img)\n        return photo_img, monet_img","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:36:59.717729Z","iopub.execute_input":"2022-05-26T14:36:59.718087Z","iopub.status.idle":"2022-05-26T14:36:59.759951Z","shell.execute_reply.started":"2022-05-26T14:36:59.718035Z","shell.execute_reply":"2022-05-26T14:36:59.759273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_ds = ImageDataset('../input/gan-getting-started/monet_jpg/', '../input/gan-getting-started/photo_jpg/')\n\nimg_dl = DataLoader(img_ds, batch_size=4, pin_memory=True) # pin_memory enables fast data transfer to CUDA-enabled GPUs","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:36:59.761452Z","iopub.execute_input":"2022-05-26T14:36:59.761632Z","iopub.status.idle":"2022-05-26T14:37:00.26281Z","shell.execute_reply.started":"2022-05-26T14:36:59.761605Z","shell.execute_reply":"2022-05-26T14:37:00.261454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"photo_img, monet_img = next(iter(img_dl))\n\ntrans = transforms.Compose([transforms.Normalize(mean=[-1], std=[2])])\n\nphoto_img = trans(photo_img)\nmonet_img = trans(monet_img)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:00.265594Z","iopub.execute_input":"2022-05-26T14:37:00.265878Z","iopub.status.idle":"2022-05-26T14:37:03.225688Z","shell.execute_reply.started":"2022-05-26T14:37:00.265841Z","shell.execute_reply":"2022-05-26T14:37:03.22501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2)\n\naxes[0].axis('off')\naxes[0].imshow(photo_img[0].numpy().transpose(1, 2, 0))\naxes[0].set_title(\"Photo\")\n\naxes[1].axis('off')\naxes[1].imshow(monet_img[0].numpy().transpose(1, 2, 0))\naxes[1].set_title(\"Photo\")\n\nfig.set_figwidth(12)   \nfig.set_figheight(6) \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.226846Z","iopub.execute_input":"2022-05-26T14:37:03.227185Z","iopub.status.idle":"2022-05-26T14:37:03.564521Z","shell.execute_reply.started":"2022-05-26T14:37:03.227146Z","shell.execute_reply":"2022-05-26T14:37:03.562138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_checkpoint(state, save_path):\n    torch.save(state, save_path)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.567292Z","iopub.execute_input":"2022-05-26T14:37:03.568115Z","iopub.status.idle":"2022-05-26T14:37:03.606607Z","shell.execute_reply.started":"2022-05-26T14:37:03.568078Z","shell.execute_reply":"2022-05-26T14:37:03.605902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Upsample(in_ch, out_ch, dropout_ratio=0.5):\n    return nn.Sequential(\n            nn.ConvTranspose2d(in_ch, out_ch, 3, stride=2, padding=1, output_padding=1),\n            nn.InstanceNorm2d(out_ch),\n            nn.Dropout(dropout_ratio),\n            nn.SiLU()\n        )\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.608167Z","iopub.execute_input":"2022-05-26T14:37:03.608509Z","iopub.status.idle":"2022-05-26T14:37:03.647855Z","shell.execute_reply.started":"2022-05-26T14:37:03.608427Z","shell.execute_reply":"2022-05-26T14:37:03.646974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Convlayer(in_ch, out_ch, kernel_size=3, stride=2, use_pad=True):\n    if use_pad:\n        conv = nn.Conv2d(in_ch, out_ch, kernel_size, stride, 1, bias=True)\n    else:\n        conv = nn.Conv2d(in_ch, out_ch, kernel_size, stride, 0, bias=True)\n\n    actv = nn.LeakyReLU(negative_slope=0.2, inplace=True)\n\n\n    norm = nn.InstanceNorm2d(out_ch) #\n\n    return nn.Sequential(conv, norm, actv)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.648907Z","iopub.execute_input":"2022-05-26T14:37:03.649168Z","iopub.status.idle":"2022-05-26T14:37:03.68632Z","shell.execute_reply.started":"2022-05-26T14:37:03.649134Z","shell.execute_reply":"2022-05-26T14:37:03.685586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Resblock(nn.Module):\n    def __init__(self, in_features, use_dropout=True, dropout_ratio=0.5):\n        super().__init__()\n        layers = list()\n        layers.append(nn.ReflectionPad2d(1))    # making 1-layer padding near input tensor\n        layers.append(Convlayer(in_features, in_features, 3, 1, use_pad=False))\n        layers.append(nn.Dropout(dropout_ratio))\n        layers.append(nn.ReflectionPad2d(1))\n        layers.append(nn.Conv2d(in_features, in_features, 3, 1, padding=0, bias=True))\n        layers.append(nn.InstanceNorm2d(in_features))\n        layers.append(nn.SiLU())\n        self.res = nn.Sequential(*layers)\n\n    def forward(self, x):\n        return x + self.res(x)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.687365Z","iopub.execute_input":"2022-05-26T14:37:03.687666Z","iopub.status.idle":"2022-05-26T14:37:03.725508Z","shell.execute_reply.started":"2022-05-26T14:37:03.68763Z","shell.execute_reply":"2022-05-26T14:37:03.724832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Generator(nn.Module):\n    def __init__(self, in_ch, out_ch, num_res_blocks=6):\n        super().__init__()\n        model = list()\n        model.append(nn.ReflectionPad2d(3))    # making 3-layer padding near input tensor\n        model.append(Convlayer(in_ch, 64, 7, 1, use_pad=False))\n        model.append(Convlayer(64, 128, 3, 2))\n        model.append(Convlayer(128, 256, 3, 2))\n        for _ in range(num_res_blocks):\n            model.append(Resblock(256))\n        model.append(Upsample(256, 128))\n        model.append(Upsample(128, 64))\n        model.append(nn.ReflectionPad2d(3))\n        model.append(nn.Conv2d(64, out_ch, kernel_size=7, padding=0)) # taking away old padding\n        model.append(nn.Tanh()) # activation function\n\n        self.gen = nn.Sequential(*model)\n\n    def forward(self, x):\n        return self.gen(x) # returning our ready generator model","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.727866Z","iopub.execute_input":"2022-05-26T14:37:03.728091Z","iopub.status.idle":"2022-05-26T14:37:03.767145Z","shell.execute_reply.started":"2022-05-26T14:37:03.728048Z","shell.execute_reply":"2022-05-26T14:37:03.766372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Discriminator(nn.Module):\n    def __init__(self, in_ch, num_layers=4):\n        super().__init__()\n        model = list()\n        model.append(nn.Conv2d(in_ch, 64, 4, stride=2, padding=1))\n        model.append(nn.LeakyReLU(negative_slope=0.2, inplace=True))\n        for i in range(1, num_layers):\n            in_chs = 64 * 2**(i-1)\n            out_chs = in_chs * 2\n            if i == num_layers-1:\n                model.append(Convlayer(in_chs, out_chs, 4, 1)) # for the penultimate case stride will be 1\n            else:\n                model.append(Convlayer(in_chs, out_chs, 4, 2)) # stride 2\n        model.append(nn.Conv2d(512, 1, kernel_size=4, stride=1, padding=1)) # and the same situation for the last case\n        self.disc = nn.Sequential(*model)\n\n    def forward(self, x):\n        return self.disc(x) # returning our ready dicriminator model\n","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.7687Z","iopub.execute_input":"2022-05-26T14:37:03.768915Z","iopub.status.idle":"2022-05-26T14:37:03.808014Z","shell.execute_reply.started":"2022-05-26T14:37:03.768892Z","shell.execute_reply":"2022-05-26T14:37:03.807337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_weights(net, gain=0.02):\n    def init_func(m):\n        \"\"\"\n        Checking if our classes Generator and Discriminatore have methods weight or bias, in this case we fill\n        the input weight tensor from weight.data or bias.data with values drawn from the normal distribution \n        N(mean, std**2) and the input bias tensor we fill with 0\n        \"\"\"\n        classname = m.__class__.__name__ # here we're getting our class name (Generator or Discriminator)\n        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1): \n            init.normal_(m.weight.data, 0.0, gain) # initalizing weight tensor\n            if hasattr(m, 'bias') and m.bias is not None:\n                init.zeros_(m.bias.data) # filling bias tensor with zeroes\n        elif classname.find('BatchNorm2d') != -1:\n            init.normal_(m.weight.data, 1.0, gain)\n            init.zeros_(m.bias.data)\n    net.apply(init_func) # applying function to every submodule of our hand-made net","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.809136Z","iopub.execute_input":"2022-05-26T14:37:03.809465Z","iopub.status.idle":"2022-05-26T14:37:03.847332Z","shell.execute_reply.started":"2022-05-26T14:37:03.809397Z","shell.execute_reply":"2022-05-26T14:37:03.846632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_req_grad(models, requires_grad=True):\n    \"\"\"\n    We need gradients to be computed for Discriminator and not to be computed for Generator\n    \"\"\"\n    for model in models:\n        for param in model.parameters():\n            param.requires_grad = requires_grad","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.848662Z","iopub.execute_input":"2022-05-26T14:37:03.84921Z","iopub.status.idle":"2022-05-26T14:37:03.884754Z","shell.execute_reply.started":"2022-05-26T14:37:03.849177Z","shell.execute_reply":"2022-05-26T14:37:03.883973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class sample_fake(object):\n    def __init__(self, max_imgs=50):\n        self.max_imgs = max_imgs\n        self.cur_img = 0\n        self.imgs = list()\n\n    def __call__(self, imgs): # useful method which helps us not to write the name of method just exmp(args)\n        ret = list()\n        for img in imgs:\n            if self.cur_img < self.max_imgs:\n                self.imgs.append(img)\n                ret.append(img)\n                self.cur_img += 1\n            else:\n                if np.random.ranf() > 0.5:\n                    idx = np.random.randint(0, self.max_imgs)\n                    ret.append(self.imgs[idx])\n                    self.imgs[idx] = img\n                else:\n                    ret.append(img)\n        return ret","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.886597Z","iopub.execute_input":"2022-05-26T14:37:03.886871Z","iopub.status.idle":"2022-05-26T14:37:03.924055Z","shell.execute_reply.started":"2022-05-26T14:37:03.886829Z","shell.execute_reply":"2022-05-26T14:37:03.923444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AvgStats(object):\n    def __init__(self):\n        self.reset()\n        \n    def reset(self):\n        self.losses =[]\n        self.its = []\n        \n    def append(self, loss, it):\n        self.losses.append(loss)\n        self.its.append(it)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.925326Z","iopub.execute_input":"2022-05-26T14:37:03.92572Z","iopub.status.idle":"2022-05-26T14:37:03.962166Z","shell.execute_reply.started":"2022-05-26T14:37:03.925686Z","shell.execute_reply":"2022-05-26T14:37:03.961545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\nclass CycleGAN(object):\n    def __init__(self, in_ch, out_ch, epochs, device, start_lr=2e-4, lmbda=10, idt_coef=0.5, decay_epoch=0):\n        self.epochs = epochs\n        self.decay_epoch = decay_epoch if decay_epoch > 0 else int(self.epochs/2)\n        self.lmbda = lmbda\n        self.idt_coef = idt_coef\n        self.device = device\n        self.gen_mtp = Generator(in_ch, out_ch)\n        self.gen_ptm = Generator(in_ch, out_ch)\n        self.desc_m = Discriminator(in_ch)\n        self.desc_p = Discriminator(in_ch)\n        self.init_models()\n        self.mse_loss = nn.MSELoss()\n        self.l1_loss = nn.L1Loss()\n        \n        self.sgd_gen = torch.optim.SGD(\n            itertools.chain(self.gen_mtp.parameters(), self.gen_ptm.parameters()),\n            lr=start_lr,\n            momentum = 0.9\n        )\n        self.sgd_desc = torch.optim.SGD(\n            itertools.chain(self.desc_m.parameters(), self.desc_p.parameters()), \n            lr = start_lr, \n            momentum = 0.9\n        )\n        \n        self.adam_gen = torch.optim.RAdam(\n            itertools.chain(self.gen_mtp.parameters(), self.gen_ptm.parameters()), \n            lr = start_lr, \n            betas=(0.5, 0.999)\n        )\n        self.adam_desc = torch.optim.RAdam(\n            itertools.chain(self.desc_m.parameters(), self.desc_p.parameters()), \n            lr = start_lr, \n            betas=(0.5, 0.999)\n        )\n        \n        self.sample_monet = sample_fake()\n        self.sample_photo = sample_fake()\n        \n        self.gen_sgd_lr_sched = torch.optim.lr_scheduler.LambdaLR(self.sgd_gen, \n                                                              lr_lambda=lambda epoch: 1 - max(0, epoch-decay_epoch) / (epochs-decay_epoch))\n        self.desc_sgd_lr_sched = torch.optim.lr_scheduler.LambdaLR(self.sgd_desc, \n                                                               lr_lambda=lambda epoch: 1 - max(0, epoch-decay_epoch) / (epochs-decay_epoch))\n        \n        self.gen_lr_sched = torch.optim.lr_scheduler.LambdaLR(self.adam_gen, \n                                                              lr_lambda=lambda epoch: 1 - max(0, epoch-decay_epoch) / (epochs-decay_epoch))\n        self.desc_lr_sched = torch.optim.lr_scheduler.LambdaLR(self.adam_desc, \n                                                               lr_lambda=lambda epoch: 1 - max(0, epoch-decay_epoch) / (epochs-decay_epoch))\n        self.gen_stats = AvgStats()\n        self.desc_stats = AvgStats()\n        \n    def init_models(self):\n        init_weights(self.gen_mtp)\n        init_weights(self.gen_ptm)\n        init_weights(self.desc_m)\n        init_weights(self.desc_p)\n        self.gen_mtp = self.gen_mtp.to(self.device)\n        self.gen_ptm = self.gen_ptm.to(self.device)\n        self.desc_m = self.desc_m.to(self.device)\n        self.desc_p = self.desc_p.to(self.device)\n        \n    def train(self, photo_dl, flag=True):\n        for epoch in range(self.epochs):\n            start_time = time.time()\n            avg_gen_loss = 0.0\n            avg_desc_loss = 0.0\n            t = tqdm(photo_dl, leave=False, total=photo_dl.__len__())\n            \n            for i, (photo_real, monet_real) in enumerate(t):\n                photo_img, monet_img = photo_real.to(device), monet_real.to(device)\n                update_req_grad([self.desc_m, self.desc_p], False)\n                \n                if flag:\n                    self.adam_gen.zero_grad()\n                else:\n                    self.sgd_gen.zero_grad()\n\n                # Forward pass through generator\n                fake_photo = self.gen_mtp(monet_img)\n                fake_monet = self.gen_ptm(photo_img)\n\n                cycl_monet = self.gen_ptm(fake_photo)\n                cycl_photo = self.gen_mtp(fake_monet)\n\n                id_monet = self.gen_ptm(monet_img)\n                id_photo = self.gen_mtp(photo_img)\n\n                # generator losses - identity, Adversarial, cycle consistency\n                idt_loss_monet = self.l1_loss(id_monet, monet_img) * self.lmbda * self.idt_coef\n                idt_loss_photo = self.l1_loss(id_photo, photo_img) * self.lmbda * self.idt_coef\n\n                cycle_loss_monet = self.l1_loss(cycl_monet, monet_img) * self.lmbda\n                cycle_loss_photo = self.l1_loss(cycl_photo, photo_img) * self.lmbda\n\n                monet_desc = self.desc_m(fake_monet)\n                photo_desc = self.desc_p(fake_photo)\n\n                real = torch.ones(monet_desc.size()).to(self.device)\n\n                adv_loss_monet = self.mse_loss(monet_desc, real)\n                adv_loss_photo = self.mse_loss(photo_desc, real)\n\n                # total generator loss\n                total_gen_loss = cycle_loss_monet + adv_loss_monet\\\n                              + cycle_loss_photo + adv_loss_photo\\\n                              + idt_loss_monet + idt_loss_photo\n                \n                avg_gen_loss += total_gen_loss.item()\n\n                # backward pass\n                total_gen_loss.backward()\n                if flag:\n                    self.adam_gen.step()\n                else:\n                    self.sgd_gen.step()\n\n                # Forward pass through Descriminator\n                update_req_grad([self.desc_m, self.desc_p], True)\n                if flag:\n                    self.adam_desc.zero_grad()\n                else:\n                    self.sgd_desc.zero_grad()\n\n                fake_monet = self.sample_monet([fake_monet.cpu().data.numpy()])[0]\n                fake_photo = self.sample_photo([fake_photo.cpu().data.numpy()])[0]\n                fake_monet = torch.tensor(fake_monet).to(self.device)\n                fake_photo = torch.tensor(fake_photo).to(self.device)\n\n                monet_desc_real = self.desc_m(monet_img)\n                monet_desc_fake = self.desc_m(fake_monet)\n                photo_desc_real = self.desc_p(photo_img)\n                photo_desc_fake = self.desc_p(fake_photo)\n\n                real = torch.ones(monet_desc_real.size()).to(self.device)\n                fake = torch.zeros(monet_desc_fake.size()).to(self.device)\n\n                # Descriminator losses\n                # --------------------\n                monet_desc_real_loss = self.mse_loss(monet_desc_real, real)\n                monet_desc_fake_loss = self.mse_loss(monet_desc_fake, fake)\n                photo_desc_real_loss = self.mse_loss(photo_desc_real, real)\n                photo_desc_fake_loss = self.mse_loss(photo_desc_fake, fake)\n\n                monet_desc_loss = (monet_desc_real_loss + monet_desc_fake_loss) / 2\n                photo_desc_loss = (photo_desc_real_loss + photo_desc_fake_loss) / 2\n                total_desc_loss = monet_desc_loss + photo_desc_loss\n                avg_desc_loss += total_desc_loss.item()\n\n                # Backward\n                monet_desc_loss.backward()\n                photo_desc_loss.backward()\n                if flag:\n                    self.adam_desc.step()\n                else:\n                    self.sgd_desc.step()\n                \n                t.set_postfix(gen_loss=total_gen_loss.item(), desc_loss=total_desc_loss.item())\n\n            save_dict = {\n                'epoch': epoch+1,\n                'gen_mtp': gan.gen_mtp.state_dict(),\n                'gen_ptm': gan.gen_ptm.state_dict(),\n                'desc_m': gan.desc_m.state_dict(),\n                'desc_p': gan.desc_p.state_dict(),\n                'optimizer_gen': gan.adam_gen.state_dict(),\n                'optimizer_desc': gan.adam_desc.state_dict()\n            }\n            save_checkpoint(save_dict, 'current.ckpt')\n            \n            avg_gen_loss /= photo_dl.__len__()\n            avg_desc_loss /= photo_dl.__len__()\n            time_req = time.time() - start_time\n            \n            self.gen_stats.append(avg_gen_loss, time_req)\n            self.desc_stats.append(avg_desc_loss, time_req)\n            \n            print(\"Epoch: (%d) | Generator Loss:%f | Discriminator Loss:%f\" % \n                                                (epoch+1, avg_gen_loss, avg_desc_loss))\n            if flag:\n                self.gen_lr_sched.step()\n                self.desc_lr_sched.step()\n            else:\n                self.gen_sgd_lr_sched.step()\n                self.desc_sgd_lr_sched.step()\n            \n            gc.collect()\n            torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:03.963486Z","iopub.execute_input":"2022-05-26T14:37:03.963829Z","iopub.status.idle":"2022-05-26T14:37:04.030872Z","shell.execute_reply.started":"2022-05-26T14:37:03.963793Z","shell.execute_reply":"2022-05-26T14:37:04.030096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:04.032266Z","iopub.execute_input":"2022-05-26T14:37:04.032873Z","iopub.status.idle":"2022-05-26T14:37:04.068393Z","shell.execute_reply.started":"2022-05-26T14:37:04.032831Z","shell.execute_reply":"2022-05-26T14:37:04.067644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gan = CycleGAN(3, 3, 200, device)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:04.069499Z","iopub.execute_input":"2022-05-26T14:37:04.069788Z","iopub.status.idle":"2022-05-26T14:37:04.459125Z","shell.execute_reply.started":"2022-05-26T14:37:04.069754Z","shell.execute_reply":"2022-05-26T14:37:04.458395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save before train\nsave_dict = {\n    'epoch': 0,\n    'gen_mtp': gan.gen_mtp.state_dict(),\n    'gen_ptm': gan.gen_ptm.state_dict(),\n    'desc_m': gan.desc_m.state_dict(),\n    'desc_p': gan.desc_p.state_dict(),\n    'optimizer_gen': gan.adam_gen.state_dict(),\n    'optimizer_desc': gan.adam_desc.state_dict()\n}","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:04.46041Z","iopub.execute_input":"2022-05-26T14:37:04.460637Z","iopub.status.idle":"2022-05-26T14:37:04.50133Z","shell.execute_reply.started":"2022-05-26T14:37:04.460605Z","shell.execute_reply":"2022-05-26T14:37:04.50068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_checkpoint(save_dict, 'init.ckpt')","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:04.50281Z","iopub.execute_input":"2022-05-26T14:37:04.503152Z","iopub.status.idle":"2022-05-26T14:37:04.670447Z","shell.execute_reply.started":"2022-05-26T14:37:04.503058Z","shell.execute_reply":"2022-05-26T14:37:04.669683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gan.train(img_dl)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:37:04.711491Z","iopub.execute_input":"2022-05-26T14:37:04.711748Z","iopub.status.idle":"2022-05-26T14:38:09.474647Z","shell.execute_reply.started":"2022-05-26T14:37:04.711713Z","shell.execute_reply":"2022-05-26T14:38:09.472868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gan.epochs = 100","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.47566Z","iopub.status.idle":"2022-05-26T14:38:09.47655Z","shell.execute_reply.started":"2022-05-26T14:38:09.476288Z","shell.execute_reply":"2022-05-26T14:38:09.476315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gan.train(img_dl, False)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.477608Z","iopub.status.idle":"2022-05-26T14:38:09.478651Z","shell.execute_reply.started":"2022-05-26T14:38:09.478374Z","shell.execute_reply":"2022-05-26T14:38:09.4784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.xlabel(\"Epochs\")\nplt.ylabel(\"Losses\")\nplt.plot(gan.gen_stats.losses, 'r', label='Generator Loss')\nplt.plot(gan.desc_stats.losses, 'b', label='Descriminator Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.479951Z","iopub.status.idle":"2022-05-26T14:38:09.480601Z","shell.execute_reply.started":"2022-05-26T14:38:09.480356Z","shell.execute_reply":"2022-05-26T14:38:09.480381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(5, 2)\n\nfor i in range(5):\n    photo_img, _ = next(iter(img_dl))\n    pred_monet = gan.gen_ptm(photo_img.to(device)).cpu().detach()\n    photo_img = trans(photo_img)\n    pred_monet = trans(pred_monet)\n    \n    axes[i, 0].axis('off')\n    axes[i, 0].imshow(photo_img[0].numpy().transpose(1, 2, 0))\n    axes[i, 0].set_title(\"Input Photo\")\n\n    axes[i, 1].axis('off')\n    axes[i, 1].imshow(pred_monet[0].numpy().transpose(1, 2, 0))\n    axes[i, 1].set_title(\"Monet-esque Photo\")\n\nfig.set_figwidth(12)   \nfig.set_figheight(28) \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.481775Z","iopub.status.idle":"2022-05-26T14:38:09.482426Z","shell.execute_reply.started":"2022-05-26T14:38:09.482159Z","shell.execute_reply":"2022-05-26T14:38:09.482184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PhotoDataset(Dataset):\n    def __init__(self, photo_dir, size=(256, 256), normalize=True):\n        super().__init__()\n        self.photo_dir = photo_dir\n        self.photo_idx = dict()\n        if normalize:\n            self.transform = transforms.Compose([\n                transforms.Resize(size),\n                transforms.ToTensor(),\n                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                                \n            ])\n        else:\n            self.transform = transforms.Compose([\n                transforms.Resize(size),\n                transforms.ToTensor()                               \n            ])\n        for i, fl in enumerate(os.listdir(self.photo_dir)):\n            self.photo_idx[i] = fl\n\n    def __getitem__(self, idx):\n        photo_path = os.path.join(self.photo_dir, self.photo_idx[idx])\n        photo_img = Image.open(photo_path)\n        photo_img = self.transform(photo_img)\n        return photo_img\n    \n    def __len__(self):\n        return len(self.photo_idx.keys())","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.483672Z","iopub.status.idle":"2022-05-26T14:38:09.484331Z","shell.execute_reply.started":"2022-05-26T14:38:09.484062Z","shell.execute_reply":"2022-05-26T14:38:09.484102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ph_ds = PhotoDataset('../input/gan-getting-started/photo_jpg/')","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.485599Z","iopub.status.idle":"2022-05-26T14:38:09.486244Z","shell.execute_reply.started":"2022-05-26T14:38:09.485986Z","shell.execute_reply":"2022-05-26T14:38:09.486011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ph_dl = DataLoader(ph_ds, batch_size=1, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.487468Z","iopub.status.idle":"2022-05-26T14:38:09.488114Z","shell.execute_reply.started":"2022-05-26T14:38:09.487848Z","shell.execute_reply":"2022-05-26T14:38:09.487872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ../images","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.489355Z","iopub.status.idle":"2022-05-26T14:38:09.489985Z","shell.execute_reply.started":"2022-05-26T14:38:09.489748Z","shell.execute_reply":"2022-05-26T14:38:09.489772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pil_trans = transforms.ToPILImage()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.491211Z","iopub.status.idle":"2022-05-26T14:38:09.491848Z","shell.execute_reply.started":"2022-05-26T14:38:09.491607Z","shell.execute_reply":"2022-05-26T14:38:09.491633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unnorm(img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]):\n    for t, m, s in zip(img, mean, std):\n        t.mul_(s).add_(s)\n        \n    return img","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.493016Z","iopub.status.idle":"2022-05-26T14:38:09.493699Z","shell.execute_reply.started":"2022-05-26T14:38:09.49343Z","shell.execute_reply":"2022-05-26T14:38:09.493472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = tqdm(ph_dl, leave=False, total=ph_dl.__len__())\nfor i, photo in enumerate(t):\n    with torch.no_grad():\n        pred_monet = gan.gen_ptm(photo.to(device)).cpu().detach()\n    pred_monet = unnorm(pred_monet)\n    img = pil_trans(pred_monet[0]).convert(\"RGB\")\n    img.save(\"../images/\" + str(i+1) + \".jpg\")","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.495053Z","iopub.status.idle":"2022-05-26T14:38:09.495698Z","shell.execute_reply.started":"2022-05-26T14:38:09.495454Z","shell.execute_reply":"2022-05-26T14:38:09.495477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{"execution":{"iopub.status.busy":"2022-05-26T14:38:09.497018Z","iopub.status.idle":"2022-05-26T14:38:09.497717Z","shell.execute_reply.started":"2022-05-26T14:38:09.497462Z","shell.execute_reply":"2022-05-26T14:38:09.497488Z"},"trusted":true},"execution_count":null,"outputs":[]}]}