{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21755,"databundleVersionId":1475600,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Monet paintings GAN transformer\n\n#### The idea for this project (taken from kaggle contests) was to utilize photos taken in 256x256 RGB format and turn them into *Monet-styled* paintings.\n#### We will be utilizing a simple GAN in order to take on this challenge!","metadata":{"execution":{"iopub.status.busy":"2024-01-12T20:16:02.694311Z","iopub.execute_input":"2024-01-12T20:16:02.694633Z","iopub.status.idle":"2024-01-12T20:16:02.699249Z","shell.execute_reply.started":"2024-01-12T20:16:02.694608Z","shell.execute_reply":"2024-01-12T20:16:02.698256Z"}}},{"cell_type":"markdown","source":"## Basic imports\n#### The idea behind these imports was to try and learn how to use torch convolutional networks whilst making two separate networks to tackle this problem (the generator and the discriminator)","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom tqdm.notebook import tqdm\nimport torch.optim as optim\nimport torch.nn as nn\nfrom PIL import Image\nimport torchvision\nimport shutil\nimport torch\nimport os\nimport random\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom torchvision.utils import make_grid","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-12T12:09:23.624465Z","iopub.execute_input":"2024-01-12T12:09:23.625655Z","iopub.status.idle":"2024-01-12T12:09:23.632057Z","shell.execute_reply.started":"2024-01-12T12:09:23.625610Z","shell.execute_reply":"2024-01-12T12:09:23.631091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset constructor\n\n#### More specs on how this constructor works are included below but in a nutshell it will just take in the data in a folder and allow it to be used by the Dataloader in torch.utils","metadata":{}},{"cell_type":"code","source":"'''\n    Basic dataset class, will be used to manipulate the monet dataset \n    \n    __init__ args:          folder_path       :       path to folder with dataset images\n                            transform         :       transform module from torchvision for data augmentation\n                            __len__           :       number of images in dataset\n                            __get_item__      :       returns image at index idx tranformed\n                            __get_crude_item_ :       returns image at index idx \n                            \n                \n'''\n\n\nclass PaintingsDataset(Dataset):\n    def __init__(self, folder_path, transform=None):\n        self.folder_path = folder_path\n        self.transform = transform\n        self.images = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith(('.jpg', '.jpeg', '.png'))]\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        image_path = self.images[idx]\n        image = Image.open(image_path).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        return image\n    \n    def __get_crude_item__(self, idx):\n        image_path = self.images[idx]\n        image = Image.open(image_path).convert('RGB')\n        return image\n        ","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:23.636474Z","iopub.execute_input":"2024-01-12T12:09:23.636774Z","iopub.status.idle":"2024-01-12T12:09:23.649138Z","shell.execute_reply.started":"2024-01-12T12:09:23.636750Z","shell.execute_reply":"2024-01-12T12:09:23.648202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Device adptation, hyperparameters used in training/creating both networks and data augmentation\n\n#### Basically, we will be using <code>torch.cuda.is_available()</code> to check if cuda is available and change model to cuda based on the result\n#### Additionally, the hyperparameters and filters in data augmentation (in order to reduce overfitting) were fine-tuned during training and also searched from other public kaggle notebooks for reference (more on that later, in References)","metadata":{}},{"cell_type":"code","source":"# Device and tensors configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nTensor = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.Tensor\n\n# Hyperparameters\nbatch_size = 6\nlr = 0.0001\nbeta1 = 0.56\nbeta2 = 0.992\nn_epoches = 96\ndisplay_epoch = 16\n\n\"\"\"\nTranform module:\n\nonly applying GaussianBlur to dataset, more transformations seemed like a over-kill\n\n\"\"\"\ntransform = transforms.Compose([\n    transforms.Resize((256, 256)),\n\n    # Data augmentation \n    #transforms.ElasticTransform(alpha=150.0),\n    transforms.GaussianBlur(kernel_size=(5, 9), sigma=(0.1, 3.)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.15, 0.15, 0.15], std=[0.5, 0.5, 0.5])\n])\n\n# Initialize datasets\nmonet_dataset = PaintingsDataset('/kaggle/input/gan-getting-started/monet_jpg', transform)\nphoto_dataset = PaintingsDataset('/kaggle/input/gan-getting-started/photo_jpg', transform)\n\n\n# Initialize data loaders\nmonet_loader = DataLoader(monet_dataset, batch_size=batch_size, shuffle=True)\nphoto_loader = DataLoader(photo_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:23.650665Z","iopub.execute_input":"2024-01-12T12:09:23.651374Z","iopub.status.idle":"2024-01-12T12:09:23.682106Z","shell.execute_reply.started":"2024-01-12T12:09:23.651349Z","shell.execute_reply":"2024-01-12T12:09:23.681411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliary function to display images before and after data augmentation ","metadata":{}},{"cell_type":"code","source":"\"\"\"\nShow a random image n from dataset with it's original format and transformed format\n\nargs:\n        n : example image index\n        dataset : dataset of interest\n\nreturns:\n        plot's with original and transformed image side-by-side\n\"\"\"\n\ndef show_images(n, dataset):\n    ex_im_transf = dataset.__getitem__(n)\n    ex_im_crude  = dataset.__get_crude_item__(n)\n\n    im_transf = np.squeeze(ex_im_transf) \n    im_crude = np.squeeze(ex_im_crude)\n\n    f, axarr = plt.subplots(1,2)\n\n    for i in range(2):\n        axarr[i].axes.get_xaxis().set_visible(False)\n        axarr[i].axes.get_yaxis().set_visible(False)\n\n    axarr[1].imshow(im_transf.permute(1, 2, 0))\n    axarr[0].imshow(im_crude)\n    axarr[1].set_title(\"Transformed image\")\n    axarr[0].set_title(\"Original image\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:23.683036Z","iopub.execute_input":"2024-01-12T12:09:23.683298Z","iopub.status.idle":"2024-01-12T12:09:23.690340Z","shell.execute_reply.started":"2024-01-12T12:09:23.683275Z","shell.execute_reply":"2024-01-12T12:09:23.689400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize data augmentation in both datasets\nrandn = random.randint(1,298)\nshow_images(n=randn, dataset=monet_dataset)\nshow_images(n=randn, dataset=photo_dataset)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:23.692432Z","iopub.execute_input":"2024-01-12T12:09:23.693003Z","iopub.status.idle":"2024-01-12T12:09:24.003211Z","shell.execute_reply.started":"2024-01-12T12:09:23.692971Z","shell.execute_reply":"2024-01-12T12:09:24.002334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models used for the generator and discriminator\n\n#### One of the most important parts of the notebook, by far.\n#### The idea behind these models was (among other aspects) to to try to make the generator more complex than the discriminator, otherwise the discriminator almost always outpaces the generator and the losses in the generator estabilize way too soon.\n#### This model and its training was inspired by <a href=\" https://www.kaggle.com/code/adikaboost/cyclegan-resnet-pytorch/notebook\"> this notebook </a> and <a href=\" https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html\"> this torch tutorial</a>","metadata":{}},{"cell_type":"code","source":"class Discriminator(nn.Module):\n    def __init__(self):\n        super(Discriminator, self).__init__()\n\n        self.model = nn.Sequential(\n    \n            nn.Conv2d(3, 64, kernel_size=4, stride=2, padding=1),\n            # using ReLU instead of LeakyReLU for faster processing, data is relatively clean and has few outliers.\n            nn.ReLU(inplace=True),\n        \n            nn.Conv2d(64, 128, kernel_size=4, stride=2, padding=1),\n            # IstanceNorm because we are using few n in batch\n            nn.InstanceNorm2d(128),\n            nn.ReLU(inplace=True),\n            \n            nn.Conv2d(128, 256, kernel_size=4, stride=2, padding=1),\n            nn.InstanceNorm2d(256),\n            nn.ReLU(inplace=True),\n            \n            nn.Conv2d(256, 512, kernel_size=4, stride=2, padding=1),\n            nn.InstanceNorm2d(512),\n            nn.ReLU(inplace=True),\n            \n            nn.ConvTranspose2d(512, 256, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.InstanceNorm2d(256),\n            nn.ReLU(inplace=True),   \n            \n            nn.ConvTranspose2d(256, 128, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.InstanceNorm2d(128),\n            nn.ReLU(inplace=True),   \n            \n            nn.ConvTranspose2d(128, 64, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.InstanceNorm2d(128),\n            nn.ReLU(inplace=True),\n            \n            nn.ConvTranspose2d(64, 1, kernel_size=3, stride=2, padding=1, output_padding=1)\n\n            \n        )\n\n    def forward(self, x):\n        return self.model(x)\n    \n\nclass ResidualBlock(nn.Module):\n    def __init__(self):\n        super(ResidualBlock, self).__init__()\n        self.block = nn.Sequential(\n            nn.ReflectionPad2d(1),\n            nn.Conv2d(256, 256, 3),\n            nn.InstanceNorm2d(256),\n            nn.ReLU(inplace=True),\n            nn.ReflectionPad2d(1),\n            nn.Conv2d(256, 256, 3),\n            nn.InstanceNorm2d(256)\n        )\n\n    def forward(self, x):\n        return x + self.block(x)\n\nclass Generator(nn.Module):\n    def __init__(self, num_residual_blocks=9):\n        super(Generator, self).__init__()\n\n        self.initial = nn.Sequential(\n            \n            nn.ReflectionPad2d(3),\n            nn.Conv2d(3, 64, 2 * 3 + 1),\n            nn.InstanceNorm2d(64),\n            nn.ReLU(inplace=True)\n            \n        )\n\n        self.downsample_blocks = nn.Sequential(\n            \n            nn.Conv2d(64, 128, 3, stride=2, padding=1),\n            nn.InstanceNorm2d(128),\n            nn.ReLU(inplace=True),\n            \n            nn.Conv2d(128, 256, 3, stride=2, padding=1),\n            nn.InstanceNorm2d(256),\n            nn.ReLU(inplace=True)\n            \n        )\n\n        self.residual_blocks = nn.Sequential(*[ResidualBlock() for _ in range(num_residual_blocks)])\n\n        self.upsample_blocks = nn.Sequential(\n            \n            nn.ConvTranspose2d(256, 128, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.InstanceNorm2d(128),\n            nn.ReLU(inplace=True),   \n            \n            nn.ConvTranspose2d(128, 64, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.InstanceNorm2d(64),\n            nn.ReLU(inplace=True)\n            \n        )\n\n        self.output = nn.Sequential(\n            nn.ReflectionPad2d(3),\n            nn.Conv2d(64, 3, 2 * 3 + 1),\n            nn.Tanh()\n        )\n\n    def forward(self, x):\n        x = self.initial(x)\n        x = self.downsample_blocks(x)\n        x = self.residual_blocks(x)\n        x = self.upsample_blocks(x)\n        return self.output(x)","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:24.004773Z","iopub.execute_input":"2024-01-12T12:09:24.005440Z","iopub.status.idle":"2024-01-12T12:09:24.026457Z","shell.execute_reply.started":"2024-01-12T12:09:24.005388Z","shell.execute_reply":"2024-01-12T12:09:24.025750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Instantiating generator and discriminator object and declaring loss functions and optimezers used\n#### Mean Squared Error (MSE) and L1 loss were used as they proved to be more effective and hastly converged toward a local minimun when compared with other loss functions such as Huber loss\n#### Adam optimizer was chosen for the same reason\n","metadata":{}},{"cell_type":"code","source":"G = Generator()\nD = Discriminator()\n\nmodels = [G, D]\nfor model in models:\n    if torch.cuda.is_available():\n        model.cuda()\n        \ngan_loss = nn.MSELoss().cuda()\ncycle_loss = nn.L1Loss().cuda()\nidentity_loss = nn.L1Loss().cuda()\n\noptimizer_G = torch.optim.Adam(G.parameters(), lr=lr, betas=(beta1, beta2))\noptimizer_D = torch.optim.Adam(D.parameters(), lr=lr, betas=(beta1, beta2))","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:24.028808Z","iopub.execute_input":"2024-01-12T12:09:24.029448Z","iopub.status.idle":"2024-01-12T12:09:24.167771Z","shell.execute_reply.started":"2024-01-12T12:09:24.029399Z","shell.execute_reply":"2024-01-12T12:09:24.167011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliary function to display artificial monet images before and after generator training\n#### This function proved very effective at demonstrating the progress of the generator with traning.","metadata":{}},{"cell_type":"code","source":"\"\"\"\nSample images from batch size n with current generator model parameters\n\nargs:\n        images with format (n x 3 x size(image) x size(image))\n\nreturns:\n        plot's with original and transformed image side-by-side\n\"\"\"\n\ndef sample_images(real_X):    \n    # turn off some learning parameters in order to evaluate model:\n    G.eval()\n    \n    # transform to tensor and plot images\n    real_pic = real_X.type(Tensor)\n    artificial_pic = G(real_X).detach()\n\n    ncols = real_X.size(0)\n    real_pic_grid = make_grid(real_pic, nrow=ncols, normalize=True)\n    artificial_pic_grid = make_grid(artificial_pic, nrow=ncols, normalize=True)\n\n    fig, axs = plt.subplots(2, 1, figsize=(20, 4))  \n\n    axs[0].imshow(real_pic_grid.permute(1, 2, 0).cpu())\n    axs[0].set_title(\"Augmented real photos from photo dataset\")\n    axs[0].axis('off')\n\n    axs[1].imshow(artificial_pic_grid.permute(1, 2, 0).cpu())\n    axs[1].set_title(\"Generated monet paintings from photos\")\n    axs[1].axis('off')\n\n\n    plt.show()\n\nreal_X = next(iter(photo_loader))\nreal_Y = next(iter(photo_loader))\nsample_images(real_X.to(device))","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:24.168917Z","iopub.execute_input":"2024-01-12T12:09:24.169264Z","iopub.status.idle":"2024-01-12T12:09:24.723079Z","shell.execute_reply.started":"2024-01-12T12:09:24.169231Z","shell.execute_reply":"2024-01-12T12:09:24.722170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Network training\n#### Another very significant part of the project!\n#### In a nutshell, the generator losses were divided in 2 parts: <code>identity loss</code> and <code>gan loss</code> and both had to impact the backward propagation in different proportions. Therefore both were added and used in the optimizer.\n#### The dicriminator loss was also divided in two parts:  <code>real painting missclassification loss</code> and  <code>artificial painting missclassification loss</code>. Basically the loss when misclassifying a fake painting as a true painting and misclassifying a true painting as a fake paiting.","metadata":{}},{"cell_type":"code","source":"for epoch in range(n_epoches):\n    for i, (real_photo, real_monet) in enumerate(zip(photo_loader, monet_loader)):\n\n        real_photo = real_photo.type(Tensor)\n        real_monet = real_monet.type(Tensor)\n        \n        out_shape = [real_photo.size(0), 1, real_photo.size(2) , real_photo.size(3)]\n        \n        valid = torch.ones(out_shape).type(Tensor)\n        fake = torch.zeros(out_shape).type(Tensor)\n        \n        \n        # training generator\n        G.train()\n        optimizer_G.zero_grad()\n        artificial_monet = G(real_photo)\n\n        \n        # identity loss\n        loss_identity = identity_loss(artificial_monet, real_photo)\n        \n        # gan loss\n        loss_GAN = gan_loss(D(artificial_monet), valid) \n        \n        # total gan loss\n        loss_G = 5.0 * loss_identity + loss_GAN \n        \n        loss_G.backward()\n        optimizer_G.step()\n        \n        # training discriminator \n        optimizer_D.zero_grad()\n        \n        # real painting misclassification loss\n        loss_real = gan_loss(D(real_monet), valid)\n        \n        # artificial painting misclassification loss\n        loss_fake = gan_loss(D(artificial_monet.detach()), fake)\n        loss_D = (loss_real + loss_fake) / 2\n        \n        loss_D.backward()\n        optimizer_D.step()\n    \n    # sample results every n=display_epoch\n    \n    if (epoch + 1) % display_epoch == 0:\n        test_real_photo, test_real_monet = next(iter(zip(photo_loader, monet_loader)))\n        sample_images(test_real_photo.type(Tensor))\n\n        print(f'Epoch {epoch + 1} | {n_epoches}')\n        print(f'Generator loss: {loss_G.item()} =  {5 * loss_identity.item()} (5 * identity loss) + {loss_GAN.item()} (GAN loss)')\n        print(f'Discriminator loss: {loss_D.item()} = {loss_real.item()/2} (real monet loss/2)  + {loss_fake.item()} (generated monet loss/2)')","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:09:24.724371Z","iopub.execute_input":"2024-01-12T12:09:24.724745Z","iopub.status.idle":"2024-01-12T12:41:21.922509Z","shell.execute_reply.started":"2024-01-12T12:09:24.724711Z","shell.execute_reply":"2024-01-12T12:41:21.921536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create directory to save new images\n#### Pretty straightfoward, just save the images batch by batch","metadata":{}},{"cell_type":"code","source":"# create path to save created monet paintings\ntransformed_save_dir = '../images'\nif not os.path.exists(transformed_save_dir):\n    os.makedirs(transformed_save_dir)\n    \n# freeze some network params to generate imgs\nG.eval()\n\n\nfor i, real_photos in enumerate(photo_loader):\n    real_photos = real_photos.to(device)\n    with torch.no_grad():\n        monet_style_imgs = G(real_photos)\n\n    # Save each transformed image\n    for j, img in enumerate(monet_style_imgs):\n        save_path = os.path.join(transformed_save_dir, f'monet_painting_{i * batch_size + j}.png')\n        torchvision.utils.save_image(img, save_path)\n\nprint(f\"Transformed images are saved in {transformed_save_dir}\")\n\nshutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{"execution":{"iopub.status.busy":"2024-01-12T12:41:21.923900Z","iopub.execute_input":"2024-01-12T12:41:21.924247Z","iopub.status.idle":"2024-01-12T12:46:19.365395Z","shell.execute_reply.started":"2024-01-12T12:41:21.924217Z","shell.execute_reply":"2024-01-12T12:46:19.364353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## References\n#### Most of my work was inspired by two kaggle notebooks that tackled this problem before me and the Pytorch DCGAN tutorial: <a href=\" https://www.kaggle.com/code/adikaboost/cyclegan-resnet-pytorch/notebook\"> notebook 1</a>, <a href=\" https://www.kaggle.com/code/adikaboost/cyclegan-resnet-pytorch/notebook\"> notebook 2</a>, <a href=\"https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html\">DCGAN tutorial</a>.\n#### The monet and photos dataset provided belongs to <a href=\"https://www.kaggle.com\">Kaggle</a>.\n#### Thanks for checking out my project! :)","metadata":{}}]}