{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Final Project - Deep Learning \n\n\n---\n<br>\n\n### Name and ID:\nStudent 1: Ido Tziony 206534299\n<br>\nStudent 2: Itai Mondshine 207814724","metadata":{"id":"XXqIDAd1SeMX","papermill":{"duration":0.017784,"end_time":"2023-02-19T18:06:51.445423","exception":false,"start_time":"2023-02-19T18:06:51.427639","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Preprocessing\n","metadata":{"id":"Ydj3uQVPK93x","papermill":{"duration":0.013554,"end_time":"2023-02-19T18:06:51.526170","exception":false,"start_time":"2023-02-19T18:06:51.512616","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Setup and Libraries","metadata":{"id":"5v7dw9zgLGuU","papermill":{"duration":0.013158,"end_time":"2023-02-19T18:06:51.552728","exception":false,"start_time":"2023-02-19T18:06:51.539570","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%capture\n!pip install --upgrade --no-cache-dir gdown\n!gdown 1SQ9yb8gL7KU-OefmgSsJ3EmktKY7chAH&confirm=t\n!unzip /content/dl.zip","metadata":{"papermill":{"duration":6.109273,"end_time":"2023-02-19T18:06:57.675472","exception":false,"start_time":"2023-02-19T18:06:51.566199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-27T23:00:07.230288Z","iopub.execute_input":"2023-02-27T23:00:07.230628Z","iopub.status.idle":"2023-02-27T23:00:22.154893Z","shell.execute_reply.started":"2023-02-27T23:00:07.230552Z","shell.execute_reply":"2023-02-27T23:00:22.153197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import print_function\n\nimport random\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import layers\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport os\nfrom PIL import Image\nfrom keras.models import Sequential\nfrom keras.initializers import RandomNormal\nfrom keras import backend as K\nimport gc\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom pathlib import Path\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nimport copy\n\n#Load if gpu avalable\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:08:48.429514Z","iopub.execute_input":"2023-02-27T23:08:48.429987Z","iopub.status.idle":"2023-02-27T23:08:48.444376Z","shell.execute_reply.started":"2023-02-27T23:08:48.429947Z","shell.execute_reply":"2023-02-27T23:08:48.443166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(tf.config.experimental.list_physical_devices('GPU')) > 0:\n    config=tf.compat.v1.ConfigProto(log_device_placement=True, device_count={'GPU': 1})\n    tf.compat.v1.Session()\n    print(\"Using GPU for TensorFlow\")\nelse:\n    print(\"Using CPU for TensorFlow\")\n\n\nprint(\"Using tf \", tf.__version__)\nprint(\"Executing eagerly: \",tf.executing_eagerly())\n\nconfig.gpu_options.allow_growth = True\nsession = tf.compat.v1.Session(config=config)\n\nK.set_session(session)\ngc.enable()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:02:25.573750Z","iopub.execute_input":"2023-02-27T23:02:25.574382Z","iopub.status.idle":"2023-02-27T23:02:32.058085Z","shell.execute_reply.started":"2023-02-27T23:02:25.574350Z","shell.execute_reply":"2023-02-27T23:02:32.057086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_SIZE = 256\nOUTPUT_SIZE = 320\n\ndef directory_loader(folder_path):\n    folder_path = Path(folder_path)\n    image_files = [file for file in folder_path.iterdir() if file.suffix == \".jpg\"]\n    random.shuffle(image_files)\n    for file in image_files:\n      image = Image.open(file)\n      image = loader(image).unsqueeze(0)\n      yield image.to(device, torch.float)\n\nloader = transforms.Compose([\n    transforms.Resize(INPUT_SIZE), \n    transforms.ToTensor()])  \n\n\ndef image_loader(image_name):\n    image = Image.open(image_name)\n    image = loader(image).unsqueeze(0)\n    return image.to(device, torch.float)\n\nunloader = transforms.ToPILImage() \n\nplt.ion()\n\ndef imshow(tensor, title=None):\n    image = tensor.cpu().clone()  \n    image = image.squeeze(0)      \n    image = unloader(image)\n    plt.imshow(image)\n    if title is not None:\n        plt.title(title)\n\nresize_to_320 = transforms.Resize(OUTPUT_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:02:36.500583Z","iopub.execute_input":"2023-02-27T23:02:36.501003Z","iopub.status.idle":"2023-02-27T23:02:36.512358Z","shell.execute_reply.started":"2023-02-27T23:02:36.500953Z","shell.execute_reply":"2023-02-27T23:02:36.511316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loadint the images","metadata":{"id":"hN0Sf9GNPSOM","papermill":{"duration":0.014204,"end_time":"2023-02-19T18:06:57.704605","exception":false,"start_time":"2023-02-19T18:06:57.690401","status":"completed"},"tags":[]}},{"cell_type":"code","source":"style_image = image_loader(\"/kaggle/input/gan-getting-started/monet_jpg/4ad8b366c1.jpg\")\ncontent_image = image_loader(\"/kaggle/input/gan-getting-started/photo_jpg/04838b8b1e.jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:03:45.110358Z","iopub.execute_input":"2023-02-27T23:03:45.110776Z","iopub.status.idle":"2023-02-27T23:03:45.206937Z","shell.execute_reply.started":"2023-02-27T23:03:45.110742Z","shell.execute_reply":"2023-02-27T23:03:45.205761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ContentLoss(nn.Module):\n    def __init__(self, target,):\n        super(ContentLoss, self).__init__()\n        self.target = target.detach()\n\n    def forward(self, input):\n        self.loss = F.mse_loss(input, self.target)\n        return input","metadata":{"papermill":{"duration":3.787131,"end_time":"2023-02-19T18:07:01.505144","exception":false,"start_time":"2023-02-19T18:06:57.718013","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-27T23:03:54.503635Z","iopub.execute_input":"2023-02-27T23:03:54.504111Z","iopub.status.idle":"2023-02-27T23:03:54.513664Z","shell.execute_reply.started":"2023-02-27T23:03:54.504073Z","shell.execute_reply":"2023-02-27T23:03:54.512451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def gram_matrix(input_tensor):\n#     _, channels, height, width = input_tensor.size()\n#     features = input_tensor.view(channels, height * width)\n#     gram = torch.mm(features, features.t())\n#     gram /= (channels * height * width)\n\n#     return gram\n\n\n# class StyleLoss(nn.Module):\n\n#     def __init__(self, target_feature):\n#         super(StyleLoss, self).__init__()\n#         self.target = gram_matrix(target_feature).detach()\n\n#     def forward(self, input):\n#         G = gram_matrix(input)\n#         self.loss = F.mse_loss(G, self.target)\n#         return input\n\n\n# For using path gram matrix\n\ndef patch_gram_matrix(input_tensor, patch_size=3, stride=1):\n    _, channels, height, width = input_tensor.size()\n    patches = F.unfold(input_tensor, kernel_size=patch_size, stride=stride)\n    patches = patches.view(channels, patch_size * patch_size, -1)\n    gram = torch.bmm(patches, patches.transpose(1, 2))\n    gram /= (patch_size * patch_size * height * width)\n    return gram\n\n\nclass StyleLoss(nn.Module):\n\n    def __init__(self, target_feature, patch_size=3, stride=1):\n        super(StyleLoss, self).__init__()\n        self.target = patch_gram_matrix(target_feature, patch_size, stride).detach()\n\n    def forward(self, input):\n        G = patch_gram_matrix(input, 3, 1)\n        self.loss = F.mse_loss(G, self.target)\n        return input","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:04:50.734777Z","iopub.execute_input":"2023-02-27T23:04:50.735132Z","iopub.status.idle":"2023-02-27T23:04:50.744983Z","shell.execute_reply.started":"2023-02-27T23:04:50.735103Z","shell.execute_reply":"2023-02-27T23:04:50.743834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Normalization(nn.Module):\n    def __init__(self, mean, std):\n        super(Normalization, self).__init__()\n        self.mean = torch.tensor(mean).view(-1, 1, 1)\n        self.std = torch.tensor(std).view(-1, 1, 1)\n\n    def forward(self, img):\n        return (img - self.mean) / self.std","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:04:59.316943Z","iopub.execute_input":"2023-02-27T23:04:59.317642Z","iopub.status.idle":"2023-02-27T23:04:59.323903Z","shell.execute_reply.started":"2023-02-27T23:04:59.317604Z","shell.execute_reply":"2023-02-27T23:04:59.322680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn = models.vgg19(pretrained=True).features.to(device).eval()   \n\nfor param in cnn.parameters():\n    param.requires_grad_(False)\n\ncnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)\ncnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:05:06.568202Z","iopub.execute_input":"2023-02-27T23:05:06.568554Z","iopub.status.idle":"2023-02-27T23:05:11.668918Z","shell.execute_reply.started":"2023-02-27T23:05:06.568522Z","shell.execute_reply":"2023-02-27T23:05:11.667941Z"},"jupyter":{"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Construction","metadata":{}},{"cell_type":"code","source":"content_layers_default = ['conv_4']\nstyle_layers_default = ['conv_1', 'conv_2', 'conv_3', 'conv_4', 'conv_5']\nTYPE_TO_NAME = {'Conv2d': 'conv', 'ReLU': 'relu','MaxPool2d': 'max_pool', 'BatchNorm2d': 'bn'}\nOPTIMIZER = optim.LBFGS","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:05:24.204761Z","iopub.execute_input":"2023-02-27T23:05:24.205334Z","iopub.status.idle":"2023-02-27T23:05:24.211447Z","shell.execute_reply.started":"2023-02-27T23:05:24.205300Z","shell.execute_reply":"2023-02-27T23:05:24.210035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _clac_loss(model, image, loss_array, _type, i):\n  target = model(image).detach()\n  loss = ContentLoss(target) if _type == 'content' else StyleLoss(target)\n  model.add_module(f\"{_type}_loss_{i}\", loss)\n  loss_array.append(loss)\n\ndef _clamp_image(image):\n  with torch.no_grad():\n      image.clamp_(0, 1)\n\ndef from_tensor(tensor):\n    image = tensor.cpu().clone()  \n    image = image.squeeze(0)      \n    image = unloader(image)\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-02-27T23:05:40.911178Z","iopub.execute_input":"2023-02-27T23:05:40.911536Z","iopub.status.idle":"2023-02-27T23:05:40.921169Z","shell.execute_reply.started":"2023-02-27T23:05:40.911507Z","shell.execute_reply":"2023-02-27T23:05:40.920204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_style_transfer_network(cnn, cnn_normalization_mean, cnn_normalization_std, style_img, content_img,\n                              content_layers=content_layers_default,style_layers=style_layers_default):\n\n  normalization = Normalization(cnn_normalization_mean, cnn_normalization_std).to(device)\n\n  content_losses = []\n  style_losses = []\n\n  model = nn.Sequential(normalization)\n\n  i = 0  \n  for layer in cnn.children():\n      if isinstance(layer, nn.Conv2d):\n          i += 1\n      elif isinstance(layer, nn.ReLU):\n          layer = nn.ReLU(inplace=False)   \n      name = f\"{TYPE_TO_NAME[str(type(layer).__name__)]}_{i}\"\n\n      model.add_module(name, layer)\n\n      if name in content_layers:\n          _clac_loss(model, content_img, content_losses, _type='content', i=i)\n\n      if name in style_layers:\n          _clac_loss(model, style_img, style_losses, _type='style', i=i)\n\n  for i in range(len(model) - 1, -1, -1):\n      if isinstance(model[i], ContentLoss) or isinstance(model[i], StyleLoss):\n          break\n\n\n  return model[:(i + 1)], style_losses, content_losses","metadata":{"papermill":{"duration":0.023081,"end_time":"2023-02-19T18:07:01.541854","exception":false,"start_time":"2023-02-19T18:07:01.518773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-27T23:05:47.988253Z","iopub.execute_input":"2023-02-27T23:05:47.988604Z","iopub.status.idle":"2023-02-27T23:05:47.997949Z","shell.execute_reply.started":"2023-02-27T23:05:47.988575Z","shell.execute_reply":"2023-02-27T23:05:47.996921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Run this if using GPU","metadata":{"id":"hUI8lFbjGGV7","papermill":{"duration":0.018947,"end_time":"2023-02-19T18:07:10.574462","exception":false,"start_time":"2023-02-19T18:07:10.555515","status":"completed"},"tags":[]}},{"cell_type":"code","source":"input_image = content_image.clone()\n\nassert input_image.shape == torch.Size([1, 3, 256, 256])\nplt.figure()\nimshow(input_image, title='Input Image')","metadata":{"id":"cOJnNPnrGGi7","papermill":{"duration":14.015853,"end_time":"2023-02-19T18:07:24.603973","exception":false,"start_time":"2023-02-19T18:07:10.588120","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-27T23:05:57.369578Z","iopub.execute_input":"2023-02-27T23:05:57.370205Z","iopub.status.idle":"2023-02-27T23:05:57.732292Z","shell.execute_reply.started":"2023-02-27T23:05:57.370167Z","shell.execute_reply":"2023-02-27T23:05:57.731362Z"},"jupyter":{"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This imports all the necessary libraries.","metadata":{"id":"16tb-oOL_QAa","papermill":{"duration":0.013668,"end_time":"2023-02-19T18:07:24.631935","exception":false,"start_time":"2023-02-19T18:07:24.618267","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std, content_image, style_image, input_image, num_steps=300,\n                       style_weight=1000000, content_weight=1):\n    print('Building the style transfer model..')\n    model, style_losses, content_losses = create_style_transfer_network(cnn, cnn_normalization_mean, cnn_normalization_std,\n        style_image, content_image)\n\n    input_image.requires_grad_(True)\n    model.requires_grad_(False)\n\n    optimizer = OPTIMIZER([input_image]) \n\n    totla_loss = []\n    style_loss = []\n    content_loss = []\n    print('Optimizing..')\n    i = [0]\n    while i[0] <= num_steps:\n\n        def training_step():\n            _clamp_image(input_image)\n\n            optimizer.zero_grad()\n            model(input_image)\n            style_score = 0\n            content_score = 0\n\n            style_score = sum([sl.loss for sl in style_losses]) * style_weight\n            content_score = sum([cl.loss for cl in content_losses]) * content_weight\n\n            loss = style_score + content_score\n\n            totla_loss.append(loss.item())\n            style_loss.append(style_score.item())\n            content_loss.append(content_score.item())\n\n            loss.backward()\n\n            i[0] += 1\n            if i[0] % 50 == 0:\n                print(\"run {}:\".format(i))\n                print('Style Loss : {:4f} Content Loss: {:4f}'.format(\n                    style_score.item(), content_score.item()))\n                print()\n\n            return style_score + content_score\n\n        optimizer.step(training_step)\n\n    _clamp_image(input_image)\n\n    return input_image, totla_loss, style_loss, content_loss","metadata":{"id":"ONvXJE7tJwKs","papermill":{"duration":0.024078,"end_time":"2023-02-19T18:07:24.671100","exception":false,"start_time":"2023-02-19T18:07:24.647022","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-27T23:11:08.411788Z","iopub.execute_input":"2023-02-27T23:11:08.412154Z","iopub.status.idle":"2023-02-27T23:11:08.423986Z","shell.execute_reply.started":"2023-02-27T23:11:08.412116Z","shell.execute_reply":"2023-02-27T23:11:08.423009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_image = content_image.clone()\noutput, total_loss, style_loss, content_loss = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std,\n                            content_image, style_image, input_image, num_steps=500,\n                       style_weight=10000, content_weight=1)\n","metadata":{"id":"kmKE2yPo8BDe","outputId":"a11ad3c9-e9a1-4a48-9f7b-04907c258413","papermill":{"duration":0.036058,"end_time":"2023-02-19T18:07:24.748786","exception":false,"start_time":"2023-02-19T18:07:24.712728","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-27T23:06:14.483413Z","iopub.execute_input":"2023-02-27T23:06:14.489350Z","iopub.status.idle":"2023-02-27T23:06:51.986888Z","shell.execute_reply.started":"2023-02-27T23:06:14.489304Z","shell.execute_reply":"2023-02-27T23:06:51.985898Z"},"jupyter":{"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Competition Entries","metadata":{"id":"EtNskt5wKHlU","papermill":{"duration":0.303366,"end_time":"2023-02-19T19:55:38.435699","exception":false,"start_time":"2023-02-19T19:55:38.132333","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def to_image(output):\n    output = output.cpu().clone()  \n    output = output.squeeze(0)  \n    unloader = transforms.ToPILImage() \n    return unloader(output)\n\nphotos_directory = '/kaggle/input/gan-getting-started/photo_jpg/'\n\n! mkdir ../images\nimport PIL\nimport time\n\nfor i, filename in tqdm(enumerate(os.listdir(photos_directory))):\n    content_image = input_image = image_loader(os.path.join(photos_directory, filename)) \n    output, total_loss, style_loss, content_loss = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std,\n                                content_image, style_image, input_image, num_steps=50,\n                           style_weight=100000, content_weight=1)\n    im = to_image(output)\n    \n    im.save(\"../images/\" + str(i) + \".jpg\")\n    if (i+1)%4 == 0:\n        print(\"Finished \" + str(i+1) +\" images\")","metadata":{"papermill":{"duration":10363.789662,"end_time":"2023-02-19T22:48:22.532271","exception":false,"start_time":"2023-02-19T19:55:38.742609","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-27T23:23:18.635449Z","iopub.execute_input":"2023-02-27T23:23:18.635866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n","metadata":{"execution":{"iopub.status.busy":"2023-02-19T15:06:18.642781Z","iopub.status.idle":"2023-02-19T15:06:18.644633Z","shell.execute_reply":"2023-02-19T15:06:18.644364Z","shell.execute_reply.started":"2023-02-19T15:06:18.644336Z"},"papermill":{"duration":0.377515,"end_time":"2023-02-19T22:48:23.226884","exception":false,"start_time":"2023-02-19T22:48:22.849369","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Save to zip","metadata":{}},{"cell_type":"code","source":"import shutil\nshutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{"papermill":{"duration":4.975421,"end_time":"2023-02-19T22:48:28.518606","exception":false,"start_time":"2023-02-19T22:48:23.543185","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-28T15:53:11.656210Z","iopub.execute_input":"2023-02-28T15:53:11.656684Z","iopub.status.idle":"2023-02-28T15:53:11.747736Z","shell.execute_reply.started":"2023-02-28T15:53:11.656584Z","shell.execute_reply":"2023-02-28T15:53:11.746139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}