{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Neural Transfer - I’m Something of a Painter Myself\n\nImplemetion of the Neural-Transfer algorithm developed by Leon A. Gatys, Alexander S. Ecker and Matthias Bethge. Neural-Transfer, allows you to take an image and reproduce it with a new artistic style. The algorithm takes three images, an input image, a content-image, and a style-image, and changes the input to resemble the content of the content-image and the artistic style of the style-image.\n\nhttps://arxiv.org/abs/1508.06576","metadata":{}},{"cell_type":"markdown","source":"## Importing Packages","metadata":{}},{"cell_type":"code","source":"from __future__ import print_function\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nimport torchvision.transforms as transforms\nimport torchvision.models as models\n\nimport copy\nimport time\nimport os\nimport random\nimport shutil\n","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:36.704201Z","iopub.execute_input":"2022-08-28T12:25:36.705251Z","iopub.status.idle":"2022-08-28T12:25:36.712151Z","shell.execute_reply.started":"2022-08-28T12:25:36.705209Z","shell.execute_reply":"2022-08-28T12:25:36.710201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:36.718027Z","iopub.execute_input":"2022-08-28T12:25:36.718986Z","iopub.status.idle":"2022-08-28T12:25:36.724924Z","shell.execute_reply.started":"2022-08-28T12:25:36.718938Z","shell.execute_reply":"2022-08-28T12:25:36.723815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the Images","metadata":{}},{"cell_type":"code","source":"# desired size of the output image\nimsize = 256\n\nloader = transforms.Compose([\n    transforms.Resize(imsize),  # scale imported image (although they should all be the right size)\n    transforms.ToTensor()])  # transform it into a torch tensor\n\ndef image_loader(image_name):\n    image = Image.open(image_name)\n    # fake batch dimension required to fit network's input dimensions\n    image = loader(image).unsqueeze(0)\n    return image.to(device, torch.float)\n\nstyle_img = image_loader(\"../input/gan-getting-started/monet_jpg/000c1e3bff.jpg\")\ncontent_img = image_loader(\"../input/gan-getting-started/photo_jpg/00068bc07f.jpg\")","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:36.733893Z","iopub.execute_input":"2022-08-28T12:25:36.734856Z","iopub.status.idle":"2022-08-28T12:25:36.751258Z","shell.execute_reply.started":"2022-08-28T12:25:36.734817Z","shell.execute_reply":"2022-08-28T12:25:36.749996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"displaying the content and style images:","metadata":{}},{"cell_type":"code","source":"unloader = transforms.ToPILImage()  # reconvert into PIL image\n\nplt.ion()\n\ndef imshow(tensor, title=None):\n    image = tensor.cpu().clone()  # we clone the tensor to not do changes on it\n    image = image.squeeze(0)      # remove the fake batch dimension\n    image = unloader(image)\n    plt.imshow(image)\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001) # pause a bit so that plots are updated\n\n\nplt.figure(figsize=((8, 8)))\nimshow(style_img, title='Style Image')\n\nplt.figure(figsize=((8, 8)))\nimshow(content_img, title='Content Image')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:36.753230Z","iopub.execute_input":"2022-08-28T12:25:36.753792Z","iopub.status.idle":"2022-08-28T12:25:37.720453Z","shell.execute_reply.started":"2022-08-28T12:25:36.753758Z","shell.execute_reply":"2022-08-28T12:25:37.719213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss Functions","metadata":{}},{"cell_type":"markdown","source":"### Content Loss\nThe function takes the feature maps of a layer in a network processing input and returns the weighted content distance between the image and the content image. The distance is the mean square error between the two sets of feature maps.","metadata":{}},{"cell_type":"code","source":"# making the content loss layer\nclass ContentLoss(nn.Module):\n\n    def __init__(self, target,):\n        super(ContentLoss, self).__init__()\n        self.target = target.detach()   # 'detach' the target content from the tree, otherwise the forward method of the criterion will throw an error.\n\n    def forward(self, input):\n        self.loss = F.mse_loss(input, self.target)\n        return input","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:37.722764Z","iopub.execute_input":"2022-08-28T12:25:37.723117Z","iopub.status.idle":"2022-08-28T12:25:37.736182Z","shell.execute_reply.started":"2022-08-28T12:25:37.723083Z","shell.execute_reply":"2022-08-28T12:25:37.735322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Style Loss\ngram matrix - the result of multiplying a given matrix by its transposed matrix.\nIn this application the given matrix is a reshaped version of the feature maps.\nThe style distance is also computed using the mean square error between the image gram matrix and the content gram matrix.","metadata":{}},{"cell_type":"code","source":"# making the content loss layer\ndef gram_matrix(input):\n    a, b, c, d = input.size()  # a=batch size(=1), b=number of feature maps, (c,d)=dimensions of a f. map (N=c*d)\n\n    features = input.view(a * b, c * d)  # resise F_XL into \\hat F_XL\n\n    G = torch.mm(features, features.t())  # compute the gram product\n\n    #'normalize' the values of the gram matrix, by dividing by the number of element in each feature maps.\n    return G.div(a * b * c * d)\n\nclass StyleLoss(nn.Module):\n\n    def __init__(self, target_feature):\n        super(StyleLoss, self).__init__()\n        self.target = gram_matrix(target_feature).detach()\n\n    def forward(self, input):\n        G = gram_matrix(input)\n        self.loss = F.mse_loss(G, self.target)\n        return input","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:37.739945Z","iopub.execute_input":"2022-08-28T12:25:37.741676Z","iopub.status.idle":"2022-08-28T12:25:37.766045Z","shell.execute_reply.started":"2022-08-28T12:25:37.741635Z","shell.execute_reply":"2022-08-28T12:25:37.765119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing the Model","metadata":{}},{"cell_type":"code","source":"cnn = models.vgg19(pretrained=True).features.to(device).eval()   # evaluation mode - Some layers have different behavior during training than evaluation\n\n# normalize the image before sending it into the network (VGG networks are trained on images with each channel normalized by mean=[0.485, 0.456, 0.406] and std=[0.229, 0.224, 0.225])\ncnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)\ncnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)\n\n# create a module to normalize input image so we can easily put it in a nn.Sequential\nclass Normalization(nn.Module):\n    def __init__(self, mean, std):\n        super(Normalization, self).__init__()\n        self.mean = torch.tensor(mean).view(-1, 1, 1)   # make them [C x 1 x 1] so that they can directly work with image Tensor of shape [B x C x H x W]\n        self.std = torch.tensor(std).view(-1, 1, 1)\n\n    def forward(self, img):\n        # normalize img\n        return (img - self.mean) / self.std","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:37.773257Z","iopub.execute_input":"2022-08-28T12:25:37.779480Z","iopub.status.idle":"2022-08-28T12:25:40.192978Z","shell.execute_reply.started":"2022-08-28T12:25:37.779442Z","shell.execute_reply":"2022-08-28T12:25:40.191992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creatting a new Sequential module that has content loss and style loss modules correctly inserted, for adding our content loss and style loss layers immediately after the convolution layer they are detecting. ","metadata":{}},{"cell_type":"code","source":"# desired depth layers to compute style/content losses\ncontent_layers_default = ['conv_4']\nstyle_layers_default = ['conv_1', 'conv_2', 'conv_3', 'conv_4', 'conv_5']\n\ndef get_style_model_and_losses(cnn, normalization_mean, normalization_std, style_img, content_img,\n                               content_layers=content_layers_default,style_layers=style_layers_default):\n\n    normalization = Normalization(normalization_mean, normalization_std).to(device)\n\n    content_losses = []\n    style_losses = []\n\n    # make a new nn.Sequential to put in modules that are supposed to be activated sequentially\n    model = nn.Sequential(normalization)\n\n    i = 0  # increment every time we see a conv\n    for layer in cnn.children():\n        if isinstance(layer, nn.Conv2d):\n            i += 1\n            name = 'conv_{}'.format(i)\n        elif isinstance(layer, nn.ReLU):\n            name = 'relu_{}'.format(i)\n            layer = nn.ReLU(inplace=False)   # The in-place version doesn't play very nicely with the ContentLoss and StyleLoss we insert below. So we replace with out-of-place ones here.\n        elif isinstance(layer, nn.MaxPool2d):\n            name = 'pool_{}'.format(i)\n        elif isinstance(layer, nn.BatchNorm2d):\n            name = 'bn_{}'.format(i)\n        else:\n            raise RuntimeError('Unrecognized layer: {}'.format(layer.__class__.__name__))\n\n        model.add_module(name, layer)\n\n        if name in content_layers:\n            # add content loss:\n            target = model(content_img).detach()\n            content_loss = ContentLoss(target)\n            model.add_module(\"content_loss_{}\".format(i), content_loss)\n            content_losses.append(content_loss)\n\n        if name in style_layers:\n            # add style loss:\n            target_feature = model(style_img).detach()\n            style_loss = StyleLoss(target_feature)\n            model.add_module(\"style_loss_{}\".format(i), style_loss)\n            style_losses.append(style_loss)\n\n    # now trim off the layers after the last content and style losses\n    for i in range(len(model) - 1, -1, -1):\n        if isinstance(model[i], ContentLoss) or isinstance(model[i], StyleLoss):\n            break\n\n    model = model[:(i + 1)]\n\n    return model, style_losses, content_losses","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:40.194464Z","iopub.execute_input":"2022-08-28T12:25:40.195388Z","iopub.status.idle":"2022-08-28T12:25:40.208200Z","shell.execute_reply.started":"2022-08-28T12:25:40.195346Z","shell.execute_reply":"2022-08-28T12:25:40.206925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"select the input image - use a copy of the content image.","metadata":{}},{"cell_type":"code","source":"input_img = content_img.clone()\n\nplt.figure(figsize=((8, 8)))\nimshow(input_img, title='Input Image')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:40.209763Z","iopub.execute_input":"2022-08-28T12:25:40.210139Z","iopub.status.idle":"2022-08-28T12:25:40.572548Z","shell.execute_reply.started":"2022-08-28T12:25:40.210100Z","shell.execute_reply":"2022-08-28T12:25:40.571559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Gradient Descent\n\n Using L-BFGS algorithm to run our gradient descent. Unlike training a network, we want to train the input image in order to minimise the content/style losses. ","metadata":{}},{"cell_type":"code","source":"# creatting a PyTorch L-BFGS optimizer optim.LBFGS and pass our image to it as the tensor to optimize\ndef get_input_optimizer(input_img):\n    optimizer = optim.LBFGS([input_img])   # input is a parameter that requires a gradient\n    return optimizer\n\n# function that performs the neural transfer. For each iteration of the networks, it is fed an updated input and computes new losses.\n\ndef run_style_transfer(cnn, normalization_mean, normalization_std,\n                       content_img, style_img, input_img, num_steps=300,\n                       style_weight=1000000, content_weight=1):\n    print('Building the style transfer model..')\n    model, style_losses, content_losses = get_style_model_and_losses(cnn,\n        normalization_mean, normalization_std, style_img, content_img)\n\n    # We want to optimize the input and not the model parameters so we update all the requires_grad fields accordingly\n    input_img.requires_grad_(True)\n    model.requires_grad_(False)\n\n    optimizer = get_input_optimizer(input_img)\n\n    print('Optimizing..')\n    run = [0]\n    while run[0] <= num_steps:\n\n        def closure():\n            # correct the values of updated input image\n            with torch.no_grad():\n                input_img.clamp_(0, 1)\n\n            optimizer.zero_grad()\n            model(input_img)\n            style_score = 0\n            content_score = 0\n\n            for sl in style_losses:\n                style_score += sl.loss\n            for cl in content_losses:\n                content_score += cl.loss\n\n            style_score *= style_weight\n            content_score *= content_weight\n\n            loss = style_score + content_score\n            loss.backward()\n\n            run[0] += 1\n            if run[0] % 50 == 0:\n                print(\"run {}:\".format(run))\n                print('Style Loss : {:4f} Content Loss: {:4f}'.format(\n                    style_score.item(), content_score.item()))\n                print()\n\n            return style_score + content_score\n\n        optimizer.step(closure)\n\n    # a last correction...\n    with torch.no_grad():\n        input_img.clamp_(0, 1)\n\n    return input_img","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:40.574230Z","iopub.execute_input":"2022-08-28T12:25:40.575018Z","iopub.status.idle":"2022-08-28T12:25:40.586744Z","shell.execute_reply.started":"2022-08-28T12:25:40.574977Z","shell.execute_reply":"2022-08-28T12:25:40.585568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"run the algorithm:","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\noutput = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std,\n                            content_img, style_img, input_img)\n\nprint(\"{:.2f} seconds\".format(time.time() - start_time))\n\nplt.figure(figsize=((8, 8)))\nimshow(output, title='Output Image')\n\n# sphinx_gallery_thumbnail_number = 4\nplt.ioff()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:40.589484Z","iopub.execute_input":"2022-08-28T12:25:40.589916Z","iopub.status.idle":"2022-08-28T12:25:45.912964Z","shell.execute_reply.started":"2022-08-28T12:25:40.589879Z","shell.execute_reply":"2022-08-28T12:25:45.911788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Displaying the outputs & hyperparameter Tuning","metadata":{}},{"cell_type":"code","source":"! mkdir ../images_display","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:45.914758Z","iopub.execute_input":"2022-08-28T12:25:45.915195Z","iopub.status.idle":"2022-08-28T12:25:46.977551Z","shell.execute_reply.started":"2022-08-28T12:25:45.915136Z","shell.execute_reply":"2022-08-28T12:25:46.976201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"photo_directory = \"../input/gan-getting-started/photo_jpg\"\nmonet_directory = \"../input/gan-getting-started/monet_jpg\"\nmonet_list = os.listdir(monet_directory)\n\nstart_time = time.time()\n\ni = 1\nfor filename in os.listdir(photo_directory):\n    \n    content_img = image_loader(os.path.join(photo_directory, filename))\n    input_img = content_img.clone()\n    # getting random monet image as the style\n    rand_filename = random.choice(monet_list)\n    style_img = image_loader(os.path.join(monet_directory, rand_filename))\n    \n    output = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std,\n                            content_img, style_img, input_img)\n    \n    transform = transforms.ToPILImage()\n    img = transform(output.view(3, 256, 256))\n    img.save(\"../images_display/\" + str(i) + \".jpg\")\n    \n    i += 1\n    \n    print(\"   img: {} --- {:.2f} seconds ---\\n\".format(i, time.time() - start_time))\n    \n    if i==17:\n        break\n","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:25:46.982093Z","iopub.execute_input":"2022-08-28T12:25:46.982435Z","iopub.status.idle":"2022-08-28T12:27:08.971483Z","shell.execute_reply.started":"2022-08-28T12:25:46.982403Z","shell.execute_reply":"2022-08-28T12:27:08.970374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(img_dir):\n    all_img_dirs = os.listdir(img_dir)\n    img_files = [os.path.join(img_dir, file) for file in all_img_dirs]\n  \n    plt.figure(figsize=(20, 20))\n  \n    for idx, img_path in enumerate(img_files):\n        plt.subplot(4, 4, idx+1)\n    \n        img = plt.imread(img_path)\n        plt.tight_layout()         \n        plt.imshow(img, cmap='gray')\n        plt.axis('off')\n        \nplot_images('../images_display')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:27:08.972769Z","iopub.execute_input":"2022-08-28T12:27:08.973914Z","iopub.status.idle":"2022-08-28T12:27:12.024717Z","shell.execute_reply.started":"2022-08-28T12:27:08.973871Z","shell.execute_reply":"2022-08-28T12:27:12.022237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try tilting the model a little less towards the style image to get a little more realistic images.\nI will do this by changing the style-weight from 1,000,000 to 100,000, and the number of steps from 300 to 200.","metadata":{}},{"cell_type":"code","source":"! mkdir ../images_display_200_100000","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:27:12.026182Z","iopub.execute_input":"2022-08-28T12:27:12.026753Z","iopub.status.idle":"2022-08-28T12:27:13.100834Z","shell.execute_reply.started":"2022-08-28T12:27:12.026719Z","shell.execute_reply":"2022-08-28T12:27:13.099700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\n\ni = 1\nfor filename in os.listdir(photo_directory):\n    \n    content_img = image_loader(os.path.join(photo_directory, filename))\n    input_img = content_img.clone()\n    # getting random monet image as the style\n    rand_filename = random.choice(monet_list)\n    style_img = image_loader(os.path.join(monet_directory, rand_filename))\n    \n    output = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std,\n                            content_img, style_img, input_img, num_steps=200, style_weight=100000)\n    \n    transform = transforms.ToPILImage()\n    img = transform(output.view(3, 256, 256))\n    img.save(\"../images_display_200_100000/\" + str(i) + \".jpg\")\n    \n    i += 1\n    \n    print(\"   img: {} --- {:.2f} seconds ---\\n\".format(i, time.time() - start_time))\n    \n    if i==17:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:27:13.102734Z","iopub.execute_input":"2022-08-28T12:27:13.105484Z","iopub.status.idle":"2022-08-28T12:28:07.516617Z","shell.execute_reply.started":"2022-08-28T12:27:13.105431Z","shell.execute_reply":"2022-08-28T12:28:07.515523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_images('../images_display_200_100000')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:28:07.517977Z","iopub.execute_input":"2022-08-28T12:28:07.518669Z","iopub.status.idle":"2022-08-28T12:28:09.986811Z","shell.execute_reply.started":"2022-08-28T12:28:07.518626Z","shell.execute_reply":"2022-08-28T12:28:09.984863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create submission file","metadata":{}},{"cell_type":"code","source":"! mkdir ../images","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\n\ni = 1\nfor filename in os.listdir(photo_directory):\n    \n    content_img = image_loader(os.path.join(photo_directory, filename))\n    input_img = content_img.clone()\n    # getting random monet image as the style\n    rand_filename = random.choice(monet_list)\n    style_img = image_loader(os.path.join(monet_directory, rand_filename))\n    \n    output = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std,\n                            content_img, style_img, input_img, num_steps=200, style_weight=100000)\n    \n    transform = transforms.ToPILImage()\n    img = transform(output.view(3, 256, 256))\n    img.save(\"../images/\" + str(i) + \".jpg\")\n    \n    i += 1\n    \n    print(\"   img: {} --- {:.2f} seconds ---\\n\".format(i, time.time() - start_time))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive(\"/kaggle/working/images\", 'zip', \"/kaggle/images\")","metadata":{},"execution_count":null,"outputs":[]}]}