{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Drawing Monet --- Generate a Monet-style painting.\n\nHow do I generate a fake Monet?\nThis competition suggests GAN, but it is not the only way for forgers to do this.\nHere we will use a somewhat older method - the ' Neural Algorithm of Artistic Style' to create a forgery.\n\nThis method takes the features of the image from the VGG and propagates them back to the input image itself (rather than the parameters in the neural network). This (of course) changes the input image.\nIn the end, the input image is a hybrid of the style image and the original image.\n\nFor more information, please refer to the following original paper\n[https://arxiv.org/abs/1508.06576](https://arxiv.org/abs/1508.06576)","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn, utils, optim\nimport torchvision as tv\nimport os\nimport io\nimport zipfile\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\nimport imageio\nimport numpy as np\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-05T03:10:32.394904Z","iopub.execute_input":"2021-06-05T03:10:32.395433Z","iopub.status.idle":"2021-06-05T03:10:34.552814Z","shell.execute_reply.started":"2021-06-05T03:10:32.395353Z","shell.execute_reply":"2021-06-05T03:10:34.551987Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_tensor = tv.transforms.Compose([\n                tv.transforms.Resize((256,256)),\n                tv.transforms.ToTensor(),\n                tv.transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                    std=[1, 1, 1]),\n            ])\n\nunload = tv.transforms.Compose([\n                tv.transforms.Normalize(mean=[-0.485,-0.456,-0.406],\n                                    std=[1,1,1]),                \n                tv.transforms.Lambda(lambda x: x.clamp(0,1))\n            ])\nto_image = tv.transforms.ToPILImage()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:10:34.556044Z","iopub.execute_input":"2021-06-05T03:10:34.556305Z","iopub.status.idle":"2021-06-05T03:10:34.56206Z","shell.execute_reply.started":"2021-06-05T03:10:34.556279Z","shell.execute_reply":"2021-06-05T03:10:34.561294Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset:\n    def __init__(self, style=False, target='../input/gan-getting-started/photo_jpg'):\n        self.style = style\n        self.target = target\n        self.x = os.listdir(self.target)\n        if style:\n            self.y = os.listdir('../input/gan-getting-started/monet_jpg')\n\n    def __len__(self):\n        return min(7000, len(self.x))\n\n    def __getitem__(self, pos):\n        input_img = self.target + '/' + self.x[pos]\n        input_img = Image.open(input_img)\n        input_img = to_tensor(input_img)\n        if not self.style:\n            return input_img\n        style_img = '../input/gan-getting-started/monet_jpg/' + self.y[style_choice[pos]]\n        style_img = Image.open(style_img)\n        style_img = to_tensor(style_img)\n        return input_img, style_img","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:10:34.56391Z","iopub.execute_input":"2021-06-05T03:10:34.564456Z","iopub.status.idle":"2021-06-05T03:10:34.575Z","shell.execute_reply.started":"2021-06-05T03:10:34.564422Z","shell.execute_reply":"2021-06-05T03:10:34.574247Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Obtain image features from VGG19\n\nThis step has no relation with the Neural Algorithm of Artistic Style yet. We will try to use VGG for image vectorization.","metadata":{}},{"cell_type":"code","source":"VGG = tv.models.vgg19(pretrained=False)\nVGG.load_state_dict(torch.load('../input/vgg19dcbb9e9dpth/vgg19-dcbb9e9d.pth'))\nVGG.cuda()\nVGG.classifier = VGG.classifier[:4]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:10:34.57649Z","iopub.execute_input":"2021-06-05T03:10:34.576972Z","iopub.status.idle":"2021-06-05T03:10:46.93844Z","shell.execute_reply.started":"2021-06-05T03:10:34.576937Z","shell.execute_reply":"2021-06-05T03:10:46.934674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find the Monet that is simillar to the original photos.\n\nThe step select the style image.\n\nIt is not necessary if you want to select the style image randomly, but here we select the Monet that most simillar to original photo as the style image in order to make a better forgery.","metadata":{}},{"cell_type":"code","source":"VGG.eval()\nphoto_vecs = []\ndataset = MyDataset(False)\ndata_loader = utils.data.DataLoader(\n    dataset, batch_size=64, shuffle=False, num_workers=4)\nfor x in tqdm(data_loader):\n    f = VGG(x.cuda())\n    f.detach().cpu().numpy()\n    photo_vecs.extend(f.tolist())\nmonet_vecs = []\ndataset = MyDataset(False, target='../input/gan-getting-started/monet_jpg')\ndata_loader = utils.data.DataLoader(\n    dataset, batch_size=64, shuffle=False, num_workers=4)\nfor x in tqdm(data_loader):\n    f = VGG(x.cuda())\n    f.detach().cpu().numpy()\n    monet_vecs.extend(f.tolist())\nstyle_choice = cosine_similarity(photo_vecs, monet_vecs)\nstyle_choice = np.argmax(style_choice, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:10:46.942931Z","iopub.execute_input":"2021-06-05T03:10:46.943435Z","iopub.status.idle":"2021-06-05T03:11:52.839677Z","shell.execute_reply.started":"2021-06-05T03:10:46.943395Z","shell.execute_reply":"2021-06-05T03:11:52.838585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# VGG features\n\nUnique about Leon's work is that what part of the VGG features includes the style or shape.","metadata":{}},{"cell_type":"code","source":"def get_features(module, x, y):\n    features.append(y)\n    \ndef gram_matrix(x):\n    b, c, h, w = x.size()\n    F = x.view(b,c,h*w)\n    G = torch.bmm(F, F.transpose(1,2))/(h*w)\n    return G","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:11:52.84477Z","iopub.execute_input":"2021-06-05T03:11:52.847082Z","iopub.status.idle":"2021-06-05T03:11:52.855635Z","shell.execute_reply.started":"2021-06-05T03:11:52.847032Z","shell.execute_reply":"2021-06-05T03:11:52.854819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VGG = VGG.features\nVGG.eval()\n\nfeatures = []\n\nfor i, layer in enumerate(VGG):\n    \n    if i in [0,5,10,19,21,28]:\n        VGG[i].register_forward_hook(get_features)\n    \n    elif isinstance(layer, nn.MaxPool2d):\n        VGG[i] = nn.AvgPool2d(kernel_size=2)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:11:52.86107Z","iopub.execute_input":"2021-06-05T03:11:52.864054Z","iopub.status.idle":"2021-06-05T03:11:52.873329Z","shell.execute_reply.started":"2021-06-05T03:11:52.864011Z","shell.execute_reply":"2021-06-05T03:11:52.872418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Fake Monet.\n\nMix and back-propagate shape features and style features.","metadata":{}},{"cell_type":"code","source":"def make_monet(input_imgs, style_imgs):\n    global features\n    \n    for p in VGG.parameters():\n        p.requires_grad = False\n\n    features = []\n    VGG(input_imgs)\n    c_target = features[4].detach()\n\n    features = []\n    VGG(style_imgs)\n    f_targets = features[:4]+features[5:]\n    gram_targets = [gram_matrix(i).detach() for i in f_targets]\n    \n    alpha = 1\n    beta = 1e3\n    iterations = 5\n    image = input_imgs.clone()\n    optimizer = optim.LBFGS([image.requires_grad_()], lr=1)    \n    mse_loss = nn.MSELoss(reduction='mean')\n    l_c = []\n    l_s = []\n    counter = 0\n    \n    for itr in range(iterations):\n\n        features = []\n        def closure():\n            optimizer.zero_grad()\n            VGG(image)\n            t_features = features[-6:]\n            content = t_features[4]\n            style_features = t_features[:4]+t_features[5:]\n            t_features = []\n            gram_styles = [gram_matrix(i) for i in style_features]\n            c_loss = alpha * mse_loss(content, c_target)\n            s_loss = 0\n\n            for i in range(5):\n                n_c = gram_styles[i].shape[0]\n                s_loss += beta * mse_loss(gram_styles[i],gram_targets[i])/(n_c**2)\n\n            total_loss = c_loss+s_loss\n\n            l_c.append(c_loss)\n            l_s.append(s_loss)\n\n            total_loss.backward()\n            return total_loss\n\n        optimizer.step(closure)\n    \n    result = []\n    for i in range(len(X)):\n        temp = unload(image[i].cpu().detach())\n        temp = to_image(temp)\n        temp = np.array(temp)\n        result.append(temp)\n    return result","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:11:52.880212Z","iopub.execute_input":"2021-06-05T03:11:52.882482Z","iopub.status.idle":"2021-06-05T03:11:52.911554Z","shell.execute_reply.started":"2021-06-05T03:11:52.882441Z","shell.execute_reply":"2021-06-05T03:11:52.910632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Samples\n\nVisualize 4 sample fake Monets.","metadata":{}},{"cell_type":"code","source":"dataset = MyDataset(True)\ndata_loader = utils.data.DataLoader(\n    dataset, batch_size=4, shuffle=False, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:11:52.917349Z","iopub.execute_input":"2021-06-05T03:11:52.920391Z","iopub.status.idle":"2021-06-05T03:11:52.935382Z","shell.execute_reply.started":"2021-06-05T03:11:52.920349Z","shell.execute_reply":"2021-06-05T03:11:52.934571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for X, y in data_loader:\n    i = make_monet(X.cuda(), y.cuda())\n    break\nplt.imshow(i[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:11:52.939489Z","iopub.execute_input":"2021-06-05T03:11:52.941631Z","iopub.status.idle":"2021-06-05T03:12:05.009986Z","shell.execute_reply.started":"2021-06-05T03:11:52.94159Z","shell.execute_reply":"2021-06-05T03:12:05.009078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(i[1])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:12:05.011206Z","iopub.execute_input":"2021-06-05T03:12:05.011559Z","iopub.status.idle":"2021-06-05T03:12:05.15616Z","shell.execute_reply.started":"2021-06-05T03:12:05.01152Z","shell.execute_reply":"2021-06-05T03:12:05.155268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(i[2])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:12:05.157443Z","iopub.execute_input":"2021-06-05T03:12:05.157777Z","iopub.status.idle":"2021-06-05T03:12:05.301499Z","shell.execute_reply.started":"2021-06-05T03:12:05.157742Z","shell.execute_reply":"2021-06-05T03:12:05.300618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(i[3])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:12:05.302747Z","iopub.execute_input":"2021-06-05T03:12:05.303085Z","iopub.status.idle":"2021-06-05T03:12:05.439546Z","shell.execute_reply.started":"2021-06-05T03:12:05.303051Z","shell.execute_reply":"2021-06-05T03:12:05.438781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Submission\n\nThis task takes a long time - because it is generated by back propagation.\n\nNormaly, Pre-trained StyleGAN are better because they are faster to generate. However, this method also has the advantage of not requiring any training.\n\nProbably, the number of 7000 images specified by the Kaggle team is considering that such a method will not work within the time limit of the kernel.\n\nHowever, we clear this by limiting the number of back propagations to 5. If we increase the number more, the quality will improve, but it will be trapped by the competition time limit.","metadata":{}},{"cell_type":"code","source":"dataset = MyDataset(True)\ndata_loader = utils.data.DataLoader(\n    dataset, batch_size=8, shuffle=False, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:12:05.440569Z","iopub.execute_input":"2021-06-05T03:12:05.44098Z","iopub.status.idle":"2021-06-05T03:12:05.452877Z","shell.execute_reply.started":"2021-06-05T03:12:05.440948Z","shell.execute_reply":"2021-06-05T03:12:05.452179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_count = 0\nwith zipfile.ZipFile(\"images.zip\", \"w\", zipfile.ZIP_DEFLATED, False) as zip_file:\n    for X, y in tqdm(data_loader):\n        imgs = make_monet(X.cuda(), y.cuda())\n        for img in imgs:\n            with io.BytesIO() as data:\n                Image.fromarray(img).save(data, format='PNG')\n                img_bytes = data.getvalue()\n            zip_file.writestr(f\"{write_count}.png\", img_bytes)\n            write_count += 1","metadata":{"execution":{"iopub.status.busy":"2021-06-05T03:12:05.45404Z","iopub.execute_input":"2021-06-05T03:12:05.454404Z","iopub.status.idle":"2021-06-05T03:15:23.450836Z","shell.execute_reply.started":"2021-06-05T03:12:05.454368Z","shell.execute_reply":"2021-06-05T03:15:23.448979Z"},"trusted":true},"execution_count":null,"outputs":[]}]}