{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Style Transfer PyTorch Sample\nThis notebook referred to Udacity Deep Learning program.<br/>\nhttps://github.com/udacity/deep-learning-v2-pytorch","metadata":{"_uuid":"c48ae0ab26f1188fd450208d348e0b7ff8ba5b28"}},{"cell_type":"code","source":"%matplotlib inline\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms, models","metadata":{"id":"oVlHGJYMbP5N","_uuid":"7b1206e82683c9b6d0f269971def5655c3a1f7d8","execution":{"iopub.status.busy":"2021-11-02T15:29:56.917425Z","iopub.execute_input":"2021-11-02T15:29:56.918103Z","iopub.status.idle":"2021-11-02T15:29:56.925712Z","shell.execute_reply.started":"2021-11-02T15:29:56.918065Z","shell.execute_reply":"2021-11-02T15:29:56.925038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = models.vgg19(pretrained=True).features\nfor param in vgg.parameters():\n    param.requires_grad_(False)","metadata":{"id":"jVr5_jzvbP5c","_uuid":"f7710325ec38cfb299bfe8307ed5d87beaeedecc","execution":{"iopub.status.busy":"2021-11-02T15:29:56.927552Z","iopub.execute_input":"2021-11-02T15:29:56.928049Z","iopub.status.idle":"2021-11-02T15:29:58.466234Z","shell.execute_reply.started":"2021-11-02T15:29:56.928013Z","shell.execute_reply":"2021-11-02T15:29:58.465486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nvgg.to(device)","metadata":{"id":"jfKwUYt0bP6F","outputId":"84b031fd-5406-4ec9-d766-9c73803cf7bd","_uuid":"56714eecb4da53210dffc55d4aa1f7217512846d","execution":{"iopub.status.busy":"2021-11-02T15:29:58.46739Z","iopub.execute_input":"2021-11-02T15:29:58.467808Z","iopub.status.idle":"2021-11-02T15:29:58.498729Z","shell.execute_reply.started":"2021-11-02T15:29:58.467767Z","shell.execute_reply":"2021-11-02T15:29:58.497929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(img_path, max_size=400, shape=None):\n    image = Image.open(img_path).convert('RGB')\n    if max(image.size) > max_size:\n        size = max_size\n    else:\n        size = max(image.size)\n        \n    if shape is not None:\n        size = shape\n        \n    in_transform = transforms.Compose([\n                        transforms.Resize(size),\n                        transforms.ToTensor(),\n                        transforms.Normalize((0.485, 0.456, 0.406), \n                                             (0.229, 0.224, 0.225))])\n\n    image = in_transform(image)[:3,:,:].unsqueeze(0)\n    \n    return image","metadata":{"id":"v3Ycp8BobP6q","_uuid":"978cb5eac10a90aac2b2dd62ff3a7b92a373142d","execution":{"iopub.status.busy":"2021-11-02T15:29:58.50008Z","iopub.execute_input":"2021-11-02T15:29:58.500416Z","iopub.status.idle":"2021-11-02T15:29:58.508133Z","shell.execute_reply.started":"2021-11-02T15:29:58.500381Z","shell.execute_reply":"2021-11-02T15:29:58.507172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# content image and style image","metadata":{}},{"cell_type":"code","source":"# load in content and style image\ncontent = load_image('../input/gan-getting-started/photo_jpg/009ddaed1f.jpg').to(device)\n# Resize style to match content, makes code easier\nstyle = load_image('../input/gan-getting-started/monet_jpg/0e3b3292da.jpg', shape=content.shape[-2:]).to(device)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def im_convert(tensor):\n    image = tensor.to(\"cpu\").clone().detach()\n    image = image.numpy().squeeze()\n    image = image.transpose(1,2,0)\n    image = image * np.array((0.229, 0.224, 0.225)) + np.array((0.485, 0.456, 0.406))\n    image = image.clip(0, 1)\n\n    return image","metadata":{"id":"I33hNj9NbP7f","_uuid":"14c1030c91f66b174ba015f4cd72d8d4ead8ab2f","execution":{"iopub.status.busy":"2021-11-02T15:29:58.53541Z","iopub.execute_input":"2021-11-02T15:29:58.535593Z","iopub.status.idle":"2021-11-02T15:29:58.545163Z","shell.execute_reply.started":"2021-11-02T15:29:58.53557Z","shell.execute_reply":"2021-11-02T15:29:58.54347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))\nax1.imshow(im_convert(content))\nax1.set_title(\"Content Image\",fontsize = 12)\nax2.imshow(im_convert(style))\nax2.set_title(\"Style Image\", fontsize = 12)\nax1.set_xticks([])\nax1.set_yticks([])\nax2.set_xticks([])\nax2.set_yticks([])\nplt.show()","metadata":{"id":"uZSNV6glbP7n","outputId":"424d1549-87bc-4ed4-8ef1-2cabd3ff0ac6","_uuid":"d08805bd8fefe717f72f179c171f4d24db849553","execution":{"iopub.status.busy":"2021-11-02T15:29:58.549009Z","iopub.execute_input":"2021-11-02T15:29:58.549546Z","iopub.status.idle":"2021-11-02T15:29:58.743324Z","shell.execute_reply.started":"2021-11-02T15:29:58.549515Z","shell.execute_reply":"2021-11-02T15:29:58.742438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(image, model, layers=None):\n\n    if layers is None:\n        layers = {'0': 'conv1_1',\n                  '5': 'conv2_1', \n                  '10': 'conv3_1', \n                  '19': 'conv4_1',\n                  '21': 'conv4_2',\n                  '28': 'conv5_1'}\n        \n    features = {}\n    x = image\n\n    for name, layer in model._modules.items():\n        x = layer(x)\n        if name in layers:\n            features[layers[name]] = x\n            \n    return features","metadata":{"id":"FV5THOfzbP8j","_uuid":"2b46a9f4abf05d89bd7c60fe7951525bffd1640d","execution":{"iopub.status.busy":"2021-11-02T15:29:58.744707Z","iopub.execute_input":"2021-11-02T15:29:58.745054Z","iopub.status.idle":"2021-11-02T15:29:58.752744Z","shell.execute_reply.started":"2021-11-02T15:29:58.745013Z","shell.execute_reply":"2021-11-02T15:29:58.752142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gram_matrix(tensor):\n    _, d, h, w = tensor.size()\n    tensor = tensor.view(d, h * w)\n    gram = torch.mm(tensor, tensor.t())\n    \n    return gram ","metadata":{"id":"OQykPzj9bP8u","_uuid":"65e77ee27728fd35e8e1df9900b0623aa769a50e","execution":{"iopub.status.busy":"2021-11-02T15:29:58.754185Z","iopub.execute_input":"2021-11-02T15:29:58.754831Z","iopub.status.idle":"2021-11-02T15:29:58.762728Z","shell.execute_reply.started":"2021-11-02T15:29:58.75479Z","shell.execute_reply":"2021-11-02T15:29:58.761207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(content))\n\ncontent_features = get_features(content, vgg)\nstyle_features = get_features(style, vgg)\n\nprint(type(content_features))\n\nstyle_grams = {layer: gram_matrix(style_features[layer]) for layer in style_features}\ntarget = content.clone().requires_grad_(True).to(device)","metadata":{"id":"mVOziT76bP9e","_uuid":"ac7477f95eb629c63db6a8044599112bcd603423","execution":{"iopub.status.busy":"2021-11-02T15:30:51.908688Z","iopub.execute_input":"2021-11-02T15:30:51.909457Z","iopub.status.idle":"2021-11-02T15:30:51.947152Z","shell.execute_reply.started":"2021-11-02T15:30:51.909412Z","shell.execute_reply":"2021-11-02T15:30:51.945851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"style_weights = {'conv1_1': 1.,\n                 'conv2_1': 0.75,\n                 'conv3_1': 0.2,\n                 'conv4_1': 0.2,\n                 'conv5_1': 0.2}\n\ncontent_weight = 1  # alpha\nstyle_weight = 1e9  # beta","metadata":{"id":"T6xsdRMlbP9r","_uuid":"a3867b0dd38b95cb706fb0e416f4c9d7b7433f00","execution":{"iopub.status.busy":"2021-11-02T15:29:58.806575Z","iopub.status.idle":"2021-11-02T15:29:58.807449Z","shell.execute_reply.started":"2021-11-02T15:29:58.807152Z","shell.execute_reply":"2021-11-02T15:29:58.80718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_every = 400\noptimizer = optim.Adam([target], lr=0.003)\nsteps = 2000\n\nfor ii in range(1, steps+1):\n    \n    target_features = get_features(target, vgg)\n    content_loss = torch.mean((target_features['conv4_2'] - content_features['conv4_2'])**2)\n    style_loss = 0\n\n    for layer in style_weights:\n        target_feature = target_features[layer]\n        target_gram = gram_matrix(target_feature)\n        _, d, h, w = target_feature.shape\n        style_gram = style_grams[layer]\n        layer_style_loss = style_weights[layer] * torch.mean((target_gram - style_gram)**2)\n        style_loss += layer_style_loss / (d * h * w)\n        \n    total_loss = content_weight * content_loss + style_weight * style_loss\n    \n    optimizer.zero_grad()\n    total_loss.backward()\n    optimizer.step()\n    \n    if  ii % show_every == 0:     \n        print('Total loss: ', total_loss.item())\n        plt.imshow(im_convert(target))\n        plt.axis('off')\n        plt.show()","metadata":{"scrolled":true,"id":"fGgRrXjLbP94","outputId":"69e4b4d0-bff6-4884-bc4d-f76340b7565c","_uuid":"9bcc49bb9590c4a671eecaed7f359be604aa29f8","execution":{"iopub.status.busy":"2021-11-02T15:29:58.809444Z","iopub.status.idle":"2021-11-02T15:29:58.809964Z","shell.execute_reply.started":"2021-11-02T15:29:58.809701Z","shell.execute_reply":"2021-11-02T15:29:58.809728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stylized Image","metadata":{"id":"bbZVdQq_bP-1","_uuid":"3577b759c2054d0d312c004bcd2f46bfec3db14d"}},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(2,1, figsize=(5,10))\nax1.imshow(im_convert(content))\nax1.set_title(\"Content Image\", fontsize=12)\nax2.imshow(im_convert(target))\nax2.set_title(\"Stylized Image\", fontsize=12)\nax1.grid(False)\nax2.grid(False)\nax1.set_xticks([])\nax1.set_yticks([])\nax2.set_xticks([])\nax2.set_yticks([])\nplt.show()","metadata":{"id":"m5KD32b3bP-4","outputId":"e19e656f-5ac0-4a4f-95eb-42916fe98d11","_uuid":"800aba50ac20110585ba5ee02a5c8fc38976d112","execution":{"iopub.status.busy":"2021-11-02T15:29:58.811754Z","iopub.status.idle":"2021-11-02T15:29:58.812225Z","shell.execute_reply.started":"2021-11-02T15:29:58.811966Z","shell.execute_reply":"2021-11-02T15:29:58.811992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Style Transfer TensorFlow Hub Sample<br/>\nhttps://www.kaggle.com/stpeteishii/style-transfer-tensorflow-hub-sample<br/>\nStyle Transfer Tensorflow Sample<br/>\nhttps://www.kaggle.com/stpeteishii/style-transfer-tensorflow-sample<br/>\nStyle Transfer Keras Sample<br/>\nhttps://www.kaggle.com/stpeteishii/style-transfer-keras-sample<br/>\nStyle Transfer Pytorch Sample<br/>\nhttps://www.kaggle.com/stpeteishii/style-transfer-pytorch-sample<br/>","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}