{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21755,"databundleVersionId":1475600,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Style Transfer with PyTorch 🖼️\n\n

\nStyle Transfer involves copying the style of one image (the Style Image) and applying it to another image (the Content Image). The outcome is a new image (the Target Image) that combines the content of the Content Image with the style of the Style Image.

\n\n

Objective 🎯

\n\n\n\n

By balancing these losses, we aim to produce a Target Image that effectively integrates the artistic elements of the Style Image with the recognizable features of the Content Image.

\n\n

Implementation Overview 📝

\n
    \n
  1. Import Libraries
    2. \n
  2. Load Images Paths
    3. \n
  3. Preprocessing the Images
    4. \n
  4. Define the Pretrained Model
    5. \n
  5. Style Transfer Loop
    6. \n
\n\n

If you found this notebook helpful or simply enjoyed it, I'd greatly appreciate some upvotes! Your support keeps me motivated.

","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\n\nimport torch as T\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import models\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:03.138814Z","iopub.execute_input":"2024-05-22T04:20:03.139690Z","iopub.status.idle":"2024-05-22T04:20:06.695807Z","shell.execute_reply.started":"2024-05-22T04:20:03.139655Z","shell.execute_reply":"2024-05-22T04:20:06.694803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GPU\ndevice = T.device(\"cuda\" if T.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:06.697483Z","iopub.execute_input":"2024-05-22T04:20:06.698085Z","iopub.status.idle":"2024-05-22T04:20:06.718636Z","shell.execute_reply.started":"2024-05-22T04:20:06.698050Z","shell.execute_reply":"2024-05-22T04:20:06.717656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Image Paths into a DataFrame","metadata":{}},{"cell_type":"markdown","source":"## Loading Monet Images","metadata":{}},{"cell_type":"code","source":"monet_df = []\nfor root,_, files in os.walk('/kaggle/input/gan-getting-started/monet_jpg'):\n for filename in files:\n path = os.path.join(root+'/'+filename)\n monet_df.append(path)\n\nmonet_df = pd.DataFrame(monet_df, columns=['filename'])","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:07.345714Z","iopub.execute_input":"2024-05-22T04:20:07.346097Z","iopub.status.idle":"2024-05-22T04:20:07.354078Z","shell.execute_reply.started":"2024-05-22T04:20:07.346067Z","shell.execute_reply":"2024-05-22T04:20:07.352696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of Monet Images: {monet_df.shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:09.515348Z","iopub.execute_input":"2024-05-22T04:20:09.516177Z","iopub.status.idle":"2024-05-22T04:20:09.521139Z","shell.execute_reply.started":"2024-05-22T04:20:09.516142Z","shell.execute_reply":"2024-05-22T04:20:09.520155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Content Images","metadata":{}},{"cell_type":"code","source":"photo_df = []\nfor root,_, files in os.walk('/kaggle/input/gan-getting-started/photo_jpg'):\n for filename in files:\n path = os.path.join(root+'/'+filename) \n photo_df.append(path)\n\nphoto_df = pd.DataFrame(photo_df, columns=['filename'])","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:10.945423Z","iopub.execute_input":"2024-05-22T04:20:10.945782Z","iopub.status.idle":"2024-05-22T04:20:15.754811Z","shell.execute_reply.started":"2024-05-22T04:20:10.945753Z","shell.execute_reply":"2024-05-22T04:20:15.753988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of Content Images: {photo_df.shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:15.756561Z","iopub.execute_input":"2024-05-22T04:20:15.756852Z","iopub.status.idle":"2024-05-22T04:20:15.761420Z","shell.execute_reply.started":"2024-05-22T04:20:15.756828Z","shell.execute_reply":"2024-05-22T04:20:15.760521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sampling the content image dataframe\nphoto_df = photo_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:15.762548Z","iopub.execute_input":"2024-05-22T04:20:15.762867Z","iopub.status.idle":"2024-05-22T04:20:15.773418Z","shell.execute_reply.started":"2024-05-22T04:20:15.762838Z","shell.execute_reply":"2024-05-22T04:20:15.772504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformations","metadata":{}},{"cell_type":"markdown","source":"

The CNN requires images to be normalized and converted into tensor format

","metadata":{}},{"cell_type":"code","source":"mean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:16.109079Z","iopub.execute_input":"2024-05-22T04:20:16.109372Z","iopub.status.idle":"2024-05-22T04:20:16.113635Z","shell.execute_reply.started":"2024-05-22T04:20:16.109350Z","shell.execute_reply":"2024-05-22T04:20:16.112733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformations = transforms.Compose([\n transforms.Resize((256,256)), # Resize to a (256,256) image\n transforms.ToTensor(), # Convert to a Tensor\n transforms.Normalize(mean, std) # Normalize\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:17.437476Z","iopub.execute_input":"2024-05-22T04:20:17.438148Z","iopub.status.idle":"2024-05-22T04:20:17.443306Z","shell.execute_reply.started":"2024-05-22T04:20:17.438111Z","shell.execute_reply":"2024-05-22T04:20:17.442197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrained model","metadata":{}},{"cell_type":"code","source":"# Loading pretrained model\nvgg = models.vgg19(pretrained=True).features\n\n# Freeze layers\nfor params in vgg.parameters():\n params.requires_grad_(False)\n \nvgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:19.466269Z","iopub.execute_input":"2024-05-22T04:20:19.466661Z","iopub.status.idle":"2024-05-22T04:20:21.425305Z","shell.execute_reply.started":"2024-05-22T04:20:19.466632Z","shell.execute_reply":"2024-05-22T04:20:21.424267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Feature Maps and Layers

\n\n

In convolutional neural networks, earlier layers capture fine details, while higher layers capture more abstract features. When processing an image, each layer produces a feature map, which is a representation of the input image containing spatial and structural information. To use these feature maps for style transfer, we can extract features from specific layers and perform further processing to obtain the style information.

","metadata":{}},{"cell_type":"markdown","source":"

Gram Matrix

\n\n

To capture the style from the Style Image, we utilize the Gram matrix of the feature maps. The Gram matrix signifies the correlations among different channels in the feature map, encoding style information while discarding spatial details.

","metadata":{}},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"class Preprocessing():\n \n def __init__(self, weights, layers):\n # layer importance\n self.weights = weights\n # layer of interest\n self.layers_of_interest = layers\n \n def apply_model_and_extract_features(self,image, model):\n \n x = image\n \n features = {}\n \n for name, layer in model._modules.items():\n # Passing the image into the pretrained model\n x = layer(x)\n # Extract features from layers of interest\n if name in LAYERS_OF_INTEREST:\n features[LAYERS_OF_INTEREST[name]] = x\n\n return features\n \n def calculate_gram_matrix(self,tensor_image):\n \"\"\"\n Compute the Gram Matrix \n \n \"\"\"\n _, channels, height, width = tensor_image.size()\n\n tensor_image = tensor_image.view(channels, height*width)\n\n gram_matrix = T.mm(tensor_image, tensor_image.t())\n\n gram_matrix = gram_matrix.div(channels * height * width)\n\n return gram_matrix\n \n def tensor_to_image(self,image_tensor):\n \"\"\"\n Convert a Tensor Image to an Image\n \"\"\"\n image = image_tensor.clone().detach()\n image = image.cpu().numpy().squeeze()\n \n # Transpose (C, W, H) -> (W, H, C)\n image = image.transpose(1,2,0)\n # Desnormalize\n image *= np.array(std) + np.array(mean)\n # Convert to 0-255 scale\n image[:,:,0] = image[:,:,0] * 255\n image[:,:,1] = image[:,:,1] * 255\n image[:,:,2] = image[:,:,2] * 255\n image = np.clip(image, 0, 255).astype(np.uint8)\n \n return image\n \n ","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:22.223671Z","iopub.execute_input":"2024-05-22T04:20:22.224552Z","iopub.status.idle":"2024-05-22T04:20:22.235845Z","shell.execute_reply.started":"2024-05-22T04:20:22.224518Z","shell.execute_reply":"2024-05-22T04:20:22.234634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Perform Style Transfer","metadata":{}},{"cell_type":"markdown","source":"

Content Extraction

\n

To extract content information from the content image, we utilize the convolutional layer 'conv4_2'. This layer captures intermediate-level features that strike a balance between fine details and abstract representations.

\n\n

Style Extraction

\n

To extract style features, we compute Gram matrices from several layers of interest ('conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', and 'conv5_1'). These layers capture varying levels of style information, ranging from fine details to abstract patterns.

\n\n

Layer Importance

\n\n

The weights assigned to each style layer determine the significance of the style information extracted from that layer, with earlier layers usually receiving higher weights to highlight finer details.

","metadata":{}},{"cell_type":"code","source":"# Layer importance\nweights = {'conv1_1': 1.0, 'conv2_1': 0.75, 'conv3_1': 0.35,\n 'conv4_1': 0.25, 'conv5_1': 0.15}\n\n# Layer of interest\nLAYERS_OF_INTEREST = {\n '0': 'conv1_1',\n '5': 'conv2_1',\n '10': 'conv3_1',\n '19': 'conv4_1',\n '21': 'conv4_2',\n '28': 'conv5_1' \n}\n\npreprocessing = Preprocessing(weights, LAYERS_OF_INTEREST)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:25.153820Z","iopub.execute_input":"2024-05-22T04:20:25.154225Z","iopub.status.idle":"2024-05-22T04:20:25.159882Z","shell.execute_reply.started":"2024-05-22T04:20:25.154196Z","shell.execute_reply":"2024-05-22T04:20:25.158957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Output directory of the final images\noutput_dir = \"/kaggle/working/images/\"\nmonet_idx_list = []\n\nfor i, img_path in enumerate(photo_df.iloc[:,0]):\n \n print(f\"Image number: {i+1}\")\n # Load the image\n img = Image.open(img_path) \n # Apply transformations\n content_image = transformations(img).to(device)\n content_image = content_image.unsqueeze(0)\n \n # Select a random index \n monet_idx = np.random.randint(0,300, size=1)\n monet_idx_list.append(monet_idx)\n # Load the monet image\n monet_img = Image.open(monet_df.iloc[monet_idx[0],0])\n # Apply transformations\n style_image = transformations(monet_img).to(device)\n style_image = style_image.unsqueeze(0)\n \n # Copy the content of content image into the final image (target)\n target = content_image.clone().requires_grad_(True).to(device)\n \n # Optimizer\n optimizer = optim.Adam([target], lr=0.003)\n \n # Loop of performing style transfer\n for iteration in range(1, 1500): \n # Extracting features from content image, style image, and final image\n content_img_features = preprocessing.apply_model_and_extract_features(content_image, vgg)\n style_img_features = preprocessing.apply_model_and_extract_features(style_image, vgg) \n target_features = preprocessing.apply_model_and_extract_features(target, vgg)\n \n # Compute the content loss\n content_loss = F.mse_loss(target_features[\"conv4_2\"], content_img_features[\"conv4_2\"])\n\n style_loss = 0\n # Calculate the Gram Matrices of style image\n style_features_gram_matrix = {layer: preprocessing.calculate_gram_matrix(style_img_features[layer]) for layer in style_img_features}\n\n for layer in weights:\n \n target_feature = target_features[layer]\n # Calculate the Gram Matrix of final image\n target_gram_matrix = preprocessing.calculate_gram_matrix(target_feature)\n style_gram_matrix = style_features_gram_matrix[layer]\n \n # Compute the layer loss\n layer_loss = F.mse_loss(target_gram_matrix, style_gram_matrix)\n # Compute weighted layer loss\n layer_loss *= weights[layer]\n\n _, channels, height, width = target_feature.shape\n # Add the layer loss to style loss\n style_loss += layer_loss\n \n # Compute total loss\n total_loss = 1000000 * style_loss + content_loss\n\n optimizer.zero_grad()\n\n total_loss.backward()\n\n optimizer.step()\n \n # Convert final image to an array\n target_img = preprocessing.tensor_to_image(target)\n \n # Save the final image\n target_img = Image.fromarray(target_img)\n \n if not os.path.exists(output_dir):\n os.makedirs(output_dir)\n target_img.save(\"/kaggle/working/images/\" + str(i) + \".jpg\") \n ","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:20:26.643657Z","iopub.execute_input":"2024-05-22T04:20:26.644112Z","iopub.status.idle":"2024-05-22T04:23:30.099970Z","shell.execute_reply.started":"2024-05-22T04:20:26.644076Z","shell.execute_reply":"2024-05-22T04:23:30.099118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display Final Results","metadata":{}},{"cell_type":"code","source":"for i, content_path in zip(range(5), photo_df.iloc[:,0]):\n fig, (ax1, ax2, ax3) = plt.subplots(1,3, figsize=(20,10))\n \n content_image = Image.open(content_path) \n monet_image = Image.open(monet_df.iloc[monet_idx_list[i][0],0])\n final_image = Image.open(f\"/kaggle/working/images/{i}.jpg\")\n ax1.imshow(content_image)\n ax2.imshow(monet_image)\n ax3.imshow(final_image)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:23:30.101598Z","iopub.execute_input":"2024-05-22T04:23:30.101887Z","iopub.status.idle":"2024-05-22T04:23:35.060639Z","shell.execute_reply.started":"2024-05-22T04:23:30.101863Z","shell.execute_reply":"2024-05-22T04:23:35.059634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Disadvantages🚨

\n","metadata":{}}]}