{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install diffusers transformers","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:34:56.572112Z","iopub.execute_input":"2024-06-27T13:34:56.572950Z","iopub.status.idle":"2024-06-27T13:35:12.027247Z","shell.execute_reply.started":"2024-06-27T13:34:56.572914Z","shell.execute_reply":"2024-06-27T13:35:12.026308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn.functional as F\nimport torchvision\nfrom datasets import load_dataset\nfrom diffusers import DDIMScheduler, DDPMPipeline\nfrom matplotlib import pyplot as plt\nfrom PIL import Image\nfrom torchvision import transforms\nfrom tqdm.auto import tqdm\n\ndevice = \"mps\" if torch.backends.mps.is_available() else \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-27T13:36:19.231646Z","iopub.execute_input":"2024-06-27T13:36:19.232475Z","iopub.status.idle":"2024-06-27T13:36:20.305589Z","shell.execute_reply.started":"2024-06-27T13:36:19.232437Z","shell.execute_reply":"2024-06-27T13:36:20.304292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformers","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:36:08.855945Z","iopub.status.idle":"2024-06-27T13:36:08.856416Z","shell.execute_reply.started":"2024-06-27T13:36:08.856165Z","shell.execute_reply":"2024-06-27T13:36:08.856186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_pipe = DDPMPipeline.from_pretrained(\"google/ddpm-celebahq-256\")\nimage_pipe.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:30:34.150241Z","iopub.execute_input":"2024-06-27T12:30:34.151221Z","iopub.status.idle":"2024-06-27T12:30:37.027794Z","shell.execute_reply.started":"2024-06-27T12:30:34.151187Z","shell.execute_reply":"2024-06-27T12:30:37.026846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new scheduler and set num inference steps\nscheduler = DDIMScheduler.from_pretrained(\"google/ddpm-celebahq-256\")\nscheduler.set_timesteps(num_inference_steps=40)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:30:42.603910Z","iopub.execute_input":"2024-06-27T12:30:42.604476Z","iopub.status.idle":"2024-06-27T12:30:42.721778Z","shell.execute_reply.started":"2024-06-27T12:30:42.604442Z","shell.execute_reply":"2024-06-27T12:30:42.720859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Progressive Denoising","metadata":{}},{"cell_type":"code","source":"# The random starting point\nx = torch.randn(4, 3, 256, 256).to(device)  # Batch of 4, 3-channel 256 x 256 px images\n\n# Loop through the sampling timesteps\nfor i, t in tqdm(enumerate(scheduler.timesteps)):\n\n    # Prepare model input\n    model_input = scheduler.scale_model_input(x, t)\n\n    # Get the prediction\n    with torch.no_grad():\n        noise_pred = image_pipe.unet(model_input, t)[\"sample\"]\n\n    # Calculate what the updated sample should look like with the scheduler\n    scheduler_output = scheduler.step(noise_pred, t, x)\n\n    # Update x\n    x = scheduler_output.prev_sample\n\n    # Occasionally display both x and the predicted denoised images\n    if i % 10 == 0 or i == len(scheduler.timesteps) - 1:\n        fig, axs = plt.subplots(1, 2, figsize=(12, 5))\n\n        grid = torchvision.utils.make_grid(x, nrow=4).permute(1, 2, 0)\n        axs[0].imshow(grid.cpu().clip(-1, 1) * 0.5 + 0.5)\n        axs[0].set_title(f\"Current x (step {i})\")\n\n        pred_x0 = scheduler_output.pred_original_sample  # Not available for all schedulers\n        grid = torchvision.utils.make_grid(pred_x0, nrow=4).permute(1, 2, 0)\n        axs[1].imshow(grid.cpu().clip(-1, 1) * 0.5 + 0.5)\n        axs[1].set_title(f\"Predicted denoised images (step {i})\")\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:30:45.579316Z","iopub.execute_input":"2024-06-27T12:30:45.580141Z","iopub.status.idle":"2024-06-27T12:30:57.224840Z","shell.execute_reply.started":"2024-06-27T12:30:45.580109Z","shell.execute_reply":"2024-06-27T12:30:57.223914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fine Tuning - Further on a ","metadata":{}},{"cell_type":"code","source":"# dataset_name = \"huggan/anime-faces\"\ndataset_name = \"Norod78/Vintage-Faces-FFHQAligned\"\n# dataset_name = \"huggan/smithsonian_butterflies_subset\"  # @param\ndataset = load_dataset(dataset_name, split=\"train\")\nimage_size = 256  # @param\nbatch_size = 4  # @param\npreprocess = transforms.Compose(\n    [\n        transforms.Resize((image_size, image_size)),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5], [0.5]),\n    ]\n)\n\n\ndef transform(examples):\n    images = [preprocess(image.convert(\"RGB\")) for image in examples[\"image\"]]\n    return {\"images\": images}\n\n\ndataset.set_transform(transform)\n\ntrain_dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)\n\nprint(\"Previewing batch:\")\nbatch = next(iter(train_dataloader))\ngrid = torchvision.utils.make_grid(batch[\"images\"], nrow=4)\nplt.imshow(grid.permute(1, 2, 0).cpu().clip(-1, 1) * 0.5 + 0.5)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:48:55.837899Z","iopub.execute_input":"2024-06-27T12:48:55.838649Z","iopub.status.idle":"2024-06-27T12:49:01.163065Z","shell.execute_reply.started":"2024-06-27T12:48:55.838587Z","shell.execute_reply":"2024-06-27T12:49:01.162139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 10  # @param\nlr = 1e-5  # 2param\ngrad_accumulation_steps = 2  # @param\n\noptimizer = torch.optim.AdamW(image_pipe.unet.parameters(), lr=lr)\n\nlosses = []\n\nfor epoch in range(num_epochs):\n    for step, batch in tqdm(enumerate(train_dataloader), total=len(train_dataloader)):\n        clean_images = batch[\"images\"].to(device)\n        noise        = torch.randn(clean_images.shape).to(clean_images.device)\n        bs           = clean_images.shape[0]\n\n        # Sample a random timestep for each image\n        timesteps = torch.randint(0, image_pipe.scheduler.num_train_timesteps, (bs,), device=clean_images.device,).long()\n\n        # Add noise to the clean images according to the noise magnitude at each timestep\n        noisy_images = image_pipe.scheduler.add_noise(clean_images, noise, timesteps)\n\n        # Get the model prediction for the noise\n        noise_pred = image_pipe.unet(noisy_images, timesteps, return_dict=False)[0]\n\n        # Compare the prediction with the actual noise:\n        loss = F.mse_loss( noise_pred, noise)  # NB - trying to predict noise (eps) not (noisy_ims-clean_ims) or just (clean_ims)\n\n        # Store for later plotting\n        losses.append(loss.item())\n\n        # Update the model parameters with the optimizer based on this loss\n        loss.backward(loss)\n        if (step + 1) % grad_accumulation_steps == 0:         # Gradient accumulation:\n            optimizer.step()\n            optimizer.zero_grad()\n\n    print(f\"Epoch {epoch} average loss: {sum(losses[-len(train_dataloader):])/len(train_dataloader)}\")\n\n# Plot the loss curve:\nplt.plot(losses)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:49:10.709003Z","iopub.execute_input":"2024-06-27T12:49:10.709859Z","iopub.status.idle":"2024-06-27T12:56:40.445913Z","shell.execute_reply.started":"2024-06-27T12:49:10.709829Z","shell.execute_reply":"2024-06-27T12:56:40.444794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# @markdown Generate and plot some images:\nx = torch.randn(8, 3, 256, 256).to(device)  # Batch of 8\n\nfor i, t in tqdm(enumerate(scheduler.timesteps)):\n    model_input = scheduler.scale_model_input(x, t)\n    with torch.no_grad():\n        noise_pred = image_pipe.unet(model_input, t)[\"sample\"]\n    x = scheduler.step(noise_pred, t, x).prev_sample\ngrid = torchvision.utils.make_grid(x, nrow=4)\nplt.imshow(grid.permute(1, 2, 0).cpu().clip(-1, 1) * 0.5 + 0.5)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:08:37.382280Z","iopub.execute_input":"2024-06-27T13:08:37.382912Z","iopub.status.idle":"2024-06-27T13:08:54.151758Z","shell.execute_reply.started":"2024-06-27T13:08:37.382880Z","shell.execute_reply":"2024-06-27T13:08:54.150888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scheduler.set_timesteps(num_inference_steps=100)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:17:01.321413Z","iopub.execute_input":"2024-06-27T13:17:01.322143Z","iopub.status.idle":"2024-06-27T13:17:01.326857Z","shell.execute_reply.started":"2024-06-27T13:17:01.322111Z","shell.execute_reply":"2024-06-27T13:17:01.325442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# @markdown Generate and plot some images:\nx = torch.randn(8, 3, 256, 256).to(device)  # Batch of 8\n\nfor i, t in tqdm(enumerate(scheduler.timesteps)):\n    model_input = scheduler.scale_model_input(x, t)\n    with torch.no_grad():\n        noise_pred = image_pipe.unet(model_input, t)[\"sample\"]\n    x = scheduler.step(noise_pred, t, x).prev_sample\ngrid = torchvision.utils.make_grid(x, nrow=4)\nplt.imshow(grid.permute(1, 2, 0).cpu().clip(-1, 1) * 0.5 + 0.5)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:17:01.664666Z","iopub.execute_input":"2024-06-27T13:17:01.665258Z","iopub.status.idle":"2024-06-27T13:17:04.280677Z","shell.execute_reply.started":"2024-06-27T13:17:01.665229Z","shell.execute_reply":"2024-06-27T13:17:04.279358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Guidance","metadata":{}},{"cell_type":"code","source":"def color_loss(images, target_color=(0, 0.5, 0.5)):\n    \"\"\"Given a target color (R, G, B) return a loss for how far away on average\n    the images' pixels are from that color. Defaults to a light teal: (0.1, 0.9, 0.5)\"\"\"\n    target = torch.tensor(target_color).to(images.device) * 2 - 1  # Map target color to (-1, 1)\n    target = target[None, :, None, None]  # Get shape right to work with the images (b, c, h, w)\n    error = torch.abs(images - target).mean()  # Mean absolute difference between the image pixels and the target color\n    return error","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:21:50.954846Z","iopub.execute_input":"2024-06-27T13:21:50.955548Z","iopub.status.idle":"2024-06-27T13:21:50.961269Z","shell.execute_reply.started":"2024-06-27T13:21:50.955517Z","shell.execute_reply":"2024-06-27T13:21:50.960274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_pipe = DDPMPipeline.from_pretrained(\"google/ddpm-celebahq-256\")\nimage_pipe.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:18:31.132227Z","iopub.execute_input":"2024-06-27T13:18:31.132636Z","iopub.status.idle":"2024-06-27T13:18:31.941191Z","shell.execute_reply.started":"2024-06-27T13:18:31.132585Z","shell.execute_reply":"2024-06-27T13:18:31.940278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Variant 1: shortcut method\n\n# The guidance scale determines the strength of the effect\nguidance_loss_scale = 10  # Explore changing this to 5, or 100\n\nx = torch.randn(8, 3, 256, 256).to(device)\n\nfor i, t in tqdm(enumerate(scheduler.timesteps)):\n\n    # Prepare the model input\n    model_input = scheduler.scale_model_input(x, t)\n\n    # predict the noise residual\n    with torch.no_grad():\n        noise_pred = image_pipe.unet(model_input, t)[\"sample\"]\n\n    # Set x.requires_grad to True\n    x = x.detach().requires_grad_()\n\n    # Get the predicted x0\n    x0 = scheduler.step(noise_pred, t, x).pred_original_sample\n\n    # Calculate loss\n    loss = color_loss(x0) * guidance_loss_scale\n    if i % 10 == 0:\n        print(i, \"loss:\", loss.item())\n\n    # Get gradient\n    cond_grad = -torch.autograd.grad(loss, x)[0]\n\n    # Modify x based on this gradient\n    x = x.detach() + cond_grad\n\n    # Now step with scheduler\n    x = scheduler.step(noise_pred, t, x).prev_sample\n\n# View the output\ngrid = torchvision.utils.make_grid(x, nrow=4)\nim = grid.permute(1, 2, 0).cpu().clip(-1, 1) * 0.5 + 0.5\nImage.fromarray(np.array(im * 255).astype(np.uint8))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:21:55.492474Z","iopub.execute_input":"2024-06-27T13:21:55.493129Z","iopub.status.idle":"2024-06-27T13:22:37.328142Z","shell.execute_reply.started":"2024-06-27T13:21:55.493096Z","shell.execute_reply":"2024-06-27T13:22:37.327265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text / Clip Guidance","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport requests\n\nfrom transformers import CLIPProcessor, CLIPModel\n\nclip_model_pretrained   = transformers.CLIPModel.from_pretrained(\"openai/clip-vit-base-patch32\")\nclip_processor          = transformers.CLIPProcessor.from_pretrained(\"openai/clip-vit-base-patch32\")\n\ntext_descriptions   = [' Red Rose (still life), red flower painting ']\nurl                 = \"http://images.cocodataset.org/val2017/000000039769.jpg\"\nimages              = Image.open(requests.get(url, stream=True).raw)\n\ninputs  = clip_processor (text= text_descriptions, images= images, return_tensors=\"pt\", padding=True)\noutputs = clip_model_pretrained(**inputs)\n\noutputs.text_embeds, outputs.image_embeds","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:31:45.029920Z","iopub.execute_input":"2024-06-27T13:31:45.030640Z","iopub.status.idle":"2024-06-27T13:31:45.075003Z","shell.execute_reply.started":"2024-06-27T13:31:45.030589Z","shell.execute_reply":"2024-06-27T13:31:45.073892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clip_loss(image_embeds, text_embeds):\n    dists = image_embeds.sub(text_embeds).norm(dim=2).div(2).arcsin().pow(2).mul(2)  # Squared Great Circle Distance\n    avg_distance = dists.mean()\n    return avg_distance","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Label Conditioning while Training","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader\nfrom diffusers import DDPMScheduler, UNet2DModel\nfrom matplotlib import pyplot as plt\nfrom tqdm.auto import tqdm\n\ndevice = \"mps\" if torch.backends.mps.is_available() else \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using device: {device}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the dataset\ndataset = torchvision.datasets.MNIST(\n    root=\"mnist/\", train=True, download=True, transform=torchvision.transforms.ToTensor()\n)\n\n# Feed it into a dataloader (batch size 8 here just for demo)\ntrain_dataloader = DataLoader(dataset, batch_size=8, shuffle=True)\n\n# View some examples\nx, y = next(iter(train_dataloader))\nprint(\"Input shape:\", x.shape)\nprint(\"Labels:\", y)\nplt.imshow(torchvision.utils.make_grid(x)[0], cmap=\"Greys\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ClassConditionedUnet(nn.Module):\n    def __init__(self, num_classes=10, class_emb_size=4):\n        super().__init__()\n\n        # The embedding layer will map the class label to a vector of size class_emb_size\n        self.class_emb = nn.Embedding(num_classes, class_emb_size)\n\n        # Self.model is an unconditional UNet with extra input channels to accept the conditioning information (the class embedding)\n        self.model = UNet2DModel(\n            sample_size=28,  # the target image resolution\n            in_channels=1 + class_emb_size,  # Additional input channels for class cond.\n            out_channels=1,  # the number of output channels\n            layers_per_block=2,  # how many ResNet layers to use per UNet block\n            block_out_channels=(32, 64, 64),\n            down_block_types=(\n                \"DownBlock2D\",  # a regular ResNet downsampling block\n                \"AttnDownBlock2D\",  # a ResNet downsampling block with spatial self-attention\n                \"AttnDownBlock2D\",\n            ),\n            up_block_types=(\n                \"AttnUpBlock2D\",\n                \"AttnUpBlock2D\",  # a ResNet upsampling block with spatial self-attention\n                \"UpBlock2D\",  # a regular ResNet upsampling block\n            ),\n        )\n\n    # Our forward method now takes the class labels as an additional argument\n    def forward(self, x, t, class_labels):\n        # Shape of x:\n        bs, ch, w, h = x.shape\n\n        # class conditioning in right shape to add as additional input channels\n        class_cond = self.class_emb(class_labels)  # Map to embedding dimension\n        class_cond = class_cond.view(bs, class_cond.shape[1], 1, 1).expand(bs, class_cond.shape[1], w, h)\n        # x is shape (bs, 1, 28, 28) and class_cond is now (bs, 4, 28, 28)\n\n        # Net input is now x and class cond concatenated together along dimension 1\n        net_input = torch.cat((x, class_cond), 1)  # (bs, 5, 28, 28)\n\n        # Feed this to the UNet alongside the timestep and return the prediction\n        return self.model(net_input, t).sample  # (bs, 1, 28, 28)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a scheduler\nnoise_scheduler = DDPMScheduler(num_train_timesteps=1000, beta_schedule=\"squaredcos_cap_v2\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# @markdown Training loop (10 Epochs):\n\n# Redefining the dataloader to set the batch size higher than the demo of 8\ntrain_dataloader = DataLoader(dataset, batch_size=128, shuffle=True)\n\n# How many runs through the data should we do?\nn_epochs = 10\n\n# Our network\nnet = ClassConditionedUnet().to(device)\n\n# Our loss function\nloss_fn = nn.MSELoss()\n\n# The optimizer\nopt = torch.optim.Adam(net.parameters(), lr=1e-3)\n\n# Keeping a record of the losses for later viewing\nlosses = []\n\n# The training loop\nfor epoch in range(n_epochs):\n    for x, y in tqdm(train_dataloader):\n\n        # Get some data and prepare the corrupted version\n        x = x.to(device) * 2 - 1  # Data on the GPU (mapped to (-1, 1))\n        y = y.to(device)\n        noise = torch.randn_like(x)\n        timesteps = torch.randint(0, 999, (x.shape[0],)).long().to(device)\n        noisy_x = noise_scheduler.add_noise(x, noise, timesteps)\n\n        # Get the model prediction\n        pred = net(noisy_x, timesteps, y)  # Note that we pass in the labels y\n\n        # Calculate the loss\n        loss = loss_fn(pred, noise)  # How close is the output to the noise\n\n        # Backprop and update the params:\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n\n        # Store the loss for later\n        losses.append(loss.item())\n\n    # Print out the average of the last 100 loss values to get an idea of progress:\n    avg_loss = sum(losses[-100:]) / 100\n    print(f\"Finished epoch {epoch}. Average of the last 100 loss values: {avg_loss:05f}\")\n\n# View the loss curve\nplt.plot(losses)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# @markdown Sampling some different digits:\n\n# Prepare random x to start from, plus some desired labels y\nx = torch.randn(80, 1, 28, 28).to(device)\ny = torch.tensor([[i] * 8 for i in range(10)]).flatten().to(device)\n\n# Sampling loop\nfor i, t in tqdm(enumerate(noise_scheduler.timesteps)):\n\n    # Get model pred\n    with torch.no_grad():\n        residual = net(x, t, y)  # Again, note that we pass in our labels y\n\n    # Update sample with step\n    x = noise_scheduler.step(residual, t, x).prev_sample\n\n# Show the results\nfig, ax = plt.subplots(1, 1, figsize=(12, 12))\nax.imshow(torchvision.utils.make_grid(x.detach().cpu().clip(-1, 1), nrow=8)[0], cmap=\"Greys\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stable Diffusion Components","metadata":{}},{"cell_type":"code","source":"import torch\nimport requests\nfrom PIL import Image\nfrom io import BytesIO\nfrom matplotlib import pyplot as plt\n\n# We'll be exploring a number of pipelines today!\nfrom diffusers import (\n    StableDiffusionPipeline,\n    StableDiffusionImg2ImgPipeline,\n    StableDiffusionInpaintPipeline,\n    StableDiffusionDepth2ImgPipeline,\n)\n\n\n# We'll use a couple of demo images later in the notebook\ndef download_image(url):\n    response = requests.get(url)\n    return Image.open(BytesIO(response.content)).convert(\"RGB\")\n\n\n# Download images for inpainting example\nimg_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png\"\nmask_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png\"\n\ninit_image = download_image(img_url).resize((512, 512))\nmask_image = download_image(mask_url).resize((512, 512))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the pipeline\nmodel_id = \"stabilityai/stable-diffusion-2-1-base\"\npipe     = StableDiffusionPipeline.from_pretrained(model_id).to(device)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set up a generator for reproducibility\ngenerator = torch.Generator(device=device).manual_seed(42)\n\n# Run the pipeline, showing some of the available arguments\npipe_output = pipe(\n    prompt=\"Palette knife painting of an autumn cityscape\",  # What to generate\n    negative_prompt=\"Oversaturated, blurry, low quality\",  # What NOT to generate\n    height=480,\n    width=640,  # Specify the image size\n    guidance_scale=8,  # How strongly to follow the prompt\n    num_inference_steps=35,  # How many steps to take\n    generator=generator,  # Fixed random seed\n)\n\n# View the resulting image\npipe_output.images[0]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# @markdown comparing guidance scales:\ncfg_scales = [1.1, 8, 12]  # @param\nprompt = \"A collie with a pink hat\"  # @param\nfig, axs = plt.subplots(1, len(cfg_scales), figsize=(16, 5))\nfor i, ax in enumerate(axs):\n    im = pipe(\n        prompt,\n        height=480,\n        width=480,\n        guidance_scale=cfg_scales[i],\n        num_inference_steps=35,\n        generator=torch.Generator(device=device).manual_seed(42),\n    ).images[0]\n    ax.imshow(im)\n    ax.set_title(f\"CFG Scale {cfg_scales[i]}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Text Conditioned while Training","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}