pytorch_stable_diffusion_3.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/stable-diffusion-diffusers-colab/blob/main/pytorch_stable_diffusion_3.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title Install - Step 1\n",
        "!pip install -q transformers accelerate omegaconf xformers==0.0.20 opencv-contrib-python controlnet_aux\n",
        "!pip install -q git+https://github.com/huggingface/diffusers\n",
        "\n",
        "# !apt -y install -qq aria2\n",
        "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/ckpt/sd_xl_refiner_1.0/resolve/main/sd_xl_refiner_1.0_0.9vae.safetensors -d /content/models -o sd_xl_refiner_1.0.safetensors\n",
        "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/ckpt/juggernaut-xl/resolve/main/juggernautXL_version2.safetensors -d /content/models -o juggernautXL_version2.safetensors\n",
        "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M 'https://huggingface.co/lllyasviel/ControlNet-v1-1/resolve/main/control_v11p_sd15_canny.pth' -d /content/models/controlnet -o control_v11p_sd15_canny.pth\n",
        "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M 'https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_canny_fp16.safetensors' -d /content/models/controlnet -o control_v11p_sd15_canny_fp16.safetensors"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title Install - Step 2\n",
        "import random, gc, torch\n",
        "\n",
        "def image_grid(imgs, rows, cols):\n",
        "    w,h = imgs[0].size\n",
        "    grid = Image.new('RGB', size=(cols*w, rows*h))\n",
        "    for i, img in enumerate(imgs): grid.paste(img, box=(i%cols*w, i//cols*h))\n",
        "    return grid"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title Vanilla - Step 1\n",
        "from diffusers import StableDiffusionXLPipeline\n",
        "# pipeline = StableDiffusionXLPipeline.from_single_file(\"/content/models/juggernautXL_version2.safetensors\", torch_dtype=torch.float16, use_safetensors=True, safety_checker=None ).to(\"cuda\")\n",
        "pipeline = StableDiffusionXLPipeline.from_pretrained(\"stabilityai/stable-diffusion-xl-base-1.0\", torch_dtype=torch.float16, variant=\"fp16\", use_safetensors=True, safety_checker=None ).to(\"cuda\")\n",
        "pipeline.enable_xformers_memory_efficient_attention()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title Vanilla - Step 2\n",
        "# pipeline.load_lora_weights(\"minimaxir/sdxl-wrong-lora\")\n",
        "image = pipeline(prompt=\"a photo of a dog in a bucket\", negative_prompt=\"blurry\", num_inference_steps=50, height=1024, width=1024, guidance_scale=7).images[0]\n",
        "image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #Canny - Step 1\n",
        "import torch\n",
        "from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel\n",
        "# controlnet = ControlNetModel.from_single_file('/content/models/controlnet/control_v11p_sd15_canny.pth', torch_dtype=torch.float16) # invalid load key, '<'.\n",
        "# controlnet = ControlNetModel.from_single_file('https://huggingface.co/lllyasviel/ControlNet-v1-1/blob/main/control_v11p_sd15_canny.pth', torch_dtype=torch.float16).to(\"cuda\")\n",
        "controlnet = ControlNetModel.from_pretrained(\"diffusers/controlnet-canny-sdxl-1.0\", variant=\"fp16\", torch_dtype=torch.float16).to(\"cuda\")\n",
        "pipeline = StableDiffusionXLControlNetPipeline.from_pretrained(\"stabilityai/stable-diffusion-xl-base-1.0\", controlnet=controlnet, torch_dtype=torch.float16, variant=\"fp16\", use_safetensors=True).to(\"cuda\")\n",
        "# pipeline.enable_model_cpu_offload()\n",
        "pipeline.enable_xformers_memory_efficient_attention()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #Canny - Step 2\n",
        "from diffusers import UniPCMultistepScheduler\n",
        "pipeline.scheduler = UniPCMultistepScheduler.from_config(pipeline.scheduler.config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #Canny - Step 3\n",
        "from diffusers.utils import load_image\n",
        "image = load_image(\"https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png\")\n",
        "image_raw = image\n",
        "\n",
        "import cv2\n",
        "from PIL import Image\n",
        "import numpy as np\n",
        "image = np.array(image)\n",
        "low_threshold = 100\n",
        "high_threshold = 200\n",
        "image = cv2.Canny(image, low_threshold, high_threshold)\n",
        "image = image[:, :, None]\n",
        "image = np.concatenate([image, image, image], axis=2)\n",
        "canny_image = Image.fromarray(image)\n",
        "image = image_grid([image_raw, canny_image], 1, 2)\n",
        "image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #Canny - Step 4\n",
        "controlnet_conditioning_scale = 0.5\n",
        "image = pipeline(prompt=\"a photo of a dog in a bucket\", controlnet_conditioning_scale=controlnet_conditioning_scale, image=canny_image, negative_prompt=\"blurry\", num_inference_steps=50, height=1024, width=1024, guidance_scale=7).images[0]\n",
        "image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #OpenPose - Step 1\n",
        "import torch\n",
        "from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel\n",
        "controlnet = ControlNetModel.from_pretrained(\"fusing/stable-diffusion-v1-5-controlnet-openpose\", torch_dtype=torch.float16).to(\"cuda\")\n",
        "pipeline = StableDiffusionXLControlNetPipeline.from_pretrained(\"stabilityai/stable-diffusion-xl-base-1.0\", controlnet=controlnet, torch_dtype=torch.float16, variant=\"fp16\", use_safetensors=True).to(\"cuda\")\n",
        "# pipeline.enable_model_cpu_offload()\n",
        "pipeline.enable_xformers_memory_efficient_attention()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #Canny - Step 2\n",
        "from diffusers import UniPCMultistepScheduler\n",
        "pipeline.scheduler = UniPCMultistepScheduler.from_config(pipeline.scheduler.config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #OpenPose - Step 3\n",
        "from PIL import Image\n",
        "from controlnet_aux import OpenposeDetector\n",
        "from diffusers.utils import load_image\n",
        "image = load_image(\"https://hf.co/datasets/YiYiXu/controlnet-testing/resolve/main/yoga1.jpeg\")\n",
        "image_raw = image\n",
        "model = OpenposeDetector.from_pretrained(\"lllyasviel/ControlNet\")\n",
        "pose = model(image)\n",
        "# image = image_grid([image_raw, pose], 1, 2)\n",
        "# image\n",
        "pose"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title ControlNet #OpenPose - Step 4\n",
        "controlnet_conditioning_scale = 0.5\n",
        "image = pipeline(prompt=\"a photo of a dog in a bucket\", controlnet_conditioning_scale=controlnet_conditioning_scale, image=pose, negative_prompt=\"blurry\", num_inference_steps=50, height=1024, width=1024, guidance_scale=7).images[0]\n",
        "image"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}