Alternative implementation in Refiners

[deltheil]

We are building Refiners, an open source, PyTorch-based framework made to easily train and run adapters on top of foundational models. Following up your latest release (congrats!), I just wanted to let you know that T2I-Adapter is now fully supported in Refiners!

E.g. to run the canny + SDXL demo:

1. Follow these install steps
2. Run the code snippet below which gives:

import torch

from PIL import Image

from refiners.foundationals.latent_diffusion.stable_diffusion_xl.model import StableDiffusion_XL
from refiners.foundationals.latent_diffusion.stable_diffusion_xl import SDXLT2IAdapter
from refiners.foundationals.latent_diffusion.schedulers import DDIM
from refiners.fluxion.utils import image_to_tensor, load_from_safetensors, manual_seed


device = "cuda"
condition_image = Image.open("cond_canny.png")

ddim_scheduler = DDIM(num_inference_steps=30)
sdxl = StableDiffusion_XL(scheduler=ddim_scheduler, device="cuda", dtype=torch.float16)
sdxl.clip_text_encoder.load_from_safetensors("clip_text_xl.safetensors")
sdxl.lda.load_from_safetensors("lda_xl.safetensors")
sdxl.unet.load_from_safetensors("unet_xl.safetensors")

with torch.no_grad():
    prompt = "Mystical fairy in real, magic, 4k picture, high quality"
    negative_prompt = (
        "extra digit, fewer digits, cropped, worst quality, low quality, glitch, deformed, mutated, ugly, disfigured"
    )

    clip_text_embedding, pooled_text_embedding = sdxl.compute_clip_text_embedding(
        text=prompt, negative_text=negative_prompt
    )
    time_ids = sdxl.default_time_ids

    t2i_adapter = SDXLT2IAdapter(
        target=sdxl.unet, name="canny", weights=load_from_safetensors("t2i_canny_xl.safetensors")
    ).inject()
    t2i_adapter.set_scale(0.8)

    condition = image_to_tensor(condition_image.convert("RGB"), device=device, dtype=torch.float16)
    t2i_adapter.set_condition_features(features=t2i_adapter.compute_condition_features(condition))

    manual_seed(2)
    x = torch.randn(1, 4, condition_image.height // 8, condition_image.width // 8, device=device, dtype=torch.float16)

    for step in sdxl.steps:
        x = sdxl(
            x,
            step=step,
            clip_text_embedding=clip_text_embedding,
            pooled_text_embedding=pooled_text_embedding,
            time_ids=time_ids,
            condition_scale=7.5,
        )
    predicted_image = sdxl.lda.decode_latents(x)

predicted_image.save("output.png")
print("done: see output.png")

Note: T2I-Adapter for SD1.5 is also supported

Feedback welcome!