[diffusion] model: support Qwen-Image 2.1 Turbo and checkpoint sampling grids (#43391)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -9,7 +9,7 @@ SGLang diffusion features an end-to-end unified pipeline for accelerating diffus
|
||||
## Key Features
|
||||
|
||||
SGLang Diffusion has the following features:
|
||||
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
|
||||
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1 / Qwen-Image 2.1 Turbo, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
|
||||
- Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations
|
||||
- Ease of use: OpenAI-compatible api, CLI, python sdk, and a [ComfyUI plugin](apps/ComfyUI_SGLDiffusion/README.md)
|
||||
- [Conditioning reuse](https://docs.sglang.io/docs/sglang-diffusion/caching-acceleration#conditioning-cache): bounded CPU caching for repeated text/image inputs and VAE posteriors, with `--disable-conditioning-cache` to opt out
|
||||
|
||||
@@ -33,6 +33,7 @@ class QwenImage21PipelineConfig(ImagePipelineConfig):
|
||||
vae_config: QwenImage21VAEConfig = field(default_factory=QwenImage21VAEConfig)
|
||||
text_encoder_configs: tuple = field(default_factory=lambda: (Qwen3VLConfig(),))
|
||||
text_encoder_precisions: tuple[str, ...] = ("bf16",)
|
||||
sample_sigmas: list[float] | None = None
|
||||
|
||||
def validate_server_args(self, server_args: Any) -> None:
|
||||
super().validate_server_args(server_args)
|
||||
@@ -48,7 +49,9 @@ class QwenImage21PipelineConfig(ImagePipelineConfig):
|
||||
return True
|
||||
|
||||
def prepare_sigmas(self, sigmas, num_inference_steps):
|
||||
return self._prepare_sigmas(sigmas, num_inference_steps)
|
||||
if sigmas is None:
|
||||
sigmas = self.sample_sigmas
|
||||
return list(self._prepare_sigmas(sigmas, num_inference_steps))
|
||||
|
||||
def get_classifier_free_guidance_scale(self, batch, guidance_scale):
|
||||
return (
|
||||
@@ -113,6 +116,6 @@ def register():
|
||||
register_configs(
|
||||
sampling_param_cls=QwenImage21SamplingParams,
|
||||
pipeline_config_cls=QwenImage21PipelineConfig,
|
||||
hf_model_paths=["Qwen/Qwen-Image-2.1"],
|
||||
hf_model_paths=["Qwen/Qwen-Image-2.1", "Qwen/Qwen-Image-2.1-Turbo"],
|
||||
model_detectors=[lambda hf_id: "qwen-image-2.1" in hf_id.lower()],
|
||||
)
|
||||
|
||||
@@ -22,6 +22,13 @@ class QwenImage21Pipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
"scheduler",
|
||||
]
|
||||
|
||||
def _load_config(self):
|
||||
model_index = super()._load_config()
|
||||
sample_sigmas = model_index.pop("sample_sigmas", None)
|
||||
if self.server_args.pipeline_config.sample_sigmas is None:
|
||||
self.server_args.pipeline_config.sample_sigmas = sample_sigmas
|
||||
return model_index
|
||||
|
||||
def create_pipeline_stages(self, server_args):
|
||||
self.add_stage(QwenImage21InputValidationStage())
|
||||
self.add_stage_factory(
|
||||
|
||||
+7
@@ -55,6 +55,13 @@ class QwenImage21InputValidationStage(InputValidationStage):
|
||||
batch = super().forward(batch, server_args)
|
||||
if batch.height % 32 or batch.width % 32:
|
||||
raise ValueError("Qwen-Image 2.1 height and width must be divisible by 32")
|
||||
batch.sigmas = server_args.pipeline_config.prepare_sigmas(
|
||||
batch.sigmas, batch.num_inference_steps
|
||||
)
|
||||
if not batch.sigmas:
|
||||
raise ValueError("Qwen-Image 2.1 requires a non-empty sigma schedule")
|
||||
# the preset or explicit sigma grid determines the denoising step count
|
||||
batch.num_inference_steps = len(batch.sigmas)
|
||||
return batch
|
||||
|
||||
|
||||
|
||||
@@ -6,8 +6,10 @@ from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers import FlowMatchEulerDiscreteScheduler as ReferenceFlowMatchScheduler
|
||||
from diffusers.image_processor import VaeImageProcessor
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature
|
||||
@@ -45,6 +47,9 @@ from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import (
|
||||
Qwen3VLVisionRotaryEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_qwenimage21 import (
|
||||
AutoencoderKLQwenImage21,
|
||||
QwenImage21RMS_norm,
|
||||
@@ -52,6 +57,7 @@ from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_qwenimage21 import
|
||||
_patchify,
|
||||
_unpatchify,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.qwen_image21 import QwenImage21Pipeline
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
|
||||
@@ -61,11 +67,99 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.q
|
||||
QwenImage21EncodingStage,
|
||||
QwenImage21InputValidationStage,
|
||||
collapse_image_slots,
|
||||
prepare_qwen21_mu,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import (
|
||||
TimestepPreparationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms.rocm import RocmPlatform
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("steps", [2, 40])
|
||||
@pytest.mark.parametrize("preset", [False, True])
|
||||
@pytest.mark.parametrize("explicit_sigmas", [None, [1.0, 0.7, 0.3]])
|
||||
@pytest.mark.parametrize("dynamic_shifting", [False, True])
|
||||
def test_checkpoint_sigma_grid_reaches_request_scheduler(
|
||||
tmp_path, monkeypatch, steps, preset, explicit_sigmas, dynamic_shifting
|
||||
):
|
||||
sample_sigmas = [
|
||||
1.0,
|
||||
0.978453,
|
||||
0.95418,
|
||||
0.926626,
|
||||
0.89508,
|
||||
0.845148,
|
||||
0.704534,
|
||||
0.414568,
|
||||
]
|
||||
model_index = {
|
||||
"_class_name": "QwenImage21Pipeline",
|
||||
"_diffusers_version": "0.41.0.dev0",
|
||||
"scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
|
||||
}
|
||||
if preset:
|
||||
model_index["sample_sigmas"] = sample_sigmas
|
||||
(tmp_path / "model_index.json").write_text(json.dumps(model_index))
|
||||
(tmp_path / "scheduler").mkdir()
|
||||
config = QwenImage21PipelineConfig()
|
||||
args = SimpleNamespace(
|
||||
pipeline_config=config,
|
||||
model_subfolder=None,
|
||||
revision=None,
|
||||
enable_cfg_parallel=False,
|
||||
)
|
||||
pipeline = object.__new__(QwenImage21Pipeline)
|
||||
pipeline.model_path = str(tmp_path)
|
||||
pipeline.server_args = args
|
||||
loaded_config = pipeline._load_config()
|
||||
assert "sample_sigmas" not in loaded_config
|
||||
assert config.sample_sigmas == (sample_sigmas if preset else None)
|
||||
|
||||
batch = Req(
|
||||
sampling_params=QwenImage21SamplingParams(
|
||||
prompt="a ceramic teapot",
|
||||
height=1024,
|
||||
width=1024,
|
||||
num_inference_steps=steps,
|
||||
),
|
||||
sigmas=explicit_sigmas,
|
||||
)
|
||||
QwenImage21InputValidationStage().forward(batch, args)
|
||||
batch.extra["qwen21_mu"] = 0.7
|
||||
scheduler_kwargs = dict(
|
||||
use_dynamic_shifting=dynamic_shifting,
|
||||
shift=1.0,
|
||||
shift_terminal=0.02 if dynamic_shifting else None,
|
||||
)
|
||||
scheduler = FlowMatchEulerDiscreteScheduler(**scheduler_kwargs)
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation.get_local_torch_device",
|
||||
lambda: torch.device("cpu"),
|
||||
)
|
||||
TimestepPreparationStage(scheduler, [prepare_qwen21_mu]).forward(batch, args)
|
||||
expected_sigmas = explicit_sigmas
|
||||
if expected_sigmas is None:
|
||||
expected_sigmas = sample_sigmas if preset else np.linspace(1, 1 / steps, steps)
|
||||
# preserve the native scheduler's existing dynamic-shift rounding; Turbo's
|
||||
# unshifted grid must also match the Diffusers scheduler bit for bit
|
||||
scheduler_cls = (
|
||||
FlowMatchEulerDiscreteScheduler
|
||||
if dynamic_shifting
|
||||
else ReferenceFlowMatchScheduler
|
||||
)
|
||||
reference = scheduler_cls(**scheduler_kwargs)
|
||||
reference.set_timesteps(sigmas=expected_sigmas, mu=0.7, device="cpu")
|
||||
torch.testing.assert_close(batch.timesteps, reference.timesteps, atol=0, rtol=0)
|
||||
torch.testing.assert_close(batch.scheduler.sigmas, reference.sigmas, atol=0, rtol=0)
|
||||
assert batch.num_inference_steps == len(expected_sigmas)
|
||||
assert batch.scheduler.num_inference_steps == len(expected_sigmas)
|
||||
assert isinstance(batch.sigmas, list)
|
||||
# request-local schedules must not mutate the checkpoint's preset
|
||||
batch.sigmas[0] = 0.9
|
||||
assert config.sample_sigmas == (sample_sigmas if preset else None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("layout", ["contiguous", "channels_last", "transposed"])
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@@ -582,6 +676,10 @@ def test_architecture_derived_dimensions():
|
||||
|
||||
|
||||
def test_registry_routes_local_checkpoint_and_preserves_legacy():
|
||||
assert (
|
||||
_get_config_info("Qwen/Qwen-Image-2.1-Turbo").pipeline_config_cls
|
||||
is QwenImage21PipelineConfig
|
||||
)
|
||||
assert (
|
||||
_get_config_info("Qwen/Qwen-Image-2.1").pipeline_config_cls
|
||||
is QwenImage21PipelineConfig
|
||||
|
||||
Reference in New Issue
Block a user