[diffusion] model: support Qwen-Image 2.1 Turbo and checkpoint sampling grids (#43391)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-10 09:12:56 +08:00
committed by GitHub
co-authored by Mick Qian
parent e8fb52c526
commit cde16aa64d
9 changed files with 216 additions and 29 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ SGLang diffusion features an end-to-end unified pipeline for accelerating diffus
## Key Features
SGLang Diffusion has the following features:
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1 / Qwen-Image 2.1 Turbo, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
- Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations
- Ease of use: OpenAI-compatible api, CLI, python sdk, and a [ComfyUI plugin](apps/ComfyUI_SGLDiffusion/README.md)
- [Conditioning reuse](https://docs.sglang.io/docs/sglang-diffusion/caching-acceleration#conditioning-cache): bounded CPU caching for repeated text/image inputs and VAE posteriors, with `--disable-conditioning-cache` to opt out
@@ -33,6 +33,7 @@ class QwenImage21PipelineConfig(ImagePipelineConfig):
vae_config: QwenImage21VAEConfig = field(default_factory=QwenImage21VAEConfig)
text_encoder_configs: tuple = field(default_factory=lambda: (Qwen3VLConfig(),))
text_encoder_precisions: tuple[str, ...] = ("bf16",)
sample_sigmas: list[float] | None = None
def validate_server_args(self, server_args: Any) -> None:
super().validate_server_args(server_args)
@@ -48,7 +49,9 @@ class QwenImage21PipelineConfig(ImagePipelineConfig):
return True
def prepare_sigmas(self, sigmas, num_inference_steps):
return self._prepare_sigmas(sigmas, num_inference_steps)
if sigmas is None:
sigmas = self.sample_sigmas
return list(self._prepare_sigmas(sigmas, num_inference_steps))
def get_classifier_free_guidance_scale(self, batch, guidance_scale):
return (
@@ -113,6 +116,6 @@ def register():
register_configs(
sampling_param_cls=QwenImage21SamplingParams,
pipeline_config_cls=QwenImage21PipelineConfig,
hf_model_paths=["Qwen/Qwen-Image-2.1"],
hf_model_paths=["Qwen/Qwen-Image-2.1", "Qwen/Qwen-Image-2.1-Turbo"],
model_detectors=[lambda hf_id: "qwen-image-2.1" in hf_id.lower()],
)
@@ -22,6 +22,13 @@ class QwenImage21Pipeline(LoRAPipeline, ComposedPipelineBase):
"scheduler",
]
def _load_config(self):
model_index = super()._load_config()
sample_sigmas = model_index.pop("sample_sigmas", None)
if self.server_args.pipeline_config.sample_sigmas is None:
self.server_args.pipeline_config.sample_sigmas = sample_sigmas
return model_index
def create_pipeline_stages(self, server_args):
self.add_stage(QwenImage21InputValidationStage())
self.add_stage_factory(
@@ -55,6 +55,13 @@ class QwenImage21InputValidationStage(InputValidationStage):
batch = super().forward(batch, server_args)
if batch.height % 32 or batch.width % 32:
raise ValueError("Qwen-Image 2.1 height and width must be divisible by 32")
batch.sigmas = server_args.pipeline_config.prepare_sigmas(
batch.sigmas, batch.num_inference_steps
)
if not batch.sigmas:
raise ValueError("Qwen-Image 2.1 requires a non-empty sigma schedule")
# the preset or explicit sigma grid determines the denoising step count
batch.num_inference_steps = len(batch.sigmas)
return batch
@@ -6,8 +6,10 @@ from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
import pytest
import torch
from diffusers import FlowMatchEulerDiscreteScheduler as ReferenceFlowMatchScheduler
from diffusers.image_processor import VaeImageProcessor
from PIL import Image
from transformers import BatchFeature
@@ -45,6 +47,9 @@ from sglang.multimodal_gen.runtime.models.encoders.base import (
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import (
Qwen3VLVisionRotaryEmbedding,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_qwenimage21 import (
AutoencoderKLQwenImage21,
QwenImage21RMS_norm,
@@ -52,6 +57,7 @@ from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_qwenimage21 import
_patchify,
_unpatchify,
)
from sglang.multimodal_gen.runtime.pipelines.qwen_image21 import QwenImage21Pipeline
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
@@ -61,11 +67,99 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.q
QwenImage21EncodingStage,
QwenImage21InputValidationStage,
collapse_image_slots,
prepare_qwen21_mu,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import (
TimestepPreparationStage,
)
from sglang.multimodal_gen.runtime.platforms.rocm import RocmPlatform
from sglang.multimodal_gen.runtime.server_args import ServerArgs
@pytest.mark.parametrize("steps", [2, 40])
@pytest.mark.parametrize("preset", [False, True])
@pytest.mark.parametrize("explicit_sigmas", [None, [1.0, 0.7, 0.3]])
@pytest.mark.parametrize("dynamic_shifting", [False, True])
def test_checkpoint_sigma_grid_reaches_request_scheduler(
tmp_path, monkeypatch, steps, preset, explicit_sigmas, dynamic_shifting
):
sample_sigmas = [
1.0,
0.978453,
0.95418,
0.926626,
0.89508,
0.845148,
0.704534,
0.414568,
]
model_index = {
"_class_name": "QwenImage21Pipeline",
"_diffusers_version": "0.41.0.dev0",
"scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
}
if preset:
model_index["sample_sigmas"] = sample_sigmas
(tmp_path / "model_index.json").write_text(json.dumps(model_index))
(tmp_path / "scheduler").mkdir()
config = QwenImage21PipelineConfig()
args = SimpleNamespace(
pipeline_config=config,
model_subfolder=None,
revision=None,
enable_cfg_parallel=False,
)
pipeline = object.__new__(QwenImage21Pipeline)
pipeline.model_path = str(tmp_path)
pipeline.server_args = args
loaded_config = pipeline._load_config()
assert "sample_sigmas" not in loaded_config
assert config.sample_sigmas == (sample_sigmas if preset else None)
batch = Req(
sampling_params=QwenImage21SamplingParams(
prompt="a ceramic teapot",
height=1024,
width=1024,
num_inference_steps=steps,
),
sigmas=explicit_sigmas,
)
QwenImage21InputValidationStage().forward(batch, args)
batch.extra["qwen21_mu"] = 0.7
scheduler_kwargs = dict(
use_dynamic_shifting=dynamic_shifting,
shift=1.0,
shift_terminal=0.02 if dynamic_shifting else None,
)
scheduler = FlowMatchEulerDiscreteScheduler(**scheduler_kwargs)
monkeypatch.setattr(
"sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation.get_local_torch_device",
lambda: torch.device("cpu"),
)
TimestepPreparationStage(scheduler, [prepare_qwen21_mu]).forward(batch, args)
expected_sigmas = explicit_sigmas
if expected_sigmas is None:
expected_sigmas = sample_sigmas if preset else np.linspace(1, 1 / steps, steps)
# preserve the native scheduler's existing dynamic-shift rounding; Turbo's
# unshifted grid must also match the Diffusers scheduler bit for bit
scheduler_cls = (
FlowMatchEulerDiscreteScheduler
if dynamic_shifting
else ReferenceFlowMatchScheduler
)
reference = scheduler_cls(**scheduler_kwargs)
reference.set_timesteps(sigmas=expected_sigmas, mu=0.7, device="cpu")
torch.testing.assert_close(batch.timesteps, reference.timesteps, atol=0, rtol=0)
torch.testing.assert_close(batch.scheduler.sigmas, reference.sigmas, atol=0, rtol=0)
assert batch.num_inference_steps == len(expected_sigmas)
assert batch.scheduler.num_inference_steps == len(expected_sigmas)
assert isinstance(batch.sigmas, list)
# request-local schedules must not mutate the checkpoint's preset
batch.sigmas[0] = 0.9
assert config.sample_sigmas == (sample_sigmas if preset else None)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("layout", ["contiguous", "channels_last", "transposed"])
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@@ -582,6 +676,10 @@ def test_architecture_derived_dimensions():
def test_registry_routes_local_checkpoint_and_preserves_legacy():
assert (
_get_config_info("Qwen/Qwen-Image-2.1-Turbo").pipeline_config_cls
is QwenImage21PipelineConfig
)
assert (
_get_config_info("Qwen/Qwen-Image-2.1").pipeline_config_cls
is QwenImage21PipelineConfig