From 59eb71a831f6b87babc2400737c7faabed5ee01c Mon Sep 17 00:00:00 2001 From: Prozac614 Date: Fri, 9 Oct 2026 19:47:03 +0800 Subject: [PATCH] [diffusion] fix: align SGLD CFG combine formula with diffusers Z-Image (#23772) Co-authored-by: SGLang CI Co-authored-by: Mick Qian Co-authored-by: Mick --- .../diffusion/Z-Image/Z-Image-Turbo.mdx | 6 ++ .../configs/pipeline_configs/zimage.py | 21 ++++++ .../configs/sample/sampling_params.py | 8 +++ .../multimodal_gen/configs/sample/zimage.py | 27 ++++---- .../runtime/pipelines_core/schedule_batch.py | 8 +-- .../pipelines_core/stages/text_encoding.py | 6 +- .../test/unit/test_zimage_pipeline_config.py | 67 +++++++++++++++++++ 7 files changed, 119 insertions(+), 24 deletions(-) diff --git a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx index b5dc01e4ac..f38c431142 100644 --- a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx +++ b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx @@ -73,6 +73,12 @@ with open("output.png", "wb") as f: ### 4.2 Advanced Usage +Z-Image uses `positive + guidance_scale * (positive - negative)` guidance, unlike +the standard CFG convention. Any positive `guidance_scale` enables CFG; omitted +negative prompts use an empty string. Z-Image-Turbo defaults to `guidance_scale=0` +and does not run the negative branch. Enabling CFG adds a second DiT branch, so +expect additional inference cost; it is not automatically recommended for Turbo. + #### 4.2.1 Cache-DiT Acceleration SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve up to 7.4x inference speedup with minimal quality loss. You can set `SGLANG_CACHE_DIT_ENABLED=True` to enable it. For more details, please refer to the SGLang Cache-DiT [documentation](/docs/sglang-diffusion/cache_dit). diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py b/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py index f39e846a5d..adc8a978a5 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py @@ -23,6 +23,10 @@ from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config impo from sglang.multimodal_gen.configs.post_training.pipeline_configs import ( ZImageRolloutPipelineMixin, ) +from sglang.multimodal_gen.runtime.distributed.cfg_policy import ( + CFGPolicy, + _apply_cfg_postprocess, +) from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_sp_group, get_sp_parallel_rank, @@ -30,6 +34,22 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import ( ) +@dataclass +class ZImageCFGPolicy(CFGPolicy): + parallel_uses_serial_arithmetic: bool = True + + def combine( + self, predictions, batch, cfg_scale, pipeline_config, *, cfg_parallel=False + ): + if len(predictions) == 1: + return predictions[0] + # the DiT already negates its output; match diffusers' fp32 CFG arithmetic + pos = predictions[0].float() + neg = predictions[1].float() + noise_pred = pos + cfg_scale * (pos - neg) + return _apply_cfg_postprocess(noise_pred, pos, batch, pipeline_config) + + def zimage_preprocess_text(prompt: str): messages = [ {"role": "user", "content": prompt}, @@ -59,6 +79,7 @@ def zimage_postprocess_text( @dataclass class ZImagePipelineConfig(ZImageRolloutPipelineMixin, ImagePipelineConfig): + cfg_policy: CFGPolicy = field(default_factory=ZImageCFGPolicy) should_use_guidance: bool = False task_type: ModelTaskType = ModelTaskType.T2I dit_config: DiTConfig = field(default_factory=ZImageDitConfig) diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py index 51707c0477..f292b216ab 100644 --- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py +++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -538,6 +538,14 @@ class SamplingParams: """Number of final samples produced by one expanded scheduler request.""" return 1 + def should_use_classifier_free_guidance(self) -> bool: + cfg_scale = ( + self.true_cfg_scale + if self.true_cfg_scale is not None + else self.guidance_scale + ) + return cfg_scale > 1.0 and self.negative_prompt is not None + @classmethod def default_image_output_format(cls) -> str | None: """Return a model-owned default format for the image API, if any.""" diff --git a/python/sglang/multimodal_gen/configs/sample/zimage.py b/python/sglang/multimodal_gen/configs/sample/zimage.py index dd5f9d82c2..88625970bd 100644 --- a/python/sglang/multimodal_gen/configs/sample/zimage.py +++ b/python/sglang/multimodal_gen/configs/sample/zimage.py @@ -8,11 +8,24 @@ from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams @dataclass -class ZImageTurboSamplingParams(SamplingParams): +class ZImageSamplingParams(SamplingParams): + num_inference_steps: int = 50 + num_frames: int = 1 + negative_prompt: str = "" + guidance_scale: float = 5.0 + cfg_normalization: float | bool = True + + def should_use_classifier_free_guidance(self) -> bool: + if self.negative_prompt is None: + self.negative_prompt = "" + return self.guidance_scale > 0.0 + + +@dataclass +class ZImageTurboSamplingParams(ZImageSamplingParams): num_inference_steps: int = 9 num_frames: int = 1 - negative_prompt: str = None # Z-Image officially recommends starting at 1024x1024 height: int = 1024 width: int = 1024 @@ -33,13 +46,3 @@ class ZImageTurboSamplingParams(SamplingParams): ], ) ) - - -@dataclass -class ZImageSamplingParams(SamplingParams): - num_inference_steps: int = 50 - - num_frames: int = 1 - negative_prompt: str = " " - guidance_scale: float = 5.0 - cfg_normalization: float | bool = True diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py index 20ccc59500..b70920d3ac 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py @@ -394,13 +394,7 @@ class Req: self.metrics = RequestMetrics(request_id=self.request_id) return - # Prefer true_cfg_scale when it is explicitly provided. - cfg_scale = ( - self.true_cfg_scale - if self.true_cfg_scale is not None - else self.guidance_scale - ) - if cfg_scale > 1.0 and self.negative_prompt is not None: + if self.sampling_params.should_use_classifier_free_guidance(): self.do_classifier_free_guidance = True if self.negative_prompt_embeds is None: self.negative_prompt_embeds = [] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py index 5807fbce2d..444675088c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py @@ -411,11 +411,7 @@ class TextEncodingStage(ConditionEncodingStage): result.add_check( "negative_prompt", batch.negative_prompt, - lambda x: ( - not batch.do_classifier_free_guidance - or V.string_not_none(x) - or isinstance(x, str) - ), + lambda x: not batch.do_classifier_free_guidance or isinstance(x, str), ) result.add_check( "do_classifier_free_guidance", diff --git a/python/sglang/multimodal_gen/test/unit/test_zimage_pipeline_config.py b/python/sglang/multimodal_gen/test/unit/test_zimage_pipeline_config.py index 49fa50db61..ec139c84d4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_zimage_pipeline_config.py +++ b/python/sglang/multimodal_gen/test/unit/test_zimage_pipeline_config.py @@ -5,13 +5,80 @@ from unittest.mock import patch import torch from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.configs.sample.zimage import ( + ZImageSamplingParams, + ZImageTurboSamplingParams, +) from sglang.multimodal_gen.runtime.models.dits.zimage import ( ZImageRMSNorm, ZImageTransformer2DModel, ) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req class TestZImagePipelineConfig(unittest.TestCase): + def test_cfg_activation_and_default_negative_prompt(self): + for cls in (ZImageSamplingParams, ZImageTurboSamplingParams): + for scale in (0.0, 0.5, 1.0, 3.5): + for negative in (None, "", "avoid blur"): + with self.subTest(cls=cls.__name__, scale=scale, negative=negative): + req = Req( + sampling_params=cls( + guidance_scale=scale, negative_prompt=negative + ) + ) + self.assertEqual(req.do_classifier_free_guidance, scale > 0) + self.assertEqual(req.negative_prompt, negative or "") + self.assertFalse( + Req(sampling_params=ZImageTurboSamplingParams()).do_classifier_free_guidance + ) + + def test_standard_cfg_activation_is_unchanged(self): + self.assertTrue( + Req( + sampling_params=SamplingParams(guidance_scale=1.0), + do_classifier_free_guidance=True, + ).do_classifier_free_guidance + ) + for scale, negative, true_scale, expected in ( + (1.0, "", None, False), + (3.5, None, None, False), + (3.5, "", None, True), + (3.5, "", 1.0, False), + (1.0, "", 3.5, True), + ): + req = Req( + sampling_params=SamplingParams( + guidance_scale=scale, + negative_prompt=negative, + true_cfg_scale=true_scale, + ) + ) + self.assertEqual(req.do_classifier_free_guidance, expected) + + def test_cfg_matches_diffusers_fp32_arithmetic(self): + config = ZImagePipelineConfig() + pos = torch.tensor([0.1, 0.25, -0.75], dtype=torch.bfloat16) + neg = torch.tensor([-0.2, 0.5, 0.125], dtype=torch.bfloat16) + # emulate the native DiT's sign inversion from diffusers' raw predictions + for scale in (0.5, 1.0, 3.5): + batch = SimpleNamespace(cfg_normalization=False, guidance_rescale=0.0) + expected = -(pos.float() + scale * (pos.float() - neg.float())) + for parallel in (False, True): + actual = config.cfg_policy.combine( + [-pos, -neg], batch, scale, config, cfg_parallel=parallel + ) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + self.assertEqual(actual.dtype, torch.float32) + torch.testing.assert_close( + config.cfg_policy.combine([-pos], batch, 0.0, config), + -pos, + rtol=0, + atol=0, + ) + self.assertTrue(config.cfg_policy.parallel_uses_serial_arithmetic) + def test_rmsnorm_native_formula(self) -> None: norm = ZImageRMSNorm(4, eps=1e-5) with torch.no_grad():