[diffusion] fix: align SGLD CFG combine formula with diffusers Z-Image (#23772)

Co-authored-by: SGLang CI <ci@sglang.ai>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Prozac614
2026-10-09 19:47:03 +08:00
committed by GitHub
co-authored by SGLang CI Mick Qian Mick
parent f48ed2127e
commit 59eb71a831
7 changed files with 119 additions and 24 deletions
@@ -23,6 +23,10 @@ from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config impo
from sglang.multimodal_gen.configs.post_training.pipeline_configs import (
ZImageRolloutPipelineMixin,
)
from sglang.multimodal_gen.runtime.distributed.cfg_policy import (
CFGPolicy,
_apply_cfg_postprocess,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_sp_group,
get_sp_parallel_rank,
@@ -30,6 +34,22 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
)
@dataclass
class ZImageCFGPolicy(CFGPolicy):
parallel_uses_serial_arithmetic: bool = True
def combine(
self, predictions, batch, cfg_scale, pipeline_config, *, cfg_parallel=False
):
if len(predictions) == 1:
return predictions[0]
# the DiT already negates its output; match diffusers' fp32 CFG arithmetic
pos = predictions[0].float()
neg = predictions[1].float()
noise_pred = pos + cfg_scale * (pos - neg)
return _apply_cfg_postprocess(noise_pred, pos, batch, pipeline_config)
def zimage_preprocess_text(prompt: str):
messages = [
{"role": "user", "content": prompt},
@@ -59,6 +79,7 @@ def zimage_postprocess_text(
@dataclass
class ZImagePipelineConfig(ZImageRolloutPipelineMixin, ImagePipelineConfig):
cfg_policy: CFGPolicy = field(default_factory=ZImageCFGPolicy)
should_use_guidance: bool = False
task_type: ModelTaskType = ModelTaskType.T2I
dit_config: DiTConfig = field(default_factory=ZImageDitConfig)
@@ -538,6 +538,14 @@ class SamplingParams:
"""Number of final samples produced by one expanded scheduler request."""
return 1
def should_use_classifier_free_guidance(self) -> bool:
cfg_scale = (
self.true_cfg_scale
if self.true_cfg_scale is not None
else self.guidance_scale
)
return cfg_scale > 1.0 and self.negative_prompt is not None
@classmethod
def default_image_output_format(cls) -> str | None:
"""Return a model-owned default format for the image API, if any."""
@@ -8,11 +8,24 @@ from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams
@dataclass
class ZImageTurboSamplingParams(SamplingParams):
class ZImageSamplingParams(SamplingParams):
num_inference_steps: int = 50
num_frames: int = 1
negative_prompt: str = ""
guidance_scale: float = 5.0
cfg_normalization: float | bool = True
def should_use_classifier_free_guidance(self) -> bool:
if self.negative_prompt is None:
self.negative_prompt = ""
return self.guidance_scale > 0.0
@dataclass
class ZImageTurboSamplingParams(ZImageSamplingParams):
num_inference_steps: int = 9
num_frames: int = 1
negative_prompt: str = None
# Z-Image officially recommends starting at 1024x1024
height: int = 1024
width: int = 1024
@@ -33,13 +46,3 @@ class ZImageTurboSamplingParams(SamplingParams):
],
)
)
@dataclass
class ZImageSamplingParams(SamplingParams):
num_inference_steps: int = 50
num_frames: int = 1
negative_prompt: str = " "
guidance_scale: float = 5.0
cfg_normalization: float | bool = True
@@ -394,13 +394,7 @@ class Req:
self.metrics = RequestMetrics(request_id=self.request_id)
return
# Prefer true_cfg_scale when it is explicitly provided.
cfg_scale = (
self.true_cfg_scale
if self.true_cfg_scale is not None
else self.guidance_scale
)
if cfg_scale > 1.0 and self.negative_prompt is not None:
if self.sampling_params.should_use_classifier_free_guidance():
self.do_classifier_free_guidance = True
if self.negative_prompt_embeds is None:
self.negative_prompt_embeds = []
@@ -411,11 +411,7 @@ class TextEncodingStage(ConditionEncodingStage):
result.add_check(
"negative_prompt",
batch.negative_prompt,
lambda x: (
not batch.do_classifier_free_guidance
or V.string_not_none(x)
or isinstance(x, str)
),
lambda x: not batch.do_classifier_free_guidance or isinstance(x, str),
)
result.add_check(
"do_classifier_free_guidance",
@@ -5,13 +5,80 @@ from unittest.mock import patch
import torch
from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.configs.sample.zimage import (
ZImageSamplingParams,
ZImageTurboSamplingParams,
)
from sglang.multimodal_gen.runtime.models.dits.zimage import (
ZImageRMSNorm,
ZImageTransformer2DModel,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
class TestZImagePipelineConfig(unittest.TestCase):
def test_cfg_activation_and_default_negative_prompt(self):
for cls in (ZImageSamplingParams, ZImageTurboSamplingParams):
for scale in (0.0, 0.5, 1.0, 3.5):
for negative in (None, "", "avoid blur"):
with self.subTest(cls=cls.__name__, scale=scale, negative=negative):
req = Req(
sampling_params=cls(
guidance_scale=scale, negative_prompt=negative
)
)
self.assertEqual(req.do_classifier_free_guidance, scale > 0)
self.assertEqual(req.negative_prompt, negative or "")
self.assertFalse(
Req(sampling_params=ZImageTurboSamplingParams()).do_classifier_free_guidance
)
def test_standard_cfg_activation_is_unchanged(self):
self.assertTrue(
Req(
sampling_params=SamplingParams(guidance_scale=1.0),
do_classifier_free_guidance=True,
).do_classifier_free_guidance
)
for scale, negative, true_scale, expected in (
(1.0, "", None, False),
(3.5, None, None, False),
(3.5, "", None, True),
(3.5, "", 1.0, False),
(1.0, "", 3.5, True),
):
req = Req(
sampling_params=SamplingParams(
guidance_scale=scale,
negative_prompt=negative,
true_cfg_scale=true_scale,
)
)
self.assertEqual(req.do_classifier_free_guidance, expected)
def test_cfg_matches_diffusers_fp32_arithmetic(self):
config = ZImagePipelineConfig()
pos = torch.tensor([0.1, 0.25, -0.75], dtype=torch.bfloat16)
neg = torch.tensor([-0.2, 0.5, 0.125], dtype=torch.bfloat16)
# emulate the native DiT's sign inversion from diffusers' raw predictions
for scale in (0.5, 1.0, 3.5):
batch = SimpleNamespace(cfg_normalization=False, guidance_rescale=0.0)
expected = -(pos.float() + scale * (pos.float() - neg.float()))
for parallel in (False, True):
actual = config.cfg_policy.combine(
[-pos, -neg], batch, scale, config, cfg_parallel=parallel
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
self.assertEqual(actual.dtype, torch.float32)
torch.testing.assert_close(
config.cfg_policy.combine([-pos], batch, 0.0, config),
-pos,
rtol=0,
atol=0,
)
self.assertTrue(config.cfg_policy.parallel_uses_serial_arithmetic)
def test_rmsnorm_native_formula(self) -> None:
norm = ZImageRMSNorm(4, eps=1e-5)
with torch.no_grad():