[diffusion] fix: align SGLD CFG combine formula with diffusers Z-Image (#23772)
Co-authored-by: SGLang CI <ci@sglang.ai> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
co-authored by
SGLang CI
Mick Qian
Mick
parent
f48ed2127e
commit
59eb71a831
@@ -23,6 +23,10 @@ from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config impo
|
||||
from sglang.multimodal_gen.configs.post_training.pipeline_configs import (
|
||||
ZImageRolloutPipelineMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import (
|
||||
CFGPolicy,
|
||||
_apply_cfg_postprocess,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_sp_group,
|
||||
get_sp_parallel_rank,
|
||||
@@ -30,6 +34,22 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZImageCFGPolicy(CFGPolicy):
|
||||
parallel_uses_serial_arithmetic: bool = True
|
||||
|
||||
def combine(
|
||||
self, predictions, batch, cfg_scale, pipeline_config, *, cfg_parallel=False
|
||||
):
|
||||
if len(predictions) == 1:
|
||||
return predictions[0]
|
||||
# the DiT already negates its output; match diffusers' fp32 CFG arithmetic
|
||||
pos = predictions[0].float()
|
||||
neg = predictions[1].float()
|
||||
noise_pred = pos + cfg_scale * (pos - neg)
|
||||
return _apply_cfg_postprocess(noise_pred, pos, batch, pipeline_config)
|
||||
|
||||
|
||||
def zimage_preprocess_text(prompt: str):
|
||||
messages = [
|
||||
{"role": "user", "content": prompt},
|
||||
@@ -59,6 +79,7 @@ def zimage_postprocess_text(
|
||||
|
||||
@dataclass
|
||||
class ZImagePipelineConfig(ZImageRolloutPipelineMixin, ImagePipelineConfig):
|
||||
cfg_policy: CFGPolicy = field(default_factory=ZImageCFGPolicy)
|
||||
should_use_guidance: bool = False
|
||||
task_type: ModelTaskType = ModelTaskType.T2I
|
||||
dit_config: DiTConfig = field(default_factory=ZImageDitConfig)
|
||||
|
||||
@@ -538,6 +538,14 @@ class SamplingParams:
|
||||
"""Number of final samples produced by one expanded scheduler request."""
|
||||
return 1
|
||||
|
||||
def should_use_classifier_free_guidance(self) -> bool:
|
||||
cfg_scale = (
|
||||
self.true_cfg_scale
|
||||
if self.true_cfg_scale is not None
|
||||
else self.guidance_scale
|
||||
)
|
||||
return cfg_scale > 1.0 and self.negative_prompt is not None
|
||||
|
||||
@classmethod
|
||||
def default_image_output_format(cls) -> str | None:
|
||||
"""Return a model-owned default format for the image API, if any."""
|
||||
|
||||
@@ -8,11 +8,24 @@ from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZImageTurboSamplingParams(SamplingParams):
|
||||
class ZImageSamplingParams(SamplingParams):
|
||||
num_inference_steps: int = 50
|
||||
num_frames: int = 1
|
||||
negative_prompt: str = ""
|
||||
guidance_scale: float = 5.0
|
||||
cfg_normalization: float | bool = True
|
||||
|
||||
def should_use_classifier_free_guidance(self) -> bool:
|
||||
if self.negative_prompt is None:
|
||||
self.negative_prompt = ""
|
||||
return self.guidance_scale > 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZImageTurboSamplingParams(ZImageSamplingParams):
|
||||
num_inference_steps: int = 9
|
||||
|
||||
num_frames: int = 1
|
||||
negative_prompt: str = None
|
||||
# Z-Image officially recommends starting at 1024x1024
|
||||
height: int = 1024
|
||||
width: int = 1024
|
||||
@@ -33,13 +46,3 @@ class ZImageTurboSamplingParams(SamplingParams):
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZImageSamplingParams(SamplingParams):
|
||||
num_inference_steps: int = 50
|
||||
|
||||
num_frames: int = 1
|
||||
negative_prompt: str = " "
|
||||
guidance_scale: float = 5.0
|
||||
cfg_normalization: float | bool = True
|
||||
|
||||
@@ -394,13 +394,7 @@ class Req:
|
||||
self.metrics = RequestMetrics(request_id=self.request_id)
|
||||
return
|
||||
|
||||
# Prefer true_cfg_scale when it is explicitly provided.
|
||||
cfg_scale = (
|
||||
self.true_cfg_scale
|
||||
if self.true_cfg_scale is not None
|
||||
else self.guidance_scale
|
||||
)
|
||||
if cfg_scale > 1.0 and self.negative_prompt is not None:
|
||||
if self.sampling_params.should_use_classifier_free_guidance():
|
||||
self.do_classifier_free_guidance = True
|
||||
if self.negative_prompt_embeds is None:
|
||||
self.negative_prompt_embeds = []
|
||||
|
||||
@@ -411,11 +411,7 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
result.add_check(
|
||||
"negative_prompt",
|
||||
batch.negative_prompt,
|
||||
lambda x: (
|
||||
not batch.do_classifier_free_guidance
|
||||
or V.string_not_none(x)
|
||||
or isinstance(x, str)
|
||||
),
|
||||
lambda x: not batch.do_classifier_free_guidance or isinstance(x, str),
|
||||
)
|
||||
result.add_check(
|
||||
"do_classifier_free_guidance",
|
||||
|
||||
@@ -5,13 +5,80 @@ from unittest.mock import patch
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.zimage import (
|
||||
ZImageSamplingParams,
|
||||
ZImageTurboSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.zimage import (
|
||||
ZImageRMSNorm,
|
||||
ZImageTransformer2DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
|
||||
|
||||
class TestZImagePipelineConfig(unittest.TestCase):
|
||||
def test_cfg_activation_and_default_negative_prompt(self):
|
||||
for cls in (ZImageSamplingParams, ZImageTurboSamplingParams):
|
||||
for scale in (0.0, 0.5, 1.0, 3.5):
|
||||
for negative in (None, "", "avoid blur"):
|
||||
with self.subTest(cls=cls.__name__, scale=scale, negative=negative):
|
||||
req = Req(
|
||||
sampling_params=cls(
|
||||
guidance_scale=scale, negative_prompt=negative
|
||||
)
|
||||
)
|
||||
self.assertEqual(req.do_classifier_free_guidance, scale > 0)
|
||||
self.assertEqual(req.negative_prompt, negative or "")
|
||||
self.assertFalse(
|
||||
Req(sampling_params=ZImageTurboSamplingParams()).do_classifier_free_guidance
|
||||
)
|
||||
|
||||
def test_standard_cfg_activation_is_unchanged(self):
|
||||
self.assertTrue(
|
||||
Req(
|
||||
sampling_params=SamplingParams(guidance_scale=1.0),
|
||||
do_classifier_free_guidance=True,
|
||||
).do_classifier_free_guidance
|
||||
)
|
||||
for scale, negative, true_scale, expected in (
|
||||
(1.0, "", None, False),
|
||||
(3.5, None, None, False),
|
||||
(3.5, "", None, True),
|
||||
(3.5, "", 1.0, False),
|
||||
(1.0, "", 3.5, True),
|
||||
):
|
||||
req = Req(
|
||||
sampling_params=SamplingParams(
|
||||
guidance_scale=scale,
|
||||
negative_prompt=negative,
|
||||
true_cfg_scale=true_scale,
|
||||
)
|
||||
)
|
||||
self.assertEqual(req.do_classifier_free_guidance, expected)
|
||||
|
||||
def test_cfg_matches_diffusers_fp32_arithmetic(self):
|
||||
config = ZImagePipelineConfig()
|
||||
pos = torch.tensor([0.1, 0.25, -0.75], dtype=torch.bfloat16)
|
||||
neg = torch.tensor([-0.2, 0.5, 0.125], dtype=torch.bfloat16)
|
||||
# emulate the native DiT's sign inversion from diffusers' raw predictions
|
||||
for scale in (0.5, 1.0, 3.5):
|
||||
batch = SimpleNamespace(cfg_normalization=False, guidance_rescale=0.0)
|
||||
expected = -(pos.float() + scale * (pos.float() - neg.float()))
|
||||
for parallel in (False, True):
|
||||
actual = config.cfg_policy.combine(
|
||||
[-pos, -neg], batch, scale, config, cfg_parallel=parallel
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
self.assertEqual(actual.dtype, torch.float32)
|
||||
torch.testing.assert_close(
|
||||
config.cfg_policy.combine([-pos], batch, 0.0, config),
|
||||
-pos,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
self.assertTrue(config.cfg_policy.parallel_uses_serial_arithmetic)
|
||||
|
||||
def test_rmsnorm_native_formula(self) -> None:
|
||||
norm = ZImageRMSNorm(4, eps=1e-5)
|
||||
with torch.no_grad():
|
||||
|
||||
Reference in New Issue
Block a user