[Fix] diffusion: stabilize FLUX.3 serving and reuse static conditioning (#44493)
Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Co-authored-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> Co-authored-by: niehen6174 <1639206518@qq.com> Co-authored-by: Mick Qian <mickqian@radixark.ai> Co-authored-by: Nikita Korobov <14355239+nekorobov@users.noreply.github.com> Co-authored-by: Daniel Campora <961215+dcampora@users.noreply.github.com> Co-authored-by: Richard Wang <11150595+rwang5203@users.noreply.github.com> Co-authored-by: BBuf <1182563586@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Fabio Camaiora <fcamaiora@nvidia.com> Co-authored-by: Linda-Stadter <lstadter@nvidia.com>
This commit is contained in:
co-authored by
Yihao Wang
Dom Brown
niehen6174
Mick Qian
Nikita Korobov
Daniel Campora
Richard Wang
BBuf
Claude Opus 5.5
Mick Qian
Fabio Camaiora
Linda-Stadter
parent
9771e5a4f0
commit
d4191a4624
@@ -94,6 +94,7 @@ class Flux3ImageSamplingParams(SamplingParams):
|
||||
def prepare_synthetic_warmup_request_for_queue(
|
||||
self, req: Any, server_args: Any
|
||||
) -> None:
|
||||
self._explicit_fields = {"width", "height"}
|
||||
# Warmup copies the request's steps as the full-workload target before
|
||||
# any adjustment; ``None`` (server default) has to be resolved here.
|
||||
for key in (
|
||||
|
||||
@@ -35,13 +35,14 @@ _MPS_VARLEN_QUERY_CHUNK_SIZE = 128
|
||||
|
||||
|
||||
def torch_default_sdpa_backends():
|
||||
"""Context in which SDPA picks from every PyTorch backend, cuDNN included.
|
||||
"""Reference SDPA with a stable cuDNN-first preference and fallback set.
|
||||
|
||||
The CUDA platform disables cuDNN SDPA process-wide (diffusers#9704). Modules
|
||||
whose reference outputs were produced with PyTorch's defaults opt back in
|
||||
around their own ``scaled_dot_product_attention`` calls.
|
||||
opt back in locally. Priority must be explicit: restoring another SDPA
|
||||
context can move disabled backends to the end of the ambient priority list,
|
||||
otherwise changing reference outputs after the first DiT forward.
|
||||
"""
|
||||
return sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS)
|
||||
return sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True)
|
||||
|
||||
|
||||
class SDPABackend(AttentionBackend):
|
||||
|
||||
@@ -23,7 +23,7 @@ Native fused kernels are bit-close, not bit-exact, to the torch reference.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -96,6 +96,16 @@ _CONTENT_STACK_PREFIX = "content_mode_blocks."
|
||||
TEXT = "text"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StaticStreams:
|
||||
"""Request-owned work for streams whose tokens and timestep stay fixed."""
|
||||
|
||||
hidden: dict[str, Tensor] = field(default_factory=dict)
|
||||
modulation: dict[str, TokenModulation] = field(default_factory=dict)
|
||||
rope: dict[str, Flux3Rope] = field(default_factory=dict)
|
||||
joint_rope: Flux3Rope | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Packed:
|
||||
"""Mode-major tokens with per-sample bounds for segmented attention.
|
||||
@@ -114,8 +124,8 @@ class Packed:
|
||||
timesteps_ref: Tensor | None = None
|
||||
# Request-owned positional tables; the ids do not change during denoising.
|
||||
rope: dict[str, Tensor] | None = None
|
||||
# Eager-only reuse of invariant text/reference mode-block outputs.
|
||||
static_hidden: dict[str, Tensor] | None = None
|
||||
# Eager-only reuse of invariant text/reference computation.
|
||||
static: StaticStreams | None = None
|
||||
|
||||
@property
|
||||
def num_samples(self) -> int:
|
||||
@@ -133,7 +143,7 @@ class Packed:
|
||||
timesteps = {}
|
||||
mode_indices = {}
|
||||
rope = None if self.rope is None else {}
|
||||
static_hidden = None if self.static_hidden is None else {}
|
||||
static = None if self.static is None else StaticStreams()
|
||||
for mode in self.data:
|
||||
bounds = self.mode_indices[mode][first:stop]
|
||||
start, end = bounds[0][0], bounds[-1][1]
|
||||
@@ -144,14 +154,15 @@ class Packed:
|
||||
mode_indices[mode] = [(lo - start, hi - start) for lo, hi in bounds]
|
||||
if rope is not None:
|
||||
rope[mode] = self.rope[mode][start:end]
|
||||
if static_hidden is not None and mode in self.static_hidden:
|
||||
static_hidden[mode] = self.static_hidden[mode][start:end]
|
||||
if static is not None and mode in self.static.hidden:
|
||||
static.hidden[mode] = self.static.hidden[mode][start:end]
|
||||
static.modulation[mode] = tuple(
|
||||
value[start:end] for value in self.static.modulation[mode]
|
||||
)
|
||||
timesteps_ref = (
|
||||
None if self.timesteps_ref is None else self.timesteps_ref[first:stop]
|
||||
)
|
||||
return Packed(
|
||||
data, ids, timesteps, mode_indices, timesteps_ref, rope, static_hidden
|
||||
)
|
||||
return Packed(data, ids, timesteps, mode_indices, timesteps_ref, rope, static)
|
||||
|
||||
|
||||
def pack(
|
||||
@@ -797,8 +808,10 @@ class Flux3ImageDiT(Flux3Transformer):
|
||||
def mod_key(mode: str) -> str:
|
||||
return TEXT_STREAM if mode == TEXT else mode
|
||||
|
||||
vec = {mode: self._token_vector(timesteps[mode]) for mode in modes}
|
||||
self._add_reference_time(packed, modes, vec)
|
||||
cached = {} if packed.static is None else packed.static.hidden
|
||||
active = tuple(mode for mode in modes if mode not in cached)
|
||||
vec = {mode: self._token_vector(timesteps[mode]) for mode in active}
|
||||
self._add_reference_time(packed, active, vec)
|
||||
|
||||
def broadcast_modulation(values, mode):
|
||||
length = data[mode].shape[0]
|
||||
@@ -813,21 +826,23 @@ class Flux3ImageDiT(Flux3Transformer):
|
||||
mode: broadcast_modulation(
|
||||
self.early_stream_modulations[mod_key(mode)](vec[mode]), mode
|
||||
)
|
||||
for mode in modes
|
||||
for mode in active
|
||||
}
|
||||
single = {
|
||||
mode: broadcast_modulation(
|
||||
self.single_stream_modulations[mod_key(mode)](vec[mode]), mode
|
||||
)
|
||||
for mode in modes
|
||||
for mode in active
|
||||
}
|
||||
if packed.static is not None:
|
||||
single.update(packed.static.modulation)
|
||||
x = {
|
||||
mode: (
|
||||
self.txt_in(data[mode].to(self.dtype))
|
||||
if mode == TEXT
|
||||
else self.emb_in[mode](data[mode].to(self.dtype))
|
||||
)
|
||||
for mode in modes
|
||||
for mode in active
|
||||
}
|
||||
rope = {
|
||||
mode: (
|
||||
@@ -853,9 +868,14 @@ class Flux3ImageDiT(Flux3Transformer):
|
||||
|
||||
def _packed_blocks(self, packed, content, x, rope, early, single, layout):
|
||||
"""All samples in one varlen sequence per block (no sequence parallel)."""
|
||||
cached = packed.static_hidden if packed.static_hidden is not None else {}
|
||||
cached = packed.static.hidden if packed.static is not None else {}
|
||||
x.update(cached)
|
||||
mode_rope = {mode: _with_positions(rows) for mode, rows in rope.items()}
|
||||
if packed.static is not None and packed.static.rope:
|
||||
mode_rope = packed.static.rope
|
||||
else:
|
||||
mode_rope = {mode: _with_positions(rows) for mode, rows in rope.items()}
|
||||
if packed.static is not None:
|
||||
packed.static.rope = mode_rope
|
||||
for depth in range(self.depth):
|
||||
for mode in content:
|
||||
if mode in cached:
|
||||
@@ -873,12 +893,20 @@ class Flux3ImageDiT(Flux3Transformer):
|
||||
early[TEXT],
|
||||
seq_lens=_seq_lens(packed.mode_indices[TEXT]),
|
||||
)
|
||||
if packed.static_hidden is not None:
|
||||
packed.static_hidden.update(
|
||||
if packed.static is not None:
|
||||
packed.static.hidden.update(
|
||||
(mode, x[mode]) for mode in (TEXT, "image_cond") if mode in x
|
||||
)
|
||||
packed.static.modulation.update(
|
||||
(mode, single[mode]) for mode in packed.static.hidden
|
||||
)
|
||||
hidden = layout.pack(x)
|
||||
joint_rope = _with_positions(layout.pack(rope))
|
||||
if packed.static is not None and packed.static.joint_rope is not None:
|
||||
joint_rope = packed.static.joint_rope
|
||||
else:
|
||||
joint_rope = _with_positions(layout.pack(rope))
|
||||
if packed.static is not None:
|
||||
packed.static.joint_rope = joint_rope
|
||||
joint_mods = tuple(
|
||||
layout.pack({mode: single[mode][index] for mode in single})
|
||||
for index in range(3)
|
||||
|
||||
+7
-2
@@ -35,7 +35,12 @@ from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3_image import TEXT, Packed, pack
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3_image import (
|
||||
TEXT,
|
||||
Packed,
|
||||
StaticStreams,
|
||||
pack,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import use_vae_fast_path
|
||||
from sglang.multimodal_gen.runtime.pipelines.flux3_image_package import (
|
||||
Flux3ImagePackageConfig,
|
||||
@@ -791,7 +796,7 @@ class Flux3ImageDenoiseStage(DenoisingStage):
|
||||
and batch.skip_softmax_params is None
|
||||
and self.transformer.can_reuse_mode_blocks()
|
||||
):
|
||||
packed.static_hidden = {}
|
||||
packed.static = StaticStreams()
|
||||
self._maybe_enable_cache_dit_and_torch_compile(num_steps, batch)
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
|
||||
@@ -629,7 +629,14 @@ def build_warmup_reqs(
|
||||
num_inference_steps=sampling_defaults.num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
)
|
||||
if include_warmup_image:
|
||||
if (
|
||||
server_args.enable_breakable_cuda_graph is True
|
||||
and sampling_defaults.image_path
|
||||
):
|
||||
# Optional image conditioning changes the captured signature. Use
|
||||
# the configured references rather than capturing text-only inputs.
|
||||
req_kwargs["image_path"] = sampling_defaults.image_path
|
||||
elif include_warmup_image:
|
||||
if warmup_input_path is None:
|
||||
raise RuntimeError(
|
||||
"Warmup image path is required for image-input model"
|
||||
|
||||
@@ -87,6 +87,7 @@ def test_grouped_requests_multi_output_and_repeat(diffusion_server):
|
||||
|
||||
for editing in (False, True):
|
||||
serial = [generate(index, editing)[0] for index in range(2)]
|
||||
np.testing.assert_array_equal(generate(editing=editing)[0], serial[0])
|
||||
barrier = Barrier(2)
|
||||
with ThreadPoolExecutor(2) as pool:
|
||||
grouped = list(
|
||||
|
||||
@@ -22,6 +22,9 @@ from unittest.mock import MagicMock, patch
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux3_image import (
|
||||
Flux3ImageSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux_finetuned import (
|
||||
Flux2FinetunedPipelineConfig,
|
||||
)
|
||||
@@ -1062,6 +1065,35 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
|
||||
["/tmp/warmup.png", "/tmp/warmup.png"],
|
||||
)
|
||||
|
||||
def test_breakable_warmup_uses_configured_references(self):
|
||||
server_args = MagicMock()
|
||||
server_args.warmup_steps = 1
|
||||
server_args.enable_cfg_parallel = False
|
||||
server_args.enable_torch_compile = False
|
||||
server_args.enable_breakable_cuda_graph = True
|
||||
server_args.pipeline_config.task_type = ModelTaskType.TI2I
|
||||
references = ["first.png", "second.png"]
|
||||
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.warmup_request_builder.get_model_sampling_defaults",
|
||||
return_value=Flux3ImageSamplingParams(
|
||||
width=512,
|
||||
height=512,
|
||||
image_path=references,
|
||||
flux3_mode="i2i",
|
||||
num_inference_steps=16,
|
||||
),
|
||||
):
|
||||
req = build_warmup_reqs(
|
||||
server_args,
|
||||
warmup_resolutions=["512x512"],
|
||||
server_based_warmup=True,
|
||||
)[0]
|
||||
|
||||
self.assertEqual(req.image_path, references)
|
||||
self.assertEqual(req.flux3_mode, "i2i")
|
||||
self.assertEqual(req.sampling_params._explicit_fields, {"width", "height"})
|
||||
|
||||
def test_server_based_warmup_keeps_required_image_input(self):
|
||||
server_args = MagicMock()
|
||||
server_args.warmup_steps = 1
|
||||
|
||||
@@ -198,10 +198,18 @@ def test_packed_cfg_forward_matches_flux3_transformer_per_sample(
|
||||
packed.rope = {mode: native.rope(ids) for mode, ids in packed.ids.items()}
|
||||
cached = native(packed)["image"].float()
|
||||
torch.testing.assert_close(cached, predicted, rtol=0, atol=0)
|
||||
packed.static_hidden = {}
|
||||
for _ in range(2):
|
||||
reused = native(packed)["image"].float()
|
||||
torch.testing.assert_close(reused, predicted, rtol=0, atol=0)
|
||||
packed.static = flux3_image.StaticStreams()
|
||||
modulation = native.single_stream_modulations["txt"]
|
||||
with (
|
||||
mock.patch.object(
|
||||
native.txt_in, "forward", wraps=native.txt_in.forward
|
||||
) as embed,
|
||||
mock.patch.object(modulation, "forward", wraps=modulation.forward) as mod,
|
||||
):
|
||||
for _ in range(2):
|
||||
reused = native(packed)["image"].float()
|
||||
torch.testing.assert_close(reused, predicted, rtol=0, atol=0)
|
||||
assert embed.call_count == mod.call_count == 1
|
||||
for mode, rows in packed.sample(1).rope.items():
|
||||
start, end = packed.mode_indices[mode][1]
|
||||
assert torch.equal(rows, packed.rope[mode][start:end])
|
||||
@@ -231,7 +239,7 @@ def test_packed_cfg_forward_matches_flux3_transformer_per_sample(
|
||||
packed.timesteps["image"].fill_(0.3)
|
||||
packed.data["image"].mul_(0.9)
|
||||
reused = native(packed)["image"]
|
||||
packed.static_hidden = None
|
||||
packed.static = None
|
||||
uncached = native(packed)["image"]
|
||||
torch.testing.assert_close(reused, uncached, rtol=0, atol=0)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import (
|
||||
_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS,
|
||||
SDPAImpl,
|
||||
torch_default_sdpa_backends,
|
||||
)
|
||||
|
||||
NUM_HEADS, HEAD_DIM, SEQ_LEN = 8, 128, 512
|
||||
@@ -92,3 +93,19 @@ def test_opting_out_leaves_backend_selection_alone():
|
||||
def test_cpu_tensors_do_not_get_a_cuda_context():
|
||||
cpu_q = torch.randn(1, NUM_HEADS, 16, HEAD_DIM)
|
||||
assert isinstance(_impl(True)._sdpa_context(cpu_q), type(nullcontext()))
|
||||
|
||||
|
||||
def test_reference_sdpa_does_not_inherit_a_previous_attention_priority():
|
||||
q, k, v = _qkv()
|
||||
with sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True):
|
||||
with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
|
||||
try:
|
||||
F.scaled_dot_product_attention(q, k, v)
|
||||
except RuntimeError:
|
||||
pytest.skip("no cuDNN attention kernel for this shape")
|
||||
for priority in (
|
||||
_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS,
|
||||
list(reversed(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS)),
|
||||
):
|
||||
with sdpa_kernel(priority, set_priority=True):
|
||||
assert _kernel_under(torch_default_sdpa_backends(), q, k, v) == "cudnn"
|
||||
|
||||
Reference in New Issue
Block a user