[Fix] diffusion: stabilize FLUX.3 serving and reuse static conditioning (#44493)

Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com>
Co-authored-by: Dom Brown <3886319+DomBrown@users.noreply.github.com>
Co-authored-by: niehen6174 <1639206518@qq.com>
Co-authored-by: Mick Qian <mickqian@radixark.ai>
Co-authored-by: Nikita Korobov <14355239+nekorobov@users.noreply.github.com>
Co-authored-by: Daniel Campora <961215+dcampora@users.noreply.github.com>
Co-authored-by: Richard Wang <11150595+rwang5203@users.noreply.github.com>
Co-authored-by: BBuf <1182563586@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Fabio Camaiora <fcamaiora@nvidia.com>
Co-authored-by: Linda-Stadter <lstadter@nvidia.com>
This commit is contained in:
Mick
2026-10-17 20:46:13 +08:00
committed by GitHub
co-authored by Yihao Wang Dom Brown niehen6174 Mick Qian Nikita Korobov Daniel Campora Richard Wang BBuf Claude Opus 5.5 Mick Qian Fabio Camaiora Linda-Stadter
parent 9771e5a4f0
commit d4191a4624
11 changed files with 322 additions and 64 deletions
@@ -94,6 +94,7 @@ class Flux3ImageSamplingParams(SamplingParams):
def prepare_synthetic_warmup_request_for_queue(
self, req: Any, server_args: Any
) -> None:
self._explicit_fields = {"width", "height"}
# Warmup copies the request's steps as the full-workload target before
# any adjustment; ``None`` (server default) has to be resolved here.
for key in (
@@ -35,13 +35,14 @@ _MPS_VARLEN_QUERY_CHUNK_SIZE = 128
def torch_default_sdpa_backends():
"""Context in which SDPA picks from every PyTorch backend, cuDNN included.
"""Reference SDPA with a stable cuDNN-first preference and fallback set.
The CUDA platform disables cuDNN SDPA process-wide (diffusers#9704). Modules
whose reference outputs were produced with PyTorch's defaults opt back in
around their own ``scaled_dot_product_attention`` calls.
opt back in locally. Priority must be explicit: restoring another SDPA
context can move disabled backends to the end of the ambient priority list,
otherwise changing reference outputs after the first DiT forward.
"""
return sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS)
return sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True)
class SDPABackend(AttentionBackend):
@@ -23,7 +23,7 @@ Native fused kernels are bit-close, not bit-exact, to the torch reference.
from __future__ import annotations
from dataclasses import dataclass
from dataclasses import dataclass, field
import torch
import torch.nn as nn
@@ -96,6 +96,16 @@ _CONTENT_STACK_PREFIX = "content_mode_blocks."
TEXT = "text"
@dataclass
class StaticStreams:
"""Request-owned work for streams whose tokens and timestep stay fixed."""
hidden: dict[str, Tensor] = field(default_factory=dict)
modulation: dict[str, TokenModulation] = field(default_factory=dict)
rope: dict[str, Flux3Rope] = field(default_factory=dict)
joint_rope: Flux3Rope | None = None
@dataclass
class Packed:
"""Mode-major tokens with per-sample bounds for segmented attention.
@@ -114,8 +124,8 @@ class Packed:
timesteps_ref: Tensor | None = None
# Request-owned positional tables; the ids do not change during denoising.
rope: dict[str, Tensor] | None = None
# Eager-only reuse of invariant text/reference mode-block outputs.
static_hidden: dict[str, Tensor] | None = None
# Eager-only reuse of invariant text/reference computation.
static: StaticStreams | None = None
@property
def num_samples(self) -> int:
@@ -133,7 +143,7 @@ class Packed:
timesteps = {}
mode_indices = {}
rope = None if self.rope is None else {}
static_hidden = None if self.static_hidden is None else {}
static = None if self.static is None else StaticStreams()
for mode in self.data:
bounds = self.mode_indices[mode][first:stop]
start, end = bounds[0][0], bounds[-1][1]
@@ -144,14 +154,15 @@ class Packed:
mode_indices[mode] = [(lo - start, hi - start) for lo, hi in bounds]
if rope is not None:
rope[mode] = self.rope[mode][start:end]
if static_hidden is not None and mode in self.static_hidden:
static_hidden[mode] = self.static_hidden[mode][start:end]
if static is not None and mode in self.static.hidden:
static.hidden[mode] = self.static.hidden[mode][start:end]
static.modulation[mode] = tuple(
value[start:end] for value in self.static.modulation[mode]
)
timesteps_ref = (
None if self.timesteps_ref is None else self.timesteps_ref[first:stop]
)
return Packed(
data, ids, timesteps, mode_indices, timesteps_ref, rope, static_hidden
)
return Packed(data, ids, timesteps, mode_indices, timesteps_ref, rope, static)
def pack(
@@ -797,8 +808,10 @@ class Flux3ImageDiT(Flux3Transformer):
def mod_key(mode: str) -> str:
return TEXT_STREAM if mode == TEXT else mode
vec = {mode: self._token_vector(timesteps[mode]) for mode in modes}
self._add_reference_time(packed, modes, vec)
cached = {} if packed.static is None else packed.static.hidden
active = tuple(mode for mode in modes if mode not in cached)
vec = {mode: self._token_vector(timesteps[mode]) for mode in active}
self._add_reference_time(packed, active, vec)
def broadcast_modulation(values, mode):
length = data[mode].shape[0]
@@ -813,21 +826,23 @@ class Flux3ImageDiT(Flux3Transformer):
mode: broadcast_modulation(
self.early_stream_modulations[mod_key(mode)](vec[mode]), mode
)
for mode in modes
for mode in active
}
single = {
mode: broadcast_modulation(
self.single_stream_modulations[mod_key(mode)](vec[mode]), mode
)
for mode in modes
for mode in active
}
if packed.static is not None:
single.update(packed.static.modulation)
x = {
mode: (
self.txt_in(data[mode].to(self.dtype))
if mode == TEXT
else self.emb_in[mode](data[mode].to(self.dtype))
)
for mode in modes
for mode in active
}
rope = {
mode: (
@@ -853,9 +868,14 @@ class Flux3ImageDiT(Flux3Transformer):
def _packed_blocks(self, packed, content, x, rope, early, single, layout):
"""All samples in one varlen sequence per block (no sequence parallel)."""
cached = packed.static_hidden if packed.static_hidden is not None else {}
cached = packed.static.hidden if packed.static is not None else {}
x.update(cached)
mode_rope = {mode: _with_positions(rows) for mode, rows in rope.items()}
if packed.static is not None and packed.static.rope:
mode_rope = packed.static.rope
else:
mode_rope = {mode: _with_positions(rows) for mode, rows in rope.items()}
if packed.static is not None:
packed.static.rope = mode_rope
for depth in range(self.depth):
for mode in content:
if mode in cached:
@@ -873,12 +893,20 @@ class Flux3ImageDiT(Flux3Transformer):
early[TEXT],
seq_lens=_seq_lens(packed.mode_indices[TEXT]),
)
if packed.static_hidden is not None:
packed.static_hidden.update(
if packed.static is not None:
packed.static.hidden.update(
(mode, x[mode]) for mode in (TEXT, "image_cond") if mode in x
)
packed.static.modulation.update(
(mode, single[mode]) for mode in packed.static.hidden
)
hidden = layout.pack(x)
joint_rope = _with_positions(layout.pack(rope))
if packed.static is not None and packed.static.joint_rope is not None:
joint_rope = packed.static.joint_rope
else:
joint_rope = _with_positions(layout.pack(rope))
if packed.static is not None:
packed.static.joint_rope = joint_rope
joint_mods = tuple(
layout.pack({mode: single[mode][index] for mode in single})
for index in range(3)
@@ -35,7 +35,12 @@ from sglang.multimodal_gen.runtime.managers.forward_context import (
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.dits.flux3_image import TEXT, Packed, pack
from sglang.multimodal_gen.runtime.models.dits.flux3_image import (
TEXT,
Packed,
StaticStreams,
pack,
)
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import use_vae_fast_path
from sglang.multimodal_gen.runtime.pipelines.flux3_image_package import (
Flux3ImagePackageConfig,
@@ -791,7 +796,7 @@ class Flux3ImageDenoiseStage(DenoisingStage):
and batch.skip_softmax_params is None
and self.transformer.can_reuse_mode_blocks()
):
packed.static_hidden = {}
packed.static = StaticStreams()
self._maybe_enable_cache_dit_and_torch_compile(num_steps, batch)
with (
torch.inference_mode(),
@@ -629,7 +629,14 @@ def build_warmup_reqs(
num_inference_steps=sampling_defaults.num_inference_steps,
num_frames=num_frames,
)
if include_warmup_image:
if (
server_args.enable_breakable_cuda_graph is True
and sampling_defaults.image_path
):
# Optional image conditioning changes the captured signature. Use
# the configured references rather than capturing text-only inputs.
req_kwargs["image_path"] = sampling_defaults.image_path
elif include_warmup_image:
if warmup_input_path is None:
raise RuntimeError(
"Warmup image path is required for image-input model"
@@ -87,6 +87,7 @@ def test_grouped_requests_multi_output_and_repeat(diffusion_server):
for editing in (False, True):
serial = [generate(index, editing)[0] for index in range(2)]
np.testing.assert_array_equal(generate(editing=editing)[0], serial[0])
barrier = Barrier(2)
with ThreadPoolExecutor(2) as pool:
grouped = list(
@@ -22,6 +22,9 @@ from unittest.mock import MagicMock, patch
import torch
from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType
from sglang.multimodal_gen.configs.pipeline_configs.flux3_image import (
Flux3ImageSamplingParams,
)
from sglang.multimodal_gen.configs.pipeline_configs.flux_finetuned import (
Flux2FinetunedPipelineConfig,
)
@@ -1062,6 +1065,35 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
["/tmp/warmup.png", "/tmp/warmup.png"],
)
def test_breakable_warmup_uses_configured_references(self):
server_args = MagicMock()
server_args.warmup_steps = 1
server_args.enable_cfg_parallel = False
server_args.enable_torch_compile = False
server_args.enable_breakable_cuda_graph = True
server_args.pipeline_config.task_type = ModelTaskType.TI2I
references = ["first.png", "second.png"]
with patch(
"sglang.multimodal_gen.runtime.warmup_request_builder.get_model_sampling_defaults",
return_value=Flux3ImageSamplingParams(
width=512,
height=512,
image_path=references,
flux3_mode="i2i",
num_inference_steps=16,
),
):
req = build_warmup_reqs(
server_args,
warmup_resolutions=["512x512"],
server_based_warmup=True,
)[0]
self.assertEqual(req.image_path, references)
self.assertEqual(req.flux3_mode, "i2i")
self.assertEqual(req.sampling_params._explicit_fields, {"width", "height"})
def test_server_based_warmup_keeps_required_image_input(self):
server_args = MagicMock()
server_args.warmup_steps = 1
@@ -198,10 +198,18 @@ def test_packed_cfg_forward_matches_flux3_transformer_per_sample(
packed.rope = {mode: native.rope(ids) for mode, ids in packed.ids.items()}
cached = native(packed)["image"].float()
torch.testing.assert_close(cached, predicted, rtol=0, atol=0)
packed.static_hidden = {}
for _ in range(2):
reused = native(packed)["image"].float()
torch.testing.assert_close(reused, predicted, rtol=0, atol=0)
packed.static = flux3_image.StaticStreams()
modulation = native.single_stream_modulations["txt"]
with (
mock.patch.object(
native.txt_in, "forward", wraps=native.txt_in.forward
) as embed,
mock.patch.object(modulation, "forward", wraps=modulation.forward) as mod,
):
for _ in range(2):
reused = native(packed)["image"].float()
torch.testing.assert_close(reused, predicted, rtol=0, atol=0)
assert embed.call_count == mod.call_count == 1
for mode, rows in packed.sample(1).rope.items():
start, end = packed.mode_indices[mode][1]
assert torch.equal(rows, packed.rope[mode][start:end])
@@ -231,7 +239,7 @@ def test_packed_cfg_forward_matches_flux3_transformer_per_sample(
packed.timesteps["image"].fill_(0.3)
packed.data["image"].mul_(0.9)
reused = native(packed)["image"]
packed.static_hidden = None
packed.static = None
uncached = native(packed)["image"]
torch.testing.assert_close(reused, uncached, rtol=0, atol=0)
@@ -21,6 +21,7 @@ from torch.nn.attention import SDPBackend, sdpa_kernel
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import (
_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS,
SDPAImpl,
torch_default_sdpa_backends,
)
NUM_HEADS, HEAD_DIM, SEQ_LEN = 8, 128, 512
@@ -92,3 +93,19 @@ def test_opting_out_leaves_backend_selection_alone():
def test_cpu_tensors_do_not_get_a_cuda_context():
cpu_q = torch.randn(1, NUM_HEADS, 16, HEAD_DIM)
assert isinstance(_impl(True)._sdpa_context(cpu_q), type(nullcontext()))
def test_reference_sdpa_does_not_inherit_a_previous_attention_priority():
q, k, v = _qkv()
with sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True):
with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
try:
F.scaled_dot_product_attention(q, k, v)
except RuntimeError:
pytest.skip("no cuDNN attention kernel for this shape")
for priority in (
_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS,
list(reversed(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS)),
):
with sdpa_kernel(priority, set_priority=True):
assert _kernel_under(torch_default_sdpa_backends(), q, k, v) == "cudnn"