[diffusion] feat: enable cuda graphs for observation encoding and denoising steps for action models (#42171)

This commit is contained in:
WenhaoZhang
2026-10-04 22:22:08 +08:00
committed by GitHub
parent 279d3f3888
commit affa261e3d
6 changed files with 371 additions and 62 deletions
@@ -267,6 +267,18 @@ class Flux3ActionPipelineConfig(PipelineConfig):
text_output_layers: tuple[int, ...] = (4, 8, 12, 16, 20, 24, 28, 32)
# Text contexts cached per caption, bounded by their total token count.
caption_cache_max_tokens: int = 1 << 16
# CUDA graphs of the observation encoding and of each denoising step.
enable_cuda_graph: bool = True
# Per stage; one entry per (prompt length bucket, CFG) combination.
cuda_graph_max_entries: int = 4
def cuda_graph_enabled(self, server_args: Any, component: str) -> bool:
"""Graphs need resident weights and no cross-rank collectives."""
return (
self.enable_cuda_graph
and server_args.num_gpus == 1
and not server_args.should_start_component_on_cpu(component)
)
def validate_server_args(self, server_args: Any) -> None:
super().validate_server_args(server_args)
@@ -135,13 +135,13 @@ _LN_MODULATE = BitExactFusionGate("FLUX 3 fused LN+modulate", per_signature=True
_SWIGLU = BitExactFusionGate("FLUX 3 fused SwiGLU", per_signature=True)
def _eager_fast_path_allowed(x: torch.Tensor) -> bool:
return (
x.is_cuda
and x.dtype is torch.bfloat16
and not torch.compiler.is_compiling()
and not torch.cuda.is_current_stream_capturing()
)
def _fast_path_allowed(x: torch.Tensor) -> bool:
return x.is_cuda and x.dtype is torch.bfloat16 and not torch.compiler.is_compiling()
def _can_verify() -> bool:
# First-sight verification runs the eager chain and a host sync.
return not torch.cuda.is_current_stream_capturing()
def _norm_modulate(
@@ -151,12 +151,14 @@ def _norm_modulate(
scale_row, shift_row = scale[:, 0], shift[:, 0]
if (
_LN_MODULATE.disabled
or not _eager_fast_path_allowed(x)
or not _fast_path_allowed(x)
or not is_plain_layer_norm(norm, x.shape[-1])
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
):
return (1 + scale) * norm(x) + shift
sig = (x.device, x.shape[0], x.shape[-1], norm.eps)
if not _LN_MODULATE.is_verified(sig) and not _can_verify():
return (1 + scale) * norm(x) + shift
try:
out = fused_layernorm_modulate_raw(x, scale_row, shift_row, norm.eps)
except Exception as exc:
@@ -176,9 +178,11 @@ def _norm_modulate(
def _swiglu(packed: torch.Tensor) -> torch.Tensor:
"""``silu(gate) * value`` of a packed ``[gate | value]`` projection."""
gate, value = packed.chunk(2, dim=-1)
if _SWIGLU.disabled or not _eager_fast_path_allowed(packed):
if _SWIGLU.disabled or not _fast_path_allowed(packed):
return F.silu(gate) * value
sig = (packed.device, packed.shape[-1], packed.stride(-2))
if not _SWIGLU.is_verified(sig) and not _can_verify():
return F.silu(gate) * value
try:
out = fused_packed_silu_mul_bitexact(packed)
except Exception as exc:
@@ -197,7 +201,7 @@ def _swiglu(packed: torch.Tensor) -> torch.Tensor:
def _fused_qknorm_rope_enabled(q: torch.Tensor, head_dim: int) -> bool:
return (
_eager_fast_path_allowed(q)
_fast_path_allowed(q)
and os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower()
not in ("0", "false", "off", "no")
and can_use_fused_inplace_qknorm_rope(
@@ -186,13 +186,20 @@ class Flux3ActionPipeline(ComposedPipelineBase):
)
self.add_stage(
Flux3ActionObservationEncodingStage(
config, transformer=transformer, vae=self.get_module("vae")
config,
transformer=transformer,
vae=self.get_module("vae"),
use_cuda_graph=config.cuda_graph_enabled(server_args, "transformer")
and config.cuda_graph_enabled(server_args, "vae"),
),
"flux3_action_observation_encoding",
)
self.add_stage(
Flux3ActionDenoisingStage(
config, transformer, scheduler=self.get_module("scheduler")
config,
transformer,
scheduler=self.get_module("scheduler"),
use_cuda_graph=config.cuda_graph_enabled(server_args, "transformer"),
),
"flux3_action_denoise",
)
@@ -16,6 +16,7 @@ conditional and unconditional CFG passes, since mode blocks never see text).
from __future__ import annotations
import copy
import functools
import time
from collections import OrderedDict
from collections.abc import Callable
@@ -60,6 +61,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import (
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.vla.cuda_graph import VLATensorGraphRunner
logger = init_logger(__name__)
@@ -88,14 +90,22 @@ def _cartesian_ids(
return torch.cartesian_prod(t, torch.arange(h), torch.arange(w), l_coord)
def pack_video(latent: torch.Tensor, first_frame: int, fps: float):
"""Latent ``(1, C, T, h, w)`` -> tokens ``(1, T*h*w, C)`` and ids; frame ``i`` at ``i * 4 / fps`` s."""
_, _, t, h, w = latent.shape
def video_ids(*, frames: int, h: int, w: int, first_frame: int, fps: float):
"""Ids ``(1, frames*h*w, 4)`` of latent frames; frame ``i`` at ``i * 4 / fps`` s."""
seconds = (
torch.arange(first_frame, first_frame + t).float() * TEMPORAL_DOWNSAMPLE / fps
torch.arange(first_frame, first_frame + frames).float()
* TEMPORAL_DOWNSAMPLE
/ fps
)
ids = _cartesian_ids(t=_times_to_ids(seconds), h=h, w=w, l_coord=torch.arange(1))
return rearrange(latent, "b c t h w -> b (t h w) c"), ids[None]
return ids[None]
def pack_video(latent: torch.Tensor, first_frame: int, fps: float):
"""Latent ``(1, C, T, h, w)`` -> tokens ``(1, T*h*w, C)`` and ids."""
_, _, t, h, w = latent.shape
ids = video_ids(frames=t, h=h, w=w, first_frame=first_frame, fps=fps)
return rearrange(latent, "b c t h w -> b (t h w) c"), ids
def pack_action(values: torch.Tensor, seconds: torch.Tensor):
@@ -403,6 +413,34 @@ def cosmos_unipc(
return samples
# Flux3SegmentState as graph tensors: hidden, rope, vec and the 3 joint modulations.
_STATE_TENSORS = 6
def _state_tensors(state: Flux3SegmentState) -> tuple[torch.Tensor, ...]:
return (state.hidden, state.rope, state.vec, *state.joint_mod)
def _state_from_tensors(
name: str, tensors: tuple[torch.Tensor, ...]
) -> Flux3SegmentState:
hidden, rope, vec, *joint_mod = tensors
return Flux3SegmentState(
name=name, hidden=hidden, rope=rope, vec=vec, joint_mod=tuple(joint_mod)
)
def _states_from_tensors(
names: list[str], tensors: tuple[torch.Tensor, ...]
) -> list[Flux3SegmentState]:
return [
_state_from_tensors(
name, tensors[i * _STATE_TENSORS : (i + 1) * _STATE_TENSORS]
)
for i, name in enumerate(names)
]
def _cfg_parallel_policy(
contexts: list[Flux3SegmentState], server_args: ServerArgs
) -> CFGPolicy | None:
@@ -557,11 +595,17 @@ class Flux3ActionObservationEncodingStage(PipelineStage):
config: Flux3ActionPipelineConfig,
transformer: Flux3Transformer,
vae: Flux3VideoVAE,
use_cuda_graph: bool = False,
):
super().__init__()
self.config = config
self.transformer = transformer
self.vae = vae
self._graph = VLATensorGraphRunner(
"FLUX 3 observation",
enabled=use_cuda_graph,
max_entries=config.cuda_graph_max_entries,
)
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
@@ -585,6 +629,41 @@ class Flux3ActionObservationEncodingStage(PipelineStage):
values = (token[None, :, None] * cfg.action_scale).to(device)
return pack_action(values, seconds=torch.zeros(1))
def _encode_frame(self, frame: torch.Tensor) -> torch.Tensor:
h, w = self.config.latent_hw
latent = self.vae.encode_frame(frame)[..., :h, :w]
return rearrange(latent, "b c t h w -> b (t h w) c")
def _encode_streams(
self,
video: torch.Tensor,
frame_ids: torch.Tensor,
action: torch.Tensor,
action_ids: torch.Tensor,
zero: torch.Tensor,
) -> tuple[torch.Tensor, ...]:
video_cond = self.transformer.encode_stream(
name="video_cond", x=video, ids=frame_ids, timesteps=zero
)
action_cond = self.transformer.encode_stream(
name=f"{self.config.action_modality}_cond",
x=action,
ids=action_ids,
timesteps=zero,
)
return (*_state_tensors(video_cond), *_state_tensors(action_cond))
def _encode(self, frame: torch.Tensor, *stream_inputs: torch.Tensor):
return self._encode_streams(self._encode_frame(frame), *stream_inputs)
def _vae_use(self):
return self.use_declared_component(component_name="vae", module=self.vae)
def _transformer_use(self):
return self.use_declared_component(
component_name="transformer", module=self.transformer
)
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
start = time.perf_counter()
cfg = self.config
@@ -592,29 +671,30 @@ class Flux3ActionObservationEncodingStage(PipelineStage):
state = vla_state(batch)
observation: Flux3ActionObservation = state["flux3_observation"]
h, w = cfg.latent_hw
with self.use_declared_component(component_name="vae", module=self.vae):
frame = observation.canvas.to(device, torch.bfloat16)[None]
latent = self.vae.encode_frame(frame)[..., :h, :w]
video, video_ids = pack_video(latent, first_frame=0, fps=cfg.fps)
frame_ids = video_ids(frames=1, h=h, w=w, first_frame=0, fps=cfg.fps)
action, action_ids = self._state_tokens(observation.state, device)
zero = torch.zeros(1, device=device)
with (
self.use_declared_component(
component_name="transformer", module=self.transformer
),
set_forward_context(current_timestep=0, attn_metadata=None),
):
state["flux3_conditioning"] = [
self.transformer.encode_stream(
name="video_cond", x=video, ids=video_ids.to(device), timesteps=zero
),
self.transformer.encode_stream(
name=f"{cfg.action_modality}_cond",
x=action,
ids=action_ids.to(device),
timesteps=zero,
),
]
frame = observation.canvas.to(device, torch.bfloat16)[None]
stream_inputs = (
frame_ids.to(device),
action,
action_ids.to(device),
torch.zeros(1, device=device),
)
forward_context = set_forward_context(current_timestep=0, attn_metadata=None)
if self._graph.enabled:
# Both components are resident whenever graphs are enabled.
with self._vae_use(), self._transformer_use(), forward_context:
captured = self._graph.run(self._encode, (frame, *stream_inputs))
# Replays overwrite the outputs; the states outlive this request.
tensors = tuple(t.clone() for t in captured)
else:
with self._vae_use():
video = self._encode_frame(frame)
with self._transformer_use(), forward_context:
tensors = self._encode_streams(video, *stream_inputs)
state["flux3_conditioning"] = _states_from_tensors(
["video_cond", f"{cfg.action_modality}_cond"], tensors
)
vla_timings(batch)["observation_ms"] = (time.perf_counter() - start) * 1000
return batch
@@ -627,11 +707,17 @@ class Flux3ActionDenoisingStage(PipelineStage):
config: Flux3ActionPipelineConfig,
transformer: Flux3Transformer,
scheduler: FlowUniPCMultistepScheduler,
use_cuda_graph: bool = False,
):
super().__init__()
self.config = config
self.transformer = transformer
self.scheduler = scheduler
self._graph = VLATensorGraphRunner(
"FLUX 3 denoise",
enabled=use_cuda_graph,
max_entries=config.cuda_graph_max_entries,
)
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
@@ -663,6 +749,68 @@ class Flux3ActionDenoisingStage(PipelineStage):
cfg.action_modality: action_ids,
}
def _target_streams(
self,
samples: tuple[torch.Tensor, ...],
ropes: tuple[torch.Tensor, ...],
t: torch.Tensor,
*,
order: list[str],
) -> list[Flux3SegmentState]:
"""Noised ``video`` and action streams through their mode blocks."""
return [
self.transformer.encode_stream(
name=name,
x=x.to(torch.bfloat16),
ids=None,
timesteps=t,
rope=rope,
)
for name, x, rope in zip(order, samples, ropes, strict=True)
]
def _denoise_pass(
self,
context: Flux3SegmentState,
targets: list[Flux3SegmentState],
conditioning: list[Flux3SegmentState],
*,
order: list[str],
) -> tuple[torch.Tensor, ...]:
video, action = targets
video_cond, action_cond = conditioning
# Joint sequence order: video, video_cond, action, action_cond.
out = self.transformer.denoise(
context=context,
streams=[video, video_cond, action, action_cond],
targets=order,
)
return tuple(out[k] for k in order)
def _denoise_step(
self,
video: torch.Tensor,
action: torch.Tensor,
t: torch.Tensor,
video_rope: torch.Tensor,
action_rope: torch.Tensor,
*states: torch.Tensor,
order: list[str],
state_names: list[str],
) -> tuple[torch.Tensor, ...]:
"""Every CFG pass of one step, flattened; ``states`` = conditioning + contexts."""
video_cond, action_cond, *contexts = _states_from_tensors(state_names, states)
targets = self._target_streams(
(video, action), (video_rope, action_rope), t, order=order
)
return tuple(
p
for ctx in contexts
for p in self._denoise_pass(
ctx, targets, [video_cond, action_cond], order=order
)
)
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
cfg = self.config
device = get_local_torch_device()
@@ -682,10 +830,14 @@ class Flux3ActionDenoisingStage(PipelineStage):
start = time.perf_counter()
samples, ids = self._noised_streams(seed)
samples = {k: v.to(device) for k, v in samples.items()}
ropes = {k: self.transformer.rope(v.to(device)) for k, v in ids.items()}
video_cond, action_cond = conditioning
order = list(samples) # joint sequence: video, video_cond, action, action_cond
order = list(samples)
ropes = tuple(self.transformer.rope(ids[k].to(device)) for k in order)
cfg_policy = _cfg_parallel_policy(contexts, server_args)
state_names = [s.name for s in (*conditioning, *contexts)]
states = tuple(t for s in (*conditioning, *contexts) for t in _state_tensors(s))
denoise_step = functools.partial(
self._denoise_step, order=order, state_names=state_names
)
step = 0
def predict(
@@ -693,34 +845,34 @@ class Flux3ActionDenoisingStage(PipelineStage):
) -> dict[str, torch.Tensor]:
nonlocal step
timestep = torch.full((1,), t, device=device, dtype=torch.float32)
current_streams = tuple(current[k] for k in order)
with set_forward_context(
current_timestep=step, attn_metadata=None, forward_batch=batch
):
targets = {
name: self.transformer.encode_stream(
name=name,
x=current[name].to(torch.bfloat16),
ids=None,
timesteps=timestep,
rope=ropes[name],
)
for name in order
}
streams = [targets["video"], video_cond, targets[order[1]], action_cond]
def denoise(ctx: Flux3SegmentState) -> tuple[torch.Tensor, ...]:
out = self.transformer.denoise(
context=ctx, streams=streams, targets=order
)
return tuple(out[k] for k in order)
if cfg_policy is not None:
targets = self._target_streams(
current_streams, ropes, timestep, order=order
)
# Each rank runs one branch; all ranks get both predictions.
raw = run_cfg_parallel(
cfg_policy, lambda branch: denoise(branch.kwargs["context"])
cfg_policy,
lambda branch: self._denoise_pass(
branch.kwargs["context"],
targets,
conditioning,
order=order,
),
)
else:
raw = [denoise(ctx) for ctx in contexts]
flat = self._graph.run(
denoise_step,
(*current_streams, timestep, *ropes, *states),
key=(tuple(order), tuple(state_names)),
)
raw = [
flat[i : i + len(order)]
for i in range(0, len(flat), len(order))
]
preds = [dict(zip(order, p)) for p in raw]
step += 1
if len(preds) == 1:
@@ -6,6 +6,7 @@ from collections import OrderedDict
from dataclasses import dataclass
from typing import Any, Callable
import msgspec
import torch
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
@@ -490,3 +491,99 @@ class VLADenoiseGraphRunner:
exc_info=True,
)
return step_fn(prefix_context, x_t, timestep)
class _CapturedTensorGraph(msgspec.Struct):
graph: torch.cuda.CUDAGraph
static_inputs: tuple[torch.Tensor, ...]
static_outputs: tuple[torch.Tensor, ...]
class VLATensorGraphRunner:
"""CUDA graphs of a function of tensors, keyed by the input shapes and ``key``.
Every call copies ``inputs`` into the graph's static buffers, so ``fn`` must
read its tensors only from its arguments (anything else is baked in by
address). The returned tensors are overwritten by the next replay.
"""
def __init__(self, name: str, *, enabled: bool, max_entries: int):
self.name = name
self.enabled = enabled and max_entries > 0
self._cache = _BoundedCaptureCache(name, max_entries, evict_on_miss=True)
self._disabled_signatures: set[Any] = set()
self._capture_stream: torch.cuda.Stream | None = None
self._graph_pool: Any = None
def cache_info(self) -> VLAGraphCacheInfo:
return self._cache.info()
def clear(self) -> None:
self._cache.clear()
def _capture(
self,
fn: Callable[..., tuple[torch.Tensor, ...]],
inputs: tuple[torch.Tensor, ...],
) -> _CapturedTensorGraph:
device = inputs[0].device
device_module = torch.get_device_module(device)
if self._capture_stream is None:
self._capture_stream = device_module.Stream(device=device)
if self._graph_pool is None:
self._graph_pool = get_or_create_global_graph_memory_pool(device_module)
set_graph_pool_id(self._graph_pool)
static_inputs = tuple(t.detach().clone() for t in inputs)
# Warm up lazy kernels, workspaces and first-sight fusion checks.
device_module.synchronize()
with device_module.stream(self._capture_stream):
fn(*static_inputs)
self._capture_stream.synchronize()
graph = torch.cuda.CUDAGraph()
with device_module.graph(
cuda_graph=graph, pool=self._graph_pool, stream=self._capture_stream
):
static_outputs = tuple(fn(*static_inputs))
self._capture_stream.synchronize()
logger.info(
"Captured VLA %s CUDA graph for %s",
self.name,
[tuple(t.shape) for t in inputs],
)
return _CapturedTensorGraph(
graph=graph, static_inputs=static_inputs, static_outputs=static_outputs
)
def run(
self,
fn: Callable[..., tuple[torch.Tensor, ...]],
inputs: tuple[torch.Tensor, ...],
*,
key: Any = None,
) -> tuple[torch.Tensor, ...]:
if not self.enabled or not inputs or inputs[0].device.type != "cuda":
return tuple(fn(*inputs))
signature = (key, tuple((tuple(t.shape), t.dtype) for t in inputs))
if signature in self._disabled_signatures:
return tuple(fn(*inputs))
try:
captured = self._cache.get(signature)
if captured is None:
self._cache.prepare_admission(signature)
captured = self._capture(fn, inputs)
self._cache.put(signature, captured)
for static, value in zip(captured.static_inputs, inputs, strict=True):
static.copy_(value)
captured.graph.replay()
return captured.static_outputs
except Exception:
self._disabled_signatures.add(signature)
self._cache.discard(signature)
self._cache.mark_failure()
logger.warning(
"VLA %s CUDA graph disabled for signature %s",
self.name,
signature,
exc_info=True,
)
return tuple(fn(*inputs))
@@ -647,6 +647,43 @@ def test_dit_cached_streams_match_full_forward():
assert full["x_act_cond"].shape == (1, 1, 3)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA graphs")
def test_cuda_graph_replays_match_eager_denoise_steps():
"""Replays with fresh inputs must match eager steps bit for bit."""
from sglang.multimodal_gen.runtime.managers.forward_context import (
set_forward_context,
)
from sglang.multimodal_gen.runtime.models.dits.flux3 import Flux3Transformer
from sglang.multimodal_gen.runtime.vla.cuda_graph import VLATensorGraphRunner
_init_single_process_parallel()
torch.manual_seed(0)
model = Flux3Transformer(Flux3DiTConfig(arch_config=_tiny_arch())).cuda().bfloat16()
for p in model.parameters():
torch.nn.init.normal_(p, std=0.05)
_, ids = pack_video(torch.zeros(1, 96, 2, 2, 3), first_frame=1, fps=15.0)
rope = model.rope(ids.cuda())
def step(x, t, rope):
state = model.encode_stream(name="video", x=x, ids=None, timesteps=t, rope=rope)
return (state.hidden,)
runner = VLATensorGraphRunner("test", enabled=True, max_entries=2)
with torch.no_grad(), set_forward_context(current_timestep=0, attn_metadata=None):
for seed, t in ((1, 0.7), (2, 0.3)):
torch.manual_seed(seed)
inputs = (
torch.randn(1, 12, 96, device="cuda", dtype=torch.bfloat16),
torch.full((1,), t, device="cuda"),
rope,
)
(eager,) = step(*inputs)
(graph,) = runner.run(step, inputs)
assert torch.equal(graph, eager)
info = runner.cache_info()
assert (info.captures, info.hits, info.failures) == (1, 1, 0)
@pytest.mark.parametrize("fnuz", [False, True])
@pytest.mark.parametrize("tuple_output", [False, True])
def test_fp8r_native_format_preserves_values(monkeypatch, fnuz, tuple_output):