[diffusion] feat: enable cuda graphs for observation encoding and denoising steps for action models (#42171)
This commit is contained in:
@@ -267,6 +267,18 @@ class Flux3ActionPipelineConfig(PipelineConfig):
|
||||
text_output_layers: tuple[int, ...] = (4, 8, 12, 16, 20, 24, 28, 32)
|
||||
# Text contexts cached per caption, bounded by their total token count.
|
||||
caption_cache_max_tokens: int = 1 << 16
|
||||
# CUDA graphs of the observation encoding and of each denoising step.
|
||||
enable_cuda_graph: bool = True
|
||||
# Per stage; one entry per (prompt length bucket, CFG) combination.
|
||||
cuda_graph_max_entries: int = 4
|
||||
|
||||
def cuda_graph_enabled(self, server_args: Any, component: str) -> bool:
|
||||
"""Graphs need resident weights and no cross-rank collectives."""
|
||||
return (
|
||||
self.enable_cuda_graph
|
||||
and server_args.num_gpus == 1
|
||||
and not server_args.should_start_component_on_cpu(component)
|
||||
)
|
||||
|
||||
def validate_server_args(self, server_args: Any) -> None:
|
||||
super().validate_server_args(server_args)
|
||||
|
||||
@@ -135,13 +135,13 @@ _LN_MODULATE = BitExactFusionGate("FLUX 3 fused LN+modulate", per_signature=True
|
||||
_SWIGLU = BitExactFusionGate("FLUX 3 fused SwiGLU", per_signature=True)
|
||||
|
||||
|
||||
def _eager_fast_path_allowed(x: torch.Tensor) -> bool:
|
||||
return (
|
||||
x.is_cuda
|
||||
and x.dtype is torch.bfloat16
|
||||
and not torch.compiler.is_compiling()
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
)
|
||||
def _fast_path_allowed(x: torch.Tensor) -> bool:
|
||||
return x.is_cuda and x.dtype is torch.bfloat16 and not torch.compiler.is_compiling()
|
||||
|
||||
|
||||
def _can_verify() -> bool:
|
||||
# First-sight verification runs the eager chain and a host sync.
|
||||
return not torch.cuda.is_current_stream_capturing()
|
||||
|
||||
|
||||
def _norm_modulate(
|
||||
@@ -151,12 +151,14 @@ def _norm_modulate(
|
||||
scale_row, shift_row = scale[:, 0], shift[:, 0]
|
||||
if (
|
||||
_LN_MODULATE.disabled
|
||||
or not _eager_fast_path_allowed(x)
|
||||
or not _fast_path_allowed(x)
|
||||
or not is_plain_layer_norm(norm, x.shape[-1])
|
||||
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
):
|
||||
return (1 + scale) * norm(x) + shift
|
||||
sig = (x.device, x.shape[0], x.shape[-1], norm.eps)
|
||||
if not _LN_MODULATE.is_verified(sig) and not _can_verify():
|
||||
return (1 + scale) * norm(x) + shift
|
||||
try:
|
||||
out = fused_layernorm_modulate_raw(x, scale_row, shift_row, norm.eps)
|
||||
except Exception as exc:
|
||||
@@ -176,9 +178,11 @@ def _norm_modulate(
|
||||
def _swiglu(packed: torch.Tensor) -> torch.Tensor:
|
||||
"""``silu(gate) * value`` of a packed ``[gate | value]`` projection."""
|
||||
gate, value = packed.chunk(2, dim=-1)
|
||||
if _SWIGLU.disabled or not _eager_fast_path_allowed(packed):
|
||||
if _SWIGLU.disabled or not _fast_path_allowed(packed):
|
||||
return F.silu(gate) * value
|
||||
sig = (packed.device, packed.shape[-1], packed.stride(-2))
|
||||
if not _SWIGLU.is_verified(sig) and not _can_verify():
|
||||
return F.silu(gate) * value
|
||||
try:
|
||||
out = fused_packed_silu_mul_bitexact(packed)
|
||||
except Exception as exc:
|
||||
@@ -197,7 +201,7 @@ def _swiglu(packed: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
def _fused_qknorm_rope_enabled(q: torch.Tensor, head_dim: int) -> bool:
|
||||
return (
|
||||
_eager_fast_path_allowed(q)
|
||||
_fast_path_allowed(q)
|
||||
and os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower()
|
||||
not in ("0", "false", "off", "no")
|
||||
and can_use_fused_inplace_qknorm_rope(
|
||||
|
||||
@@ -186,13 +186,20 @@ class Flux3ActionPipeline(ComposedPipelineBase):
|
||||
)
|
||||
self.add_stage(
|
||||
Flux3ActionObservationEncodingStage(
|
||||
config, transformer=transformer, vae=self.get_module("vae")
|
||||
config,
|
||||
transformer=transformer,
|
||||
vae=self.get_module("vae"),
|
||||
use_cuda_graph=config.cuda_graph_enabled(server_args, "transformer")
|
||||
and config.cuda_graph_enabled(server_args, "vae"),
|
||||
),
|
||||
"flux3_action_observation_encoding",
|
||||
)
|
||||
self.add_stage(
|
||||
Flux3ActionDenoisingStage(
|
||||
config, transformer, scheduler=self.get_module("scheduler")
|
||||
config,
|
||||
transformer,
|
||||
scheduler=self.get_module("scheduler"),
|
||||
use_cuda_graph=config.cuda_graph_enabled(server_args, "transformer"),
|
||||
),
|
||||
"flux3_action_denoise",
|
||||
)
|
||||
|
||||
+202
-50
@@ -16,6 +16,7 @@ conditional and unconditional CFG passes, since mode blocks never see text).
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import functools
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
@@ -60,6 +61,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.vla.cuda_graph import VLATensorGraphRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -88,14 +90,22 @@ def _cartesian_ids(
|
||||
return torch.cartesian_prod(t, torch.arange(h), torch.arange(w), l_coord)
|
||||
|
||||
|
||||
def pack_video(latent: torch.Tensor, first_frame: int, fps: float):
|
||||
"""Latent ``(1, C, T, h, w)`` -> tokens ``(1, T*h*w, C)`` and ids; frame ``i`` at ``i * 4 / fps`` s."""
|
||||
_, _, t, h, w = latent.shape
|
||||
def video_ids(*, frames: int, h: int, w: int, first_frame: int, fps: float):
|
||||
"""Ids ``(1, frames*h*w, 4)`` of latent frames; frame ``i`` at ``i * 4 / fps`` s."""
|
||||
seconds = (
|
||||
torch.arange(first_frame, first_frame + t).float() * TEMPORAL_DOWNSAMPLE / fps
|
||||
torch.arange(first_frame, first_frame + frames).float()
|
||||
* TEMPORAL_DOWNSAMPLE
|
||||
/ fps
|
||||
)
|
||||
ids = _cartesian_ids(t=_times_to_ids(seconds), h=h, w=w, l_coord=torch.arange(1))
|
||||
return rearrange(latent, "b c t h w -> b (t h w) c"), ids[None]
|
||||
return ids[None]
|
||||
|
||||
|
||||
def pack_video(latent: torch.Tensor, first_frame: int, fps: float):
|
||||
"""Latent ``(1, C, T, h, w)`` -> tokens ``(1, T*h*w, C)`` and ids."""
|
||||
_, _, t, h, w = latent.shape
|
||||
ids = video_ids(frames=t, h=h, w=w, first_frame=first_frame, fps=fps)
|
||||
return rearrange(latent, "b c t h w -> b (t h w) c"), ids
|
||||
|
||||
|
||||
def pack_action(values: torch.Tensor, seconds: torch.Tensor):
|
||||
@@ -403,6 +413,34 @@ def cosmos_unipc(
|
||||
return samples
|
||||
|
||||
|
||||
# Flux3SegmentState as graph tensors: hidden, rope, vec and the 3 joint modulations.
|
||||
_STATE_TENSORS = 6
|
||||
|
||||
|
||||
def _state_tensors(state: Flux3SegmentState) -> tuple[torch.Tensor, ...]:
|
||||
return (state.hidden, state.rope, state.vec, *state.joint_mod)
|
||||
|
||||
|
||||
def _state_from_tensors(
|
||||
name: str, tensors: tuple[torch.Tensor, ...]
|
||||
) -> Flux3SegmentState:
|
||||
hidden, rope, vec, *joint_mod = tensors
|
||||
return Flux3SegmentState(
|
||||
name=name, hidden=hidden, rope=rope, vec=vec, joint_mod=tuple(joint_mod)
|
||||
)
|
||||
|
||||
|
||||
def _states_from_tensors(
|
||||
names: list[str], tensors: tuple[torch.Tensor, ...]
|
||||
) -> list[Flux3SegmentState]:
|
||||
return [
|
||||
_state_from_tensors(
|
||||
name, tensors[i * _STATE_TENSORS : (i + 1) * _STATE_TENSORS]
|
||||
)
|
||||
for i, name in enumerate(names)
|
||||
]
|
||||
|
||||
|
||||
def _cfg_parallel_policy(
|
||||
contexts: list[Flux3SegmentState], server_args: ServerArgs
|
||||
) -> CFGPolicy | None:
|
||||
@@ -557,11 +595,17 @@ class Flux3ActionObservationEncodingStage(PipelineStage):
|
||||
config: Flux3ActionPipelineConfig,
|
||||
transformer: Flux3Transformer,
|
||||
vae: Flux3VideoVAE,
|
||||
use_cuda_graph: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.transformer = transformer
|
||||
self.vae = vae
|
||||
self._graph = VLATensorGraphRunner(
|
||||
"FLUX 3 observation",
|
||||
enabled=use_cuda_graph,
|
||||
max_entries=config.cuda_graph_max_entries,
|
||||
)
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
@@ -585,6 +629,41 @@ class Flux3ActionObservationEncodingStage(PipelineStage):
|
||||
values = (token[None, :, None] * cfg.action_scale).to(device)
|
||||
return pack_action(values, seconds=torch.zeros(1))
|
||||
|
||||
def _encode_frame(self, frame: torch.Tensor) -> torch.Tensor:
|
||||
h, w = self.config.latent_hw
|
||||
latent = self.vae.encode_frame(frame)[..., :h, :w]
|
||||
return rearrange(latent, "b c t h w -> b (t h w) c")
|
||||
|
||||
def _encode_streams(
|
||||
self,
|
||||
video: torch.Tensor,
|
||||
frame_ids: torch.Tensor,
|
||||
action: torch.Tensor,
|
||||
action_ids: torch.Tensor,
|
||||
zero: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
video_cond = self.transformer.encode_stream(
|
||||
name="video_cond", x=video, ids=frame_ids, timesteps=zero
|
||||
)
|
||||
action_cond = self.transformer.encode_stream(
|
||||
name=f"{self.config.action_modality}_cond",
|
||||
x=action,
|
||||
ids=action_ids,
|
||||
timesteps=zero,
|
||||
)
|
||||
return (*_state_tensors(video_cond), *_state_tensors(action_cond))
|
||||
|
||||
def _encode(self, frame: torch.Tensor, *stream_inputs: torch.Tensor):
|
||||
return self._encode_streams(self._encode_frame(frame), *stream_inputs)
|
||||
|
||||
def _vae_use(self):
|
||||
return self.use_declared_component(component_name="vae", module=self.vae)
|
||||
|
||||
def _transformer_use(self):
|
||||
return self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer
|
||||
)
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
start = time.perf_counter()
|
||||
cfg = self.config
|
||||
@@ -592,29 +671,30 @@ class Flux3ActionObservationEncodingStage(PipelineStage):
|
||||
state = vla_state(batch)
|
||||
observation: Flux3ActionObservation = state["flux3_observation"]
|
||||
h, w = cfg.latent_hw
|
||||
with self.use_declared_component(component_name="vae", module=self.vae):
|
||||
frame = observation.canvas.to(device, torch.bfloat16)[None]
|
||||
latent = self.vae.encode_frame(frame)[..., :h, :w]
|
||||
video, video_ids = pack_video(latent, first_frame=0, fps=cfg.fps)
|
||||
frame_ids = video_ids(frames=1, h=h, w=w, first_frame=0, fps=cfg.fps)
|
||||
action, action_ids = self._state_tokens(observation.state, device)
|
||||
zero = torch.zeros(1, device=device)
|
||||
with (
|
||||
self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer
|
||||
),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
state["flux3_conditioning"] = [
|
||||
self.transformer.encode_stream(
|
||||
name="video_cond", x=video, ids=video_ids.to(device), timesteps=zero
|
||||
),
|
||||
self.transformer.encode_stream(
|
||||
name=f"{cfg.action_modality}_cond",
|
||||
x=action,
|
||||
ids=action_ids.to(device),
|
||||
timesteps=zero,
|
||||
),
|
||||
]
|
||||
frame = observation.canvas.to(device, torch.bfloat16)[None]
|
||||
stream_inputs = (
|
||||
frame_ids.to(device),
|
||||
action,
|
||||
action_ids.to(device),
|
||||
torch.zeros(1, device=device),
|
||||
)
|
||||
forward_context = set_forward_context(current_timestep=0, attn_metadata=None)
|
||||
if self._graph.enabled:
|
||||
# Both components are resident whenever graphs are enabled.
|
||||
with self._vae_use(), self._transformer_use(), forward_context:
|
||||
captured = self._graph.run(self._encode, (frame, *stream_inputs))
|
||||
# Replays overwrite the outputs; the states outlive this request.
|
||||
tensors = tuple(t.clone() for t in captured)
|
||||
else:
|
||||
with self._vae_use():
|
||||
video = self._encode_frame(frame)
|
||||
with self._transformer_use(), forward_context:
|
||||
tensors = self._encode_streams(video, *stream_inputs)
|
||||
state["flux3_conditioning"] = _states_from_tensors(
|
||||
["video_cond", f"{cfg.action_modality}_cond"], tensors
|
||||
)
|
||||
vla_timings(batch)["observation_ms"] = (time.perf_counter() - start) * 1000
|
||||
return batch
|
||||
|
||||
@@ -627,11 +707,17 @@ class Flux3ActionDenoisingStage(PipelineStage):
|
||||
config: Flux3ActionPipelineConfig,
|
||||
transformer: Flux3Transformer,
|
||||
scheduler: FlowUniPCMultistepScheduler,
|
||||
use_cuda_graph: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.transformer = transformer
|
||||
self.scheduler = scheduler
|
||||
self._graph = VLATensorGraphRunner(
|
||||
"FLUX 3 denoise",
|
||||
enabled=use_cuda_graph,
|
||||
max_entries=config.cuda_graph_max_entries,
|
||||
)
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
@@ -663,6 +749,68 @@ class Flux3ActionDenoisingStage(PipelineStage):
|
||||
cfg.action_modality: action_ids,
|
||||
}
|
||||
|
||||
def _target_streams(
|
||||
self,
|
||||
samples: tuple[torch.Tensor, ...],
|
||||
ropes: tuple[torch.Tensor, ...],
|
||||
t: torch.Tensor,
|
||||
*,
|
||||
order: list[str],
|
||||
) -> list[Flux3SegmentState]:
|
||||
"""Noised ``video`` and action streams through their mode blocks."""
|
||||
return [
|
||||
self.transformer.encode_stream(
|
||||
name=name,
|
||||
x=x.to(torch.bfloat16),
|
||||
ids=None,
|
||||
timesteps=t,
|
||||
rope=rope,
|
||||
)
|
||||
for name, x, rope in zip(order, samples, ropes, strict=True)
|
||||
]
|
||||
|
||||
def _denoise_pass(
|
||||
self,
|
||||
context: Flux3SegmentState,
|
||||
targets: list[Flux3SegmentState],
|
||||
conditioning: list[Flux3SegmentState],
|
||||
*,
|
||||
order: list[str],
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
video, action = targets
|
||||
video_cond, action_cond = conditioning
|
||||
# Joint sequence order: video, video_cond, action, action_cond.
|
||||
out = self.transformer.denoise(
|
||||
context=context,
|
||||
streams=[video, video_cond, action, action_cond],
|
||||
targets=order,
|
||||
)
|
||||
return tuple(out[k] for k in order)
|
||||
|
||||
def _denoise_step(
|
||||
self,
|
||||
video: torch.Tensor,
|
||||
action: torch.Tensor,
|
||||
t: torch.Tensor,
|
||||
video_rope: torch.Tensor,
|
||||
action_rope: torch.Tensor,
|
||||
*states: torch.Tensor,
|
||||
order: list[str],
|
||||
state_names: list[str],
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Every CFG pass of one step, flattened; ``states`` = conditioning + contexts."""
|
||||
video_cond, action_cond, *contexts = _states_from_tensors(state_names, states)
|
||||
targets = self._target_streams(
|
||||
(video, action), (video_rope, action_rope), t, order=order
|
||||
)
|
||||
return tuple(
|
||||
p
|
||||
for ctx in contexts
|
||||
for p in self._denoise_pass(
|
||||
ctx, targets, [video_cond, action_cond], order=order
|
||||
)
|
||||
)
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
cfg = self.config
|
||||
device = get_local_torch_device()
|
||||
@@ -682,10 +830,14 @@ class Flux3ActionDenoisingStage(PipelineStage):
|
||||
start = time.perf_counter()
|
||||
samples, ids = self._noised_streams(seed)
|
||||
samples = {k: v.to(device) for k, v in samples.items()}
|
||||
ropes = {k: self.transformer.rope(v.to(device)) for k, v in ids.items()}
|
||||
video_cond, action_cond = conditioning
|
||||
order = list(samples) # joint sequence: video, video_cond, action, action_cond
|
||||
order = list(samples)
|
||||
ropes = tuple(self.transformer.rope(ids[k].to(device)) for k in order)
|
||||
cfg_policy = _cfg_parallel_policy(contexts, server_args)
|
||||
state_names = [s.name for s in (*conditioning, *contexts)]
|
||||
states = tuple(t for s in (*conditioning, *contexts) for t in _state_tensors(s))
|
||||
denoise_step = functools.partial(
|
||||
self._denoise_step, order=order, state_names=state_names
|
||||
)
|
||||
step = 0
|
||||
|
||||
def predict(
|
||||
@@ -693,34 +845,34 @@ class Flux3ActionDenoisingStage(PipelineStage):
|
||||
) -> dict[str, torch.Tensor]:
|
||||
nonlocal step
|
||||
timestep = torch.full((1,), t, device=device, dtype=torch.float32)
|
||||
current_streams = tuple(current[k] for k in order)
|
||||
with set_forward_context(
|
||||
current_timestep=step, attn_metadata=None, forward_batch=batch
|
||||
):
|
||||
targets = {
|
||||
name: self.transformer.encode_stream(
|
||||
name=name,
|
||||
x=current[name].to(torch.bfloat16),
|
||||
ids=None,
|
||||
timesteps=timestep,
|
||||
rope=ropes[name],
|
||||
)
|
||||
for name in order
|
||||
}
|
||||
streams = [targets["video"], video_cond, targets[order[1]], action_cond]
|
||||
|
||||
def denoise(ctx: Flux3SegmentState) -> tuple[torch.Tensor, ...]:
|
||||
out = self.transformer.denoise(
|
||||
context=ctx, streams=streams, targets=order
|
||||
)
|
||||
return tuple(out[k] for k in order)
|
||||
|
||||
if cfg_policy is not None:
|
||||
targets = self._target_streams(
|
||||
current_streams, ropes, timestep, order=order
|
||||
)
|
||||
# Each rank runs one branch; all ranks get both predictions.
|
||||
raw = run_cfg_parallel(
|
||||
cfg_policy, lambda branch: denoise(branch.kwargs["context"])
|
||||
cfg_policy,
|
||||
lambda branch: self._denoise_pass(
|
||||
branch.kwargs["context"],
|
||||
targets,
|
||||
conditioning,
|
||||
order=order,
|
||||
),
|
||||
)
|
||||
else:
|
||||
raw = [denoise(ctx) for ctx in contexts]
|
||||
flat = self._graph.run(
|
||||
denoise_step,
|
||||
(*current_streams, timestep, *ropes, *states),
|
||||
key=(tuple(order), tuple(state_names)),
|
||||
)
|
||||
raw = [
|
||||
flat[i : i + len(order)]
|
||||
for i in range(0, len(flat), len(order))
|
||||
]
|
||||
preds = [dict(zip(order, p)) for p in raw]
|
||||
step += 1
|
||||
if len(preds) == 1:
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
@@ -490,3 +491,99 @@ class VLADenoiseGraphRunner:
|
||||
exc_info=True,
|
||||
)
|
||||
return step_fn(prefix_context, x_t, timestep)
|
||||
|
||||
|
||||
class _CapturedTensorGraph(msgspec.Struct):
|
||||
graph: torch.cuda.CUDAGraph
|
||||
static_inputs: tuple[torch.Tensor, ...]
|
||||
static_outputs: tuple[torch.Tensor, ...]
|
||||
|
||||
|
||||
class VLATensorGraphRunner:
|
||||
"""CUDA graphs of a function of tensors, keyed by the input shapes and ``key``.
|
||||
|
||||
Every call copies ``inputs`` into the graph's static buffers, so ``fn`` must
|
||||
read its tensors only from its arguments (anything else is baked in by
|
||||
address). The returned tensors are overwritten by the next replay.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, *, enabled: bool, max_entries: int):
|
||||
self.name = name
|
||||
self.enabled = enabled and max_entries > 0
|
||||
self._cache = _BoundedCaptureCache(name, max_entries, evict_on_miss=True)
|
||||
self._disabled_signatures: set[Any] = set()
|
||||
self._capture_stream: torch.cuda.Stream | None = None
|
||||
self._graph_pool: Any = None
|
||||
|
||||
def cache_info(self) -> VLAGraphCacheInfo:
|
||||
return self._cache.info()
|
||||
|
||||
def clear(self) -> None:
|
||||
self._cache.clear()
|
||||
|
||||
def _capture(
|
||||
self,
|
||||
fn: Callable[..., tuple[torch.Tensor, ...]],
|
||||
inputs: tuple[torch.Tensor, ...],
|
||||
) -> _CapturedTensorGraph:
|
||||
device = inputs[0].device
|
||||
device_module = torch.get_device_module(device)
|
||||
if self._capture_stream is None:
|
||||
self._capture_stream = device_module.Stream(device=device)
|
||||
if self._graph_pool is None:
|
||||
self._graph_pool = get_or_create_global_graph_memory_pool(device_module)
|
||||
set_graph_pool_id(self._graph_pool)
|
||||
static_inputs = tuple(t.detach().clone() for t in inputs)
|
||||
# Warm up lazy kernels, workspaces and first-sight fusion checks.
|
||||
device_module.synchronize()
|
||||
with device_module.stream(self._capture_stream):
|
||||
fn(*static_inputs)
|
||||
self._capture_stream.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with device_module.graph(
|
||||
cuda_graph=graph, pool=self._graph_pool, stream=self._capture_stream
|
||||
):
|
||||
static_outputs = tuple(fn(*static_inputs))
|
||||
self._capture_stream.synchronize()
|
||||
logger.info(
|
||||
"Captured VLA %s CUDA graph for %s",
|
||||
self.name,
|
||||
[tuple(t.shape) for t in inputs],
|
||||
)
|
||||
return _CapturedTensorGraph(
|
||||
graph=graph, static_inputs=static_inputs, static_outputs=static_outputs
|
||||
)
|
||||
|
||||
def run(
|
||||
self,
|
||||
fn: Callable[..., tuple[torch.Tensor, ...]],
|
||||
inputs: tuple[torch.Tensor, ...],
|
||||
*,
|
||||
key: Any = None,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
if not self.enabled or not inputs or inputs[0].device.type != "cuda":
|
||||
return tuple(fn(*inputs))
|
||||
signature = (key, tuple((tuple(t.shape), t.dtype) for t in inputs))
|
||||
if signature in self._disabled_signatures:
|
||||
return tuple(fn(*inputs))
|
||||
try:
|
||||
captured = self._cache.get(signature)
|
||||
if captured is None:
|
||||
self._cache.prepare_admission(signature)
|
||||
captured = self._capture(fn, inputs)
|
||||
self._cache.put(signature, captured)
|
||||
for static, value in zip(captured.static_inputs, inputs, strict=True):
|
||||
static.copy_(value)
|
||||
captured.graph.replay()
|
||||
return captured.static_outputs
|
||||
except Exception:
|
||||
self._disabled_signatures.add(signature)
|
||||
self._cache.discard(signature)
|
||||
self._cache.mark_failure()
|
||||
logger.warning(
|
||||
"VLA %s CUDA graph disabled for signature %s",
|
||||
self.name,
|
||||
signature,
|
||||
exc_info=True,
|
||||
)
|
||||
return tuple(fn(*inputs))
|
||||
|
||||
@@ -647,6 +647,43 @@ def test_dit_cached_streams_match_full_forward():
|
||||
assert full["x_act_cond"].shape == (1, 1, 3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA graphs")
|
||||
def test_cuda_graph_replays_match_eager_denoise_steps():
|
||||
"""Replays with fresh inputs must match eager steps bit for bit."""
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
set_forward_context,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import Flux3Transformer
|
||||
from sglang.multimodal_gen.runtime.vla.cuda_graph import VLATensorGraphRunner
|
||||
|
||||
_init_single_process_parallel()
|
||||
torch.manual_seed(0)
|
||||
model = Flux3Transformer(Flux3DiTConfig(arch_config=_tiny_arch())).cuda().bfloat16()
|
||||
for p in model.parameters():
|
||||
torch.nn.init.normal_(p, std=0.05)
|
||||
_, ids = pack_video(torch.zeros(1, 96, 2, 2, 3), first_frame=1, fps=15.0)
|
||||
rope = model.rope(ids.cuda())
|
||||
|
||||
def step(x, t, rope):
|
||||
state = model.encode_stream(name="video", x=x, ids=None, timesteps=t, rope=rope)
|
||||
return (state.hidden,)
|
||||
|
||||
runner = VLATensorGraphRunner("test", enabled=True, max_entries=2)
|
||||
with torch.no_grad(), set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
for seed, t in ((1, 0.7), (2, 0.3)):
|
||||
torch.manual_seed(seed)
|
||||
inputs = (
|
||||
torch.randn(1, 12, 96, device="cuda", dtype=torch.bfloat16),
|
||||
torch.full((1,), t, device="cuda"),
|
||||
rope,
|
||||
)
|
||||
(eager,) = step(*inputs)
|
||||
(graph,) = runner.run(step, inputs)
|
||||
assert torch.equal(graph, eager)
|
||||
info = runner.cache_info()
|
||||
assert (info.captures, info.hits, info.failures) == (1, 1, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fnuz", [False, True])
|
||||
@pytest.mark.parametrize("tuple_output", [False, True])
|
||||
def test_fp8r_native_format_preserves_values(monkeypatch, fnuz, tuple_output):
|
||||
|
||||
Reference in New Issue
Block a user