From affa261e3d289fe4f907c9b2e8d773fef0d36dba Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:22:08 +0800 Subject: [PATCH] [diffusion] feat: enable cuda graphs for observation encoding and denoising steps for action models (#42171) --- .../configs/pipeline_configs/flux3_action.py | 12 + .../runtime/models/dits/flux3.py | 24 +- .../runtime/pipelines/flux3_action.py | 11 +- .../model_specific_stages/flux3_action.py | 252 ++++++++++++++---- .../multimodal_gen/runtime/vla/cuda_graph.py | 97 +++++++ .../test/unit/test_flux3_action.py | 37 +++ 6 files changed, 371 insertions(+), 62 deletions(-) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/flux3_action.py b/python/sglang/multimodal_gen/configs/pipeline_configs/flux3_action.py index 03f0227750..46d2aacbb2 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/flux3_action.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/flux3_action.py @@ -267,6 +267,18 @@ class Flux3ActionPipelineConfig(PipelineConfig): text_output_layers: tuple[int, ...] = (4, 8, 12, 16, 20, 24, 28, 32) # Text contexts cached per caption, bounded by their total token count. caption_cache_max_tokens: int = 1 << 16 + # CUDA graphs of the observation encoding and of each denoising step. + enable_cuda_graph: bool = True + # Per stage; one entry per (prompt length bucket, CFG) combination. + cuda_graph_max_entries: int = 4 + + def cuda_graph_enabled(self, server_args: Any, component: str) -> bool: + """Graphs need resident weights and no cross-rank collectives.""" + return ( + self.enable_cuda_graph + and server_args.num_gpus == 1 + and not server_args.should_start_component_on_cpu(component) + ) def validate_server_args(self, server_args: Any) -> None: super().validate_server_args(server_args) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py index ce8e4a08d0..c6222275aa 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py @@ -135,13 +135,13 @@ _LN_MODULATE = BitExactFusionGate("FLUX 3 fused LN+modulate", per_signature=True _SWIGLU = BitExactFusionGate("FLUX 3 fused SwiGLU", per_signature=True) -def _eager_fast_path_allowed(x: torch.Tensor) -> bool: - return ( - x.is_cuda - and x.dtype is torch.bfloat16 - and not torch.compiler.is_compiling() - and not torch.cuda.is_current_stream_capturing() - ) +def _fast_path_allowed(x: torch.Tensor) -> bool: + return x.is_cuda and x.dtype is torch.bfloat16 and not torch.compiler.is_compiling() + + +def _can_verify() -> bool: + # First-sight verification runs the eager chain and a host sync. + return not torch.cuda.is_current_stream_capturing() def _norm_modulate( @@ -151,12 +151,14 @@ def _norm_modulate( scale_row, shift_row = scale[:, 0], shift[:, 0] if ( _LN_MODULATE.disabled - or not _eager_fast_path_allowed(x) + or not _fast_path_allowed(x) or not is_plain_layer_norm(norm, x.shape[-1]) or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) ): return (1 + scale) * norm(x) + shift sig = (x.device, x.shape[0], x.shape[-1], norm.eps) + if not _LN_MODULATE.is_verified(sig) and not _can_verify(): + return (1 + scale) * norm(x) + shift try: out = fused_layernorm_modulate_raw(x, scale_row, shift_row, norm.eps) except Exception as exc: @@ -176,9 +178,11 @@ def _norm_modulate( def _swiglu(packed: torch.Tensor) -> torch.Tensor: """``silu(gate) * value`` of a packed ``[gate | value]`` projection.""" gate, value = packed.chunk(2, dim=-1) - if _SWIGLU.disabled or not _eager_fast_path_allowed(packed): + if _SWIGLU.disabled or not _fast_path_allowed(packed): return F.silu(gate) * value sig = (packed.device, packed.shape[-1], packed.stride(-2)) + if not _SWIGLU.is_verified(sig) and not _can_verify(): + return F.silu(gate) * value try: out = fused_packed_silu_mul_bitexact(packed) except Exception as exc: @@ -197,7 +201,7 @@ def _swiglu(packed: torch.Tensor) -> torch.Tensor: def _fused_qknorm_rope_enabled(q: torch.Tensor, head_dim: int) -> bool: return ( - _eager_fast_path_allowed(q) + _fast_path_allowed(q) and os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() not in ("0", "false", "off", "no") and can_use_fused_inplace_qknorm_rope( diff --git a/python/sglang/multimodal_gen/runtime/pipelines/flux3_action.py b/python/sglang/multimodal_gen/runtime/pipelines/flux3_action.py index c7f374ccea..d1828f2d40 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/flux3_action.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/flux3_action.py @@ -186,13 +186,20 @@ class Flux3ActionPipeline(ComposedPipelineBase): ) self.add_stage( Flux3ActionObservationEncodingStage( - config, transformer=transformer, vae=self.get_module("vae") + config, + transformer=transformer, + vae=self.get_module("vae"), + use_cuda_graph=config.cuda_graph_enabled(server_args, "transformer") + and config.cuda_graph_enabled(server_args, "vae"), ), "flux3_action_observation_encoding", ) self.add_stage( Flux3ActionDenoisingStage( - config, transformer, scheduler=self.get_module("scheduler") + config, + transformer, + scheduler=self.get_module("scheduler"), + use_cuda_graph=config.cuda_graph_enabled(server_args, "transformer"), ), "flux3_action_denoise", ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/flux3_action.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/flux3_action.py index 2d6a7ee370..dd3f6775b0 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/flux3_action.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/flux3_action.py @@ -16,6 +16,7 @@ conditional and unconditional CFG passes, since mode blocks never see text). from __future__ import annotations import copy +import functools import time from collections import OrderedDict from collections.abc import Callable @@ -60,6 +61,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import ( ) from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.vla.cuda_graph import VLATensorGraphRunner logger = init_logger(__name__) @@ -88,14 +90,22 @@ def _cartesian_ids( return torch.cartesian_prod(t, torch.arange(h), torch.arange(w), l_coord) -def pack_video(latent: torch.Tensor, first_frame: int, fps: float): - """Latent ``(1, C, T, h, w)`` -> tokens ``(1, T*h*w, C)`` and ids; frame ``i`` at ``i * 4 / fps`` s.""" - _, _, t, h, w = latent.shape +def video_ids(*, frames: int, h: int, w: int, first_frame: int, fps: float): + """Ids ``(1, frames*h*w, 4)`` of latent frames; frame ``i`` at ``i * 4 / fps`` s.""" seconds = ( - torch.arange(first_frame, first_frame + t).float() * TEMPORAL_DOWNSAMPLE / fps + torch.arange(first_frame, first_frame + frames).float() + * TEMPORAL_DOWNSAMPLE + / fps ) ids = _cartesian_ids(t=_times_to_ids(seconds), h=h, w=w, l_coord=torch.arange(1)) - return rearrange(latent, "b c t h w -> b (t h w) c"), ids[None] + return ids[None] + + +def pack_video(latent: torch.Tensor, first_frame: int, fps: float): + """Latent ``(1, C, T, h, w)`` -> tokens ``(1, T*h*w, C)`` and ids.""" + _, _, t, h, w = latent.shape + ids = video_ids(frames=t, h=h, w=w, first_frame=first_frame, fps=fps) + return rearrange(latent, "b c t h w -> b (t h w) c"), ids def pack_action(values: torch.Tensor, seconds: torch.Tensor): @@ -403,6 +413,34 @@ def cosmos_unipc( return samples +# Flux3SegmentState as graph tensors: hidden, rope, vec and the 3 joint modulations. +_STATE_TENSORS = 6 + + +def _state_tensors(state: Flux3SegmentState) -> tuple[torch.Tensor, ...]: + return (state.hidden, state.rope, state.vec, *state.joint_mod) + + +def _state_from_tensors( + name: str, tensors: tuple[torch.Tensor, ...] +) -> Flux3SegmentState: + hidden, rope, vec, *joint_mod = tensors + return Flux3SegmentState( + name=name, hidden=hidden, rope=rope, vec=vec, joint_mod=tuple(joint_mod) + ) + + +def _states_from_tensors( + names: list[str], tensors: tuple[torch.Tensor, ...] +) -> list[Flux3SegmentState]: + return [ + _state_from_tensors( + name, tensors[i * _STATE_TENSORS : (i + 1) * _STATE_TENSORS] + ) + for i, name in enumerate(names) + ] + + def _cfg_parallel_policy( contexts: list[Flux3SegmentState], server_args: ServerArgs ) -> CFGPolicy | None: @@ -557,11 +595,17 @@ class Flux3ActionObservationEncodingStage(PipelineStage): config: Flux3ActionPipelineConfig, transformer: Flux3Transformer, vae: Flux3VideoVAE, + use_cuda_graph: bool = False, ): super().__init__() self.config = config self.transformer = transformer self.vae = vae + self._graph = VLATensorGraphRunner( + "FLUX 3 observation", + enabled=use_cuda_graph, + max_entries=config.cuda_graph_max_entries, + ) def component_uses( self, server_args: ServerArgs, stage_name: str | None = None @@ -585,6 +629,41 @@ class Flux3ActionObservationEncodingStage(PipelineStage): values = (token[None, :, None] * cfg.action_scale).to(device) return pack_action(values, seconds=torch.zeros(1)) + def _encode_frame(self, frame: torch.Tensor) -> torch.Tensor: + h, w = self.config.latent_hw + latent = self.vae.encode_frame(frame)[..., :h, :w] + return rearrange(latent, "b c t h w -> b (t h w) c") + + def _encode_streams( + self, + video: torch.Tensor, + frame_ids: torch.Tensor, + action: torch.Tensor, + action_ids: torch.Tensor, + zero: torch.Tensor, + ) -> tuple[torch.Tensor, ...]: + video_cond = self.transformer.encode_stream( + name="video_cond", x=video, ids=frame_ids, timesteps=zero + ) + action_cond = self.transformer.encode_stream( + name=f"{self.config.action_modality}_cond", + x=action, + ids=action_ids, + timesteps=zero, + ) + return (*_state_tensors(video_cond), *_state_tensors(action_cond)) + + def _encode(self, frame: torch.Tensor, *stream_inputs: torch.Tensor): + return self._encode_streams(self._encode_frame(frame), *stream_inputs) + + def _vae_use(self): + return self.use_declared_component(component_name="vae", module=self.vae) + + def _transformer_use(self): + return self.use_declared_component( + component_name="transformer", module=self.transformer + ) + def forward(self, batch: Req, server_args: ServerArgs) -> Req: start = time.perf_counter() cfg = self.config @@ -592,29 +671,30 @@ class Flux3ActionObservationEncodingStage(PipelineStage): state = vla_state(batch) observation: Flux3ActionObservation = state["flux3_observation"] h, w = cfg.latent_hw - with self.use_declared_component(component_name="vae", module=self.vae): - frame = observation.canvas.to(device, torch.bfloat16)[None] - latent = self.vae.encode_frame(frame)[..., :h, :w] - video, video_ids = pack_video(latent, first_frame=0, fps=cfg.fps) + frame_ids = video_ids(frames=1, h=h, w=w, first_frame=0, fps=cfg.fps) action, action_ids = self._state_tokens(observation.state, device) - zero = torch.zeros(1, device=device) - with ( - self.use_declared_component( - component_name="transformer", module=self.transformer - ), - set_forward_context(current_timestep=0, attn_metadata=None), - ): - state["flux3_conditioning"] = [ - self.transformer.encode_stream( - name="video_cond", x=video, ids=video_ids.to(device), timesteps=zero - ), - self.transformer.encode_stream( - name=f"{cfg.action_modality}_cond", - x=action, - ids=action_ids.to(device), - timesteps=zero, - ), - ] + frame = observation.canvas.to(device, torch.bfloat16)[None] + stream_inputs = ( + frame_ids.to(device), + action, + action_ids.to(device), + torch.zeros(1, device=device), + ) + forward_context = set_forward_context(current_timestep=0, attn_metadata=None) + if self._graph.enabled: + # Both components are resident whenever graphs are enabled. + with self._vae_use(), self._transformer_use(), forward_context: + captured = self._graph.run(self._encode, (frame, *stream_inputs)) + # Replays overwrite the outputs; the states outlive this request. + tensors = tuple(t.clone() for t in captured) + else: + with self._vae_use(): + video = self._encode_frame(frame) + with self._transformer_use(), forward_context: + tensors = self._encode_streams(video, *stream_inputs) + state["flux3_conditioning"] = _states_from_tensors( + ["video_cond", f"{cfg.action_modality}_cond"], tensors + ) vla_timings(batch)["observation_ms"] = (time.perf_counter() - start) * 1000 return batch @@ -627,11 +707,17 @@ class Flux3ActionDenoisingStage(PipelineStage): config: Flux3ActionPipelineConfig, transformer: Flux3Transformer, scheduler: FlowUniPCMultistepScheduler, + use_cuda_graph: bool = False, ): super().__init__() self.config = config self.transformer = transformer self.scheduler = scheduler + self._graph = VLATensorGraphRunner( + "FLUX 3 denoise", + enabled=use_cuda_graph, + max_entries=config.cuda_graph_max_entries, + ) def component_uses( self, server_args: ServerArgs, stage_name: str | None = None @@ -663,6 +749,68 @@ class Flux3ActionDenoisingStage(PipelineStage): cfg.action_modality: action_ids, } + def _target_streams( + self, + samples: tuple[torch.Tensor, ...], + ropes: tuple[torch.Tensor, ...], + t: torch.Tensor, + *, + order: list[str], + ) -> list[Flux3SegmentState]: + """Noised ``video`` and action streams through their mode blocks.""" + return [ + self.transformer.encode_stream( + name=name, + x=x.to(torch.bfloat16), + ids=None, + timesteps=t, + rope=rope, + ) + for name, x, rope in zip(order, samples, ropes, strict=True) + ] + + def _denoise_pass( + self, + context: Flux3SegmentState, + targets: list[Flux3SegmentState], + conditioning: list[Flux3SegmentState], + *, + order: list[str], + ) -> tuple[torch.Tensor, ...]: + video, action = targets + video_cond, action_cond = conditioning + # Joint sequence order: video, video_cond, action, action_cond. + out = self.transformer.denoise( + context=context, + streams=[video, video_cond, action, action_cond], + targets=order, + ) + return tuple(out[k] for k in order) + + def _denoise_step( + self, + video: torch.Tensor, + action: torch.Tensor, + t: torch.Tensor, + video_rope: torch.Tensor, + action_rope: torch.Tensor, + *states: torch.Tensor, + order: list[str], + state_names: list[str], + ) -> tuple[torch.Tensor, ...]: + """Every CFG pass of one step, flattened; ``states`` = conditioning + contexts.""" + video_cond, action_cond, *contexts = _states_from_tensors(state_names, states) + targets = self._target_streams( + (video, action), (video_rope, action_rope), t, order=order + ) + return tuple( + p + for ctx in contexts + for p in self._denoise_pass( + ctx, targets, [video_cond, action_cond], order=order + ) + ) + def forward(self, batch: Req, server_args: ServerArgs) -> Req: cfg = self.config device = get_local_torch_device() @@ -682,10 +830,14 @@ class Flux3ActionDenoisingStage(PipelineStage): start = time.perf_counter() samples, ids = self._noised_streams(seed) samples = {k: v.to(device) for k, v in samples.items()} - ropes = {k: self.transformer.rope(v.to(device)) for k, v in ids.items()} - video_cond, action_cond = conditioning - order = list(samples) # joint sequence: video, video_cond, action, action_cond + order = list(samples) + ropes = tuple(self.transformer.rope(ids[k].to(device)) for k in order) cfg_policy = _cfg_parallel_policy(contexts, server_args) + state_names = [s.name for s in (*conditioning, *contexts)] + states = tuple(t for s in (*conditioning, *contexts) for t in _state_tensors(s)) + denoise_step = functools.partial( + self._denoise_step, order=order, state_names=state_names + ) step = 0 def predict( @@ -693,34 +845,34 @@ class Flux3ActionDenoisingStage(PipelineStage): ) -> dict[str, torch.Tensor]: nonlocal step timestep = torch.full((1,), t, device=device, dtype=torch.float32) + current_streams = tuple(current[k] for k in order) with set_forward_context( current_timestep=step, attn_metadata=None, forward_batch=batch ): - targets = { - name: self.transformer.encode_stream( - name=name, - x=current[name].to(torch.bfloat16), - ids=None, - timesteps=timestep, - rope=ropes[name], - ) - for name in order - } - streams = [targets["video"], video_cond, targets[order[1]], action_cond] - - def denoise(ctx: Flux3SegmentState) -> tuple[torch.Tensor, ...]: - out = self.transformer.denoise( - context=ctx, streams=streams, targets=order - ) - return tuple(out[k] for k in order) - if cfg_policy is not None: + targets = self._target_streams( + current_streams, ropes, timestep, order=order + ) # Each rank runs one branch; all ranks get both predictions. raw = run_cfg_parallel( - cfg_policy, lambda branch: denoise(branch.kwargs["context"]) + cfg_policy, + lambda branch: self._denoise_pass( + branch.kwargs["context"], + targets, + conditioning, + order=order, + ), ) else: - raw = [denoise(ctx) for ctx in contexts] + flat = self._graph.run( + denoise_step, + (*current_streams, timestep, *ropes, *states), + key=(tuple(order), tuple(state_names)), + ) + raw = [ + flat[i : i + len(order)] + for i in range(0, len(flat), len(order)) + ] preds = [dict(zip(order, p)) for p in raw] step += 1 if len(preds) == 1: diff --git a/python/sglang/multimodal_gen/runtime/vla/cuda_graph.py b/python/sglang/multimodal_gen/runtime/vla/cuda_graph.py index 301473af57..7233598ca5 100644 --- a/python/sglang/multimodal_gen/runtime/vla/cuda_graph.py +++ b/python/sglang/multimodal_gen/runtime/vla/cuda_graph.py @@ -6,6 +6,7 @@ from collections import OrderedDict from dataclasses import dataclass from typing import Any, Callable +import msgspec import torch from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -490,3 +491,99 @@ class VLADenoiseGraphRunner: exc_info=True, ) return step_fn(prefix_context, x_t, timestep) + + +class _CapturedTensorGraph(msgspec.Struct): + graph: torch.cuda.CUDAGraph + static_inputs: tuple[torch.Tensor, ...] + static_outputs: tuple[torch.Tensor, ...] + + +class VLATensorGraphRunner: + """CUDA graphs of a function of tensors, keyed by the input shapes and ``key``. + + Every call copies ``inputs`` into the graph's static buffers, so ``fn`` must + read its tensors only from its arguments (anything else is baked in by + address). The returned tensors are overwritten by the next replay. + """ + + def __init__(self, name: str, *, enabled: bool, max_entries: int): + self.name = name + self.enabled = enabled and max_entries > 0 + self._cache = _BoundedCaptureCache(name, max_entries, evict_on_miss=True) + self._disabled_signatures: set[Any] = set() + self._capture_stream: torch.cuda.Stream | None = None + self._graph_pool: Any = None + + def cache_info(self) -> VLAGraphCacheInfo: + return self._cache.info() + + def clear(self) -> None: + self._cache.clear() + + def _capture( + self, + fn: Callable[..., tuple[torch.Tensor, ...]], + inputs: tuple[torch.Tensor, ...], + ) -> _CapturedTensorGraph: + device = inputs[0].device + device_module = torch.get_device_module(device) + if self._capture_stream is None: + self._capture_stream = device_module.Stream(device=device) + if self._graph_pool is None: + self._graph_pool = get_or_create_global_graph_memory_pool(device_module) + set_graph_pool_id(self._graph_pool) + static_inputs = tuple(t.detach().clone() for t in inputs) + # Warm up lazy kernels, workspaces and first-sight fusion checks. + device_module.synchronize() + with device_module.stream(self._capture_stream): + fn(*static_inputs) + self._capture_stream.synchronize() + graph = torch.cuda.CUDAGraph() + with device_module.graph( + cuda_graph=graph, pool=self._graph_pool, stream=self._capture_stream + ): + static_outputs = tuple(fn(*static_inputs)) + self._capture_stream.synchronize() + logger.info( + "Captured VLA %s CUDA graph for %s", + self.name, + [tuple(t.shape) for t in inputs], + ) + return _CapturedTensorGraph( + graph=graph, static_inputs=static_inputs, static_outputs=static_outputs + ) + + def run( + self, + fn: Callable[..., tuple[torch.Tensor, ...]], + inputs: tuple[torch.Tensor, ...], + *, + key: Any = None, + ) -> tuple[torch.Tensor, ...]: + if not self.enabled or not inputs or inputs[0].device.type != "cuda": + return tuple(fn(*inputs)) + signature = (key, tuple((tuple(t.shape), t.dtype) for t in inputs)) + if signature in self._disabled_signatures: + return tuple(fn(*inputs)) + try: + captured = self._cache.get(signature) + if captured is None: + self._cache.prepare_admission(signature) + captured = self._capture(fn, inputs) + self._cache.put(signature, captured) + for static, value in zip(captured.static_inputs, inputs, strict=True): + static.copy_(value) + captured.graph.replay() + return captured.static_outputs + except Exception: + self._disabled_signatures.add(signature) + self._cache.discard(signature) + self._cache.mark_failure() + logger.warning( + "VLA %s CUDA graph disabled for signature %s", + self.name, + signature, + exc_info=True, + ) + return tuple(fn(*inputs)) diff --git a/python/sglang/multimodal_gen/test/unit/test_flux3_action.py b/python/sglang/multimodal_gen/test/unit/test_flux3_action.py index 909c02546d..acb006114d 100644 --- a/python/sglang/multimodal_gen/test/unit/test_flux3_action.py +++ b/python/sglang/multimodal_gen/test/unit/test_flux3_action.py @@ -647,6 +647,43 @@ def test_dit_cached_streams_match_full_forward(): assert full["x_act_cond"].shape == (1, 1, 3) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA graphs") +def test_cuda_graph_replays_match_eager_denoise_steps(): + """Replays with fresh inputs must match eager steps bit for bit.""" + from sglang.multimodal_gen.runtime.managers.forward_context import ( + set_forward_context, + ) + from sglang.multimodal_gen.runtime.models.dits.flux3 import Flux3Transformer + from sglang.multimodal_gen.runtime.vla.cuda_graph import VLATensorGraphRunner + + _init_single_process_parallel() + torch.manual_seed(0) + model = Flux3Transformer(Flux3DiTConfig(arch_config=_tiny_arch())).cuda().bfloat16() + for p in model.parameters(): + torch.nn.init.normal_(p, std=0.05) + _, ids = pack_video(torch.zeros(1, 96, 2, 2, 3), first_frame=1, fps=15.0) + rope = model.rope(ids.cuda()) + + def step(x, t, rope): + state = model.encode_stream(name="video", x=x, ids=None, timesteps=t, rope=rope) + return (state.hidden,) + + runner = VLATensorGraphRunner("test", enabled=True, max_entries=2) + with torch.no_grad(), set_forward_context(current_timestep=0, attn_metadata=None): + for seed, t in ((1, 0.7), (2, 0.3)): + torch.manual_seed(seed) + inputs = ( + torch.randn(1, 12, 96, device="cuda", dtype=torch.bfloat16), + torch.full((1,), t, device="cuda"), + rope, + ) + (eager,) = step(*inputs) + (graph,) = runner.run(step, inputs) + assert torch.equal(graph, eager) + info = runner.cache_info() + assert (info.captures, info.hits, info.failures) == (1, 1, 0) + + @pytest.mark.parametrize("fnuz", [False, True]) @pytest.mark.parametrize("tuple_output", [False, True]) def test_fp8r_native_format_preserves_values(monkeypatch, fnuz, tuple_output):