[Perf] deepseek-v4.1: Run decoder replay tail layers from CUDA graphs on eager prefill steps (#45059)

This commit is contained in:
Liangsheng Yin
2026-10-18 23:07:50 -07:00
committed by GitHub
parent 04c8c9703f
commit a3330237db
6 changed files with 525 additions and 2 deletions
+3
View File
@@ -1630,6 +1630,9 @@ class Envs:
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE = EnvBool(False)
# Decoder SWA bounded replay: eager prefill steps with up to this many tail rows
# run the tail layers from CUDA graphs (buckets of 128 rows); 0 keeps them eager.
SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS = EnvInt(2048)
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
@@ -3314,8 +3314,14 @@ class DeepseekV4AttnBackend(
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
is_in_breakable_cuda_graph,
)
from sglang.srt.models.deepseek_v4_replay_tail import in_replay_tail_graph
return self.low_ratio_prefill_graph and is_in_breakable_cuda_graph()
# Decoder tail graphs run the low-ratio sources as an eager break.
return (
self.low_ratio_prefill_graph
and is_in_breakable_cuda_graph()
and not in_replay_tail_graph()
)
def _low_ratio_compress_decode(self, layer, x, req, pos) -> None:
# Projection layout and fused-write support are fixed together at load time.
@@ -0,0 +1,220 @@
"""Breakable CUDA graphs of a layer range that runs over a varying number of rows,
one graph per row bucket (a multiple of ``row_step``).
Ops of the body that read per-step state (KV slots, attention metadata) must be
eager breaks; they read the step's real row count from
``forward_batch.global_num_token_non_padded_cpu``.
"""
from __future__ import annotations
import copy
import logging
import time
from typing import TYPE_CHECKING, Callable, ContextManager, List, Optional
import torch
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
set_graph_pool_id,
)
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
BreakableCUDAGraph,
BreakableCUDAGraphCapture,
)
from sglang.srt.model_executor.runner_utils.pool import (
get_or_create_global_graph_capture_stream,
get_or_create_global_graph_memory_pool,
graph_pool_capture_scope,
graph_pool_replay_scope,
)
from sglang.srt.runtime_context import get_parallel, get_schedule
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
logger = logging.getLogger(__name__)
Body = Callable[[List[torch.Tensor], "ForwardBatch"], List[Optional[torch.Tensor]]]
class _Graph:
__slots__ = ("graph", "num_token_non_padded", "outputs", "batch")
def __init__(self, graph, num_token_non_padded, outputs, batch):
self.graph = graph
# MoE top-k masks rows at or past this count; captured by address.
self.num_token_non_padded = num_token_non_padded
self.outputs = outputs
# Kernels write buffers cached on the capture batch and attention metadata.
self.batch = batch
class RowBucketGraphRunner:
def __init__(self, *, row_step: int, max_rows: int, label: str) -> None:
self.row_step = row_step
self.max_rows = max_rows - max_rows % row_step
self.label = label
self._graphs: dict[tuple, _Graph] = {}
self._static: dict[tuple, List[torch.Tensor]] = {}
self._warm = False
self.capture_seconds = 0.0
self.capture_bytes = 0
def ready(self, num_tokens: int) -> bool:
"""Whether graphs may run from this step on; every extend forward reports
its token count here. Kernels that size cached workspaces lazily by row
count reallocate them on a bigger step, which would leave earlier captures
reading freed memory; a full prefill chunk has grown them all."""
if not self._warm:
chunk = get_schedule().chunked_prefill_size
self._warm = num_tokens >= (chunk if chunk and chunk > 0 else 8192)
return self._warm
@property
def warm(self) -> bool:
return self._warm
def bucket_rows(self, num_rows: int) -> Optional[int]:
if num_rows == 0 or num_rows > self.max_rows:
return None
return -(-num_rows // self.row_step) * self.row_step
def run(
self,
*,
key: tuple,
live: List[torch.Tensor],
forward_batch: ForwardBatch,
body: Body,
context: Callable[[ForwardBatch], ContextManager],
) -> List[Optional[torch.Tensor]]:
"""Run ``body`` on the ``live`` rows from their bucket's graph, capturing
it on first use. ``key`` names whatever besides row count and input shapes
changes what ``body`` records; ``context`` wraps capture and replay with
the batch the eager breaks read. Returns the outputs cut to the live
rows, as views of graph buffers valid until the next replay."""
num_rows = live[0].shape[0]
rows = self.bucket_rows(num_rows)
assert rows is not None, num_rows
# Inputs of one shape share their static buffers across keys.
input_key = tuple((t.dtype, t.shape[1:]) for t in live)
static = self._static_inputs(input_key, live)
for buf, t in zip(static, live):
buf[:num_rows].copy_(t)
graph_key = (key, input_key)
# The eager breaks read the step's real row count off this view.
step_batch = copy.copy(forward_batch)
step_batch.global_num_token_non_padded_cpu = num_rows
with context(step_batch):
if (rows, graph_key) not in self._graphs:
# Largest first, so smaller buckets reuse the pool memory the
# larger ones took; a bucket below this step's rows cannot be
# captured here (the breaks write every live row).
for r in range(self.max_rows, rows - 1, -self.row_step):
if (r, graph_key) not in self._graphs:
self._graphs[(r, graph_key)] = self._capture(
r, num_rows, static, forward_batch, body
)
graph = self._graphs[(rows, graph_key)]
if graph.num_token_non_padded is not None:
graph.num_token_non_padded.fill_(num_rows)
with graph_pool_replay_scope():
graph.graph.replay()
return [None if t is None else t[:num_rows] for t in graph.outputs]
def _static_inputs(self, input_key, live) -> List[torch.Tensor]:
static = self._static.get(input_key)
if static is None:
static = self._static[input_key] = [
t.new_zeros((self.max_rows, *t.shape[1:])) for t in live
]
return static
def _capture(
self,
rows: int,
num_rows: int,
static: List[torch.Tensor],
forward_batch: ForwardBatch,
body: Body,
) -> _Graph:
start = time.perf_counter()
free_before = torch.cuda.mem_get_info()[0]
inputs = [buf[:rows] for buf in static]
# The layers read the step's batch; the copy's count tensor is the graph's.
# Host-side row counts read during capture cover the whole bucket.
capture_batch = copy.copy(forward_batch)
capture_batch.global_num_token_non_padded_cpu = rows
num_token_non_padded = None
if forward_batch.num_token_non_padded is not None:
num_token_non_padded = torch.full_like(
forward_batch.num_token_non_padded, num_rows
)
capture_batch.num_token_non_padded = num_token_non_padded
return self._record(
lambda: body(inputs, capture_batch),
rows,
num_token_non_padded,
capture_batch,
start,
free_before,
)
def _record(
self,
body,
rows,
num_token_non_padded,
capture_batch,
start,
free_before,
) -> _Graph:
pool = get_or_create_global_graph_memory_pool(torch.cuda)
set_graph_pool_id(pool)
tp_group = get_parallel().tp_group
graph = BreakableCUDAGraph()
with graph_capture(stream=get_or_create_global_graph_capture_stream()) as ctx:
# The warmups run eagerly; their KV writes repeat the replay's.
for _ in range(2):
torch.cuda.synchronize()
tp_group.barrier()
body()
with (
graph_pool_capture_scope(),
BreakableCUDAGraphCapture(
cuda_graph=graph,
pool=pool,
stream=ctx.stream,
barrier_fn=tp_group.barrier,
),
):
outputs = body()
torch.cuda.current_stream().wait_stream(ctx.stream)
torch.cuda.synchronize()
recorded = _Graph(
graph,
num_token_non_padded,
outputs,
(capture_batch, get_attn_backend().forward_metadata),
)
seconds = time.perf_counter() - start
used = free_before - torch.cuda.mem_get_info()[0]
self.capture_seconds += seconds
self.capture_bytes += used
if get_parallel().tp_rank == 0:
logger.info(
"%s: captured %d rows (%d segments) in %.2f s, %.3f GiB; "
"%d graphs, %.2f s, %.3f GiB",
self.label,
rows,
len(graph._segments),
seconds,
used / 2**30,
len(self._graphs) + 1,
self.capture_seconds,
self.capture_bytes / 2**30,
)
return recorded
+36 -1
View File
@@ -172,6 +172,11 @@ from sglang.srt.models.deepseek_v4_platform_stage import (
NonCudaDecoderMixin,
forward_layer_fused_boundary,
)
from sglang.srt.models.deepseek_v4_replay_tail import (
DecoderReplayTail,
bcg_tail_kv_store,
in_replay_tail_graph,
)
from sglang.srt.models.deepseek_v41_vit import Aligner, ViT
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.multimodal.deepseek_v41_image_processing import (
@@ -2305,6 +2310,10 @@ class MQALayer(MqaAttentionBase):
positions=global_positions,
)
kv = None
elif in_replay_tail_graph():
assert not fuse_q_rope
bcg_tail_kv_store(self, x_linear, positions, qkv_a)
kv = None
else:
self._compute_kv_to_cache(
x_linear,
@@ -2324,7 +2333,10 @@ class MQALayer(MqaAttentionBase):
if (
forward_batch.forward_mode.is_extend()
and is_in_breakable_cuda_graph()
and not getattr(attn_backend, "low_ratio_prefill_graph", False)
and (
not getattr(attn_backend, "low_ratio_prefill_graph", False)
or in_replay_tail_graph()
)
):
bcg_deepseek_v4_low_ratio_sources(self, x, q_lora, positions)
else:
@@ -3532,6 +3544,8 @@ DeepseekV4DecoderLayer = (
class DeepseekV4Model(nn.Module):
fall_back_to_pt_during_load = False
# Class-level default for instances built without running __init__.
decoder_replay_tail: Optional[DecoderReplayTail] = None
def __init__(
self,
@@ -3698,6 +3712,7 @@ class DeepseekV4Model(nn.Module):
and config.vision_n_layers > 0
and not getattr(config, "language_model_only", False)
)
self.decoder_replay_tail = DecoderReplayTail.maybe_create(self)
def get_input_embeddings(self) -> nn.Module:
return self.embed_tokens
@@ -3831,6 +3846,13 @@ class DeepseekV4Model(nn.Module):
),
in_graph=tail_in_graph,
)
use_tail_graph = (
self.decoder_replay_tail is not None
and tail is not None
and self.decoder_replay_tail.plan(
tail, num_tokens=input_ids.shape[0], tail_in_graph=tail_in_graph
)
)
tail_sentinel = positions.new_zeros((1,)) if tail_in_graph else None
saved_full = None
# The HIP defer_post gate excludes residual readers. CUDA materializes
@@ -3890,6 +3912,19 @@ class DeepseekV4Model(nn.Module):
positions = tail.positions
if hash_ids is not None:
hash_ids = tail.rows(hash_ids)
if use_tail_graph:
residual, pre, aux = self.decoder_replay_tail.run(
state=state,
positions=positions,
input_ids=input_ids,
input_ids_global=input_ids_global,
forward_batch=forward_batch,
capture_dspark=capture_dspark,
)
dspark_aux_hidden_states.extend(aux)
residual_batch.clear(forward_batch)
attn_backend.exit_late_layer_tail(saved_full, forward_batch)
return residual, pre, tail
if late_dp_counts is not None and i == self.late_layer_start:
saved_dp = enter_late_layer_dp(forward_batch, late_dp_counts)
if get_moe_a2a_backend().is_none():
@@ -0,0 +1,258 @@
"""DeepSeek-V4.1's decoder-replay tail layers (``late_layer_start`` on), replayed
from `RowBucketGraphRunner` graphs on eager prefill steps. The SWA KV store, the
attention and the low-ratio sources stay eager breaks of those graphs.
"""
from __future__ import annotations
from contextlib import contextmanager, nullcontext
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
check_cuda_graph_backend,
)
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.model_runner_components.layer_setup import (
compute_attention_and_moe_layers,
)
from sglang.srt.model_executor.runner.row_bucket_graph_runner import (
RowBucketGraphRunner,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
eager_on_graph,
enable_breakable_cuda_graph,
)
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.context_manager import (
get_tc_piecewise_forward_context,
set_tc_piecewise_forward_context,
)
from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
from sglang.srt.models.deepseek_v4_mhc import HcState
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_cuda
if TYPE_CHECKING:
from sglang.srt.layers.attention.deepseek_v4_backend import LateLayerTail
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
_in_replay_tail = False
def in_replay_tail_graph() -> bool:
"""True while the tail layers capture or replay a decoder tail graph."""
return _in_replay_tail
@contextmanager
def _replay_tail_scope():
global _in_replay_tail
_in_replay_tail = True
try:
yield
finally:
_in_replay_tail = False
def _tail_kv_store(attention, x, positions, qkv_a) -> None:
# The SWA store writes the step's live tail slots, so a tail graph breaks here.
forward_batch = get_tc_piecewise_forward_context().forward_batch
num_rows = forward_batch.global_num_token_non_padded_cpu
if num_rows == 0:
return
attention._compute_kv_to_cache(
x[:num_rows],
positions[:num_rows],
forward_batch,
get_attn_backend(),
qkv_a=None if qkv_a is None else qkv_a[:num_rows],
)
bcg_tail_kv_store = eager_on_graph(True)(_tail_kv_store)
def _flatten_state(state: HcState):
"""The state's row tensors, whether ``pre`` is among them, and the inverse.
The normed input and mix stats are dropped (the first tail layer rebuilds
them), so every step keys the same graphs."""
assert isinstance(state.streams, torch.Tensor), "tail graphs are CUDA only"
has_pre = state.pre is not None
leaves = [state.streams, state.pre] if has_pre else [state.streams]
def rebuild(tensors: List[torch.Tensor]) -> HcState:
return HcState(tensors[0], tensors[1] if has_pre else None)
return leaves, has_pre, rebuild
class DecoderReplayTail:
def __init__(
self,
*,
model: torch.nn.Module,
layer_ids: range,
row_step: int,
max_rows: int,
) -> None:
self._model = model
self._layer_ids = layer_ids
self._layers = None
self.runner = RowBucketGraphRunner(
row_step=row_step, max_rows=max_rows, label="Decoder tail graph"
)
@classmethod
def maybe_create(cls, model: torch.nn.Module) -> Optional[DecoderReplayTail]:
max_rows = envs.SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS.get()
if not (
model.tail_writes_source_kv
and is_cuda()
and max_rows > 0
# Borrowed graph-pool storage forbids capturing mid-serving.
and not envs.SGLANG_ENABLE_GRAPH_POOL_BORROW.get()
):
return None
from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW
return cls(
model=model,
layer_ids=range(model.late_layer_start, model.end_layer),
row_step=SWA_WINDOW,
max_rows=max_rows,
)
def plan(
self, tail: LateLayerTail, *, num_tokens: int, tail_in_graph: bool
) -> bool:
"""Whether this step runs its tail layers from a graph; also reports the
step's token count to the warm gate."""
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_triton,
)
self.runner.ready(num_tokens)
return not (
tail_in_graph
or get_parallel().attn_dp_size > 1
or get_is_capture_mode()
or is_unified_kv_triton()
or get_global_expert_distribution_recorder().recording
or not self.runner.warm
or tail.cp_metadata is not None
or self.runner.bucket_rows(tail.positions.shape[0]) is None
)
def run(
self,
*,
state: HcState,
positions: torch.Tensor,
input_ids: torch.Tensor,
input_ids_global: torch.Tensor,
forward_batch: ForwardBatch,
capture_dspark: bool,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
"""Returns the tail rows' materialized residual and pre and the DSpark
auxiliary states, as views of graph buffers valid until the next replay."""
leaves, has_pre, rebuild = _flatten_state(state)
num_state = len(leaves)
def body(inputs: List[torch.Tensor], batch: ForwardBatch):
residual, pre, aux = self._layer_stack(
rebuild(inputs[:num_state]),
positions=inputs[num_state],
input_ids=inputs[num_state + 1],
input_ids_global=inputs[num_state + 2],
forward_batch=batch,
capture_dspark=capture_dspark,
)
return [residual, pre, *aux]
residual, pre, *aux = self.runner.run(
key=(capture_dspark, has_pre),
live=[*leaves, positions, input_ids, input_ids_global],
forward_batch=forward_batch,
body=body,
context=self._break_context,
)
return residual, pre, aux
@contextmanager
def _break_context(self, tail_batch: ForwardBatch):
if self._layers is None:
self._layers = compute_attention_and_moe_layers(self._model)
layers = self._layers
with (
set_tc_piecewise_forward_context(
tail_batch,
layers.attention_layers,
None,
layers.moe_layers,
layers.moe_fusions,
dsa_indexers=layers.dsa_indexers,
mha_companion_layers=layers.mha_companion_layers,
),
enable_breakable_cuda_graph(),
_replay_tail_scope(),
):
yield
def _layer_stack(
self,
state: HcState,
*,
positions: torch.Tensor,
input_ids: torch.Tensor,
input_ids_global: torch.Tensor,
forward_batch: ForwardBatch,
capture_dspark: bool,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
"""The graphs' body; mirrors the model loop past the tail switch."""
model = self._model
aux: List[torch.Tensor] = []
residual_batch.start(forward_batch)
hidden_states = residual_batch.set_written(state, forward_batch)
try:
for i in self._layer_ids:
layer = model.layers[i]
assert layer.engram is None
if capture_dspark and i in model.dspark_layers_to_capture:
call = hc_boundary.LayerCall(input_ids, input_ids_global)
next_boundary = (
hc_boundary.HcRead(layer, True)
.bind(forward_batch, call)
.next_boundary
)
hidden_states = state = hc_boundary.materialize(
hidden_states, forward_batch, next=next_boundary
)
# The draft head reads the attention input of its target layers.
aux.append(state.residual.mean(dim=1))
ctx = (
nullcontext()
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
else get_global_expert_distribution_recorder().with_current_layer(i)
)
with ctx:
hidden_states = layer(
positions=positions,
hidden_states=hidden_states,
input_ids=input_ids,
forward_batch=forward_batch,
input_ids_global=input_ids_global,
)
state = residual_batch.read_output(
hidden_states, forward_batch, hc_boundary.FinalRead(), None
)
finally:
residual_batch.clear(forward_batch)
state = state.materialized(model.layers[self._layer_ids[-1]].hc_cfg)
return state.residual, state.pre, aux