diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 7037952167..268e6fad24 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1630,6 +1630,9 @@ class Envs: # Quantize the SWA fp8 KV cache from bf16-rounded values (matches # trainer-side QAT and the DSA-CP path) instead of fp32 registers. SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE = EnvBool(False) + # Decoder SWA bounded replay: eager prefill steps with up to this many tail rows + # run the tail layers from CUDA graphs (buckets of 128 rows); 0 keeps them eager. + SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS = EnvInt(2048) # Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every # GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or # fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere). diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index fc43cf13b3..aebd30824f 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -3314,8 +3314,14 @@ class DeepseekV4AttnBackend( from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( is_in_breakable_cuda_graph, ) + from sglang.srt.models.deepseek_v4_replay_tail import in_replay_tail_graph - return self.low_ratio_prefill_graph and is_in_breakable_cuda_graph() + # Decoder tail graphs run the low-ratio sources as an eager break. + return ( + self.low_ratio_prefill_graph + and is_in_breakable_cuda_graph() + and not in_replay_tail_graph() + ) def _low_ratio_compress_decode(self, layer, x, req, pos) -> None: # Projection layout and fused-write support are fixed together at load time. diff --git a/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py b/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py new file mode 100644 index 0000000000..157a9a0aae --- /dev/null +++ b/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py @@ -0,0 +1,220 @@ +"""Breakable CUDA graphs of a layer range that runs over a varying number of rows, +one graph per row bucket (a multiple of ``row_step``). + +Ops of the body that read per-step state (KV slots, attention metadata) must be +eager breaks; they read the step's real row count from +``forward_batch.global_num_token_non_padded_cpu``. +""" + +from __future__ import annotations + +import copy +import logging +import time +from typing import TYPE_CHECKING, Callable, ContextManager, List, Optional + +import torch + +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + set_graph_pool_id, +) +from sglang.srt.distributed.parallel_state import graph_capture +from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + BreakableCUDAGraph, + BreakableCUDAGraphCapture, +) +from sglang.srt.model_executor.runner_utils.pool import ( + get_or_create_global_graph_capture_stream, + get_or_create_global_graph_memory_pool, + graph_pool_capture_scope, + graph_pool_replay_scope, +) +from sglang.srt.runtime_context import get_parallel, get_schedule + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +logger = logging.getLogger(__name__) + +Body = Callable[[List[torch.Tensor], "ForwardBatch"], List[Optional[torch.Tensor]]] + + +class _Graph: + __slots__ = ("graph", "num_token_non_padded", "outputs", "batch") + + def __init__(self, graph, num_token_non_padded, outputs, batch): + self.graph = graph + # MoE top-k masks rows at or past this count; captured by address. + self.num_token_non_padded = num_token_non_padded + self.outputs = outputs + # Kernels write buffers cached on the capture batch and attention metadata. + self.batch = batch + + +class RowBucketGraphRunner: + def __init__(self, *, row_step: int, max_rows: int, label: str) -> None: + self.row_step = row_step + self.max_rows = max_rows - max_rows % row_step + self.label = label + self._graphs: dict[tuple, _Graph] = {} + self._static: dict[tuple, List[torch.Tensor]] = {} + self._warm = False + self.capture_seconds = 0.0 + self.capture_bytes = 0 + + def ready(self, num_tokens: int) -> bool: + """Whether graphs may run from this step on; every extend forward reports + its token count here. Kernels that size cached workspaces lazily by row + count reallocate them on a bigger step, which would leave earlier captures + reading freed memory; a full prefill chunk has grown them all.""" + if not self._warm: + chunk = get_schedule().chunked_prefill_size + self._warm = num_tokens >= (chunk if chunk and chunk > 0 else 8192) + return self._warm + + @property + def warm(self) -> bool: + return self._warm + + def bucket_rows(self, num_rows: int) -> Optional[int]: + if num_rows == 0 or num_rows > self.max_rows: + return None + return -(-num_rows // self.row_step) * self.row_step + + def run( + self, + *, + key: tuple, + live: List[torch.Tensor], + forward_batch: ForwardBatch, + body: Body, + context: Callable[[ForwardBatch], ContextManager], + ) -> List[Optional[torch.Tensor]]: + """Run ``body`` on the ``live`` rows from their bucket's graph, capturing + it on first use. ``key`` names whatever besides row count and input shapes + changes what ``body`` records; ``context`` wraps capture and replay with + the batch the eager breaks read. Returns the outputs cut to the live + rows, as views of graph buffers valid until the next replay.""" + num_rows = live[0].shape[0] + rows = self.bucket_rows(num_rows) + assert rows is not None, num_rows + # Inputs of one shape share their static buffers across keys. + input_key = tuple((t.dtype, t.shape[1:]) for t in live) + static = self._static_inputs(input_key, live) + for buf, t in zip(static, live): + buf[:num_rows].copy_(t) + graph_key = (key, input_key) + # The eager breaks read the step's real row count off this view. + step_batch = copy.copy(forward_batch) + step_batch.global_num_token_non_padded_cpu = num_rows + with context(step_batch): + if (rows, graph_key) not in self._graphs: + # Largest first, so smaller buckets reuse the pool memory the + # larger ones took; a bucket below this step's rows cannot be + # captured here (the breaks write every live row). + for r in range(self.max_rows, rows - 1, -self.row_step): + if (r, graph_key) not in self._graphs: + self._graphs[(r, graph_key)] = self._capture( + r, num_rows, static, forward_batch, body + ) + graph = self._graphs[(rows, graph_key)] + if graph.num_token_non_padded is not None: + graph.num_token_non_padded.fill_(num_rows) + with graph_pool_replay_scope(): + graph.graph.replay() + return [None if t is None else t[:num_rows] for t in graph.outputs] + + def _static_inputs(self, input_key, live) -> List[torch.Tensor]: + static = self._static.get(input_key) + if static is None: + static = self._static[input_key] = [ + t.new_zeros((self.max_rows, *t.shape[1:])) for t in live + ] + return static + + def _capture( + self, + rows: int, + num_rows: int, + static: List[torch.Tensor], + forward_batch: ForwardBatch, + body: Body, + ) -> _Graph: + start = time.perf_counter() + free_before = torch.cuda.mem_get_info()[0] + inputs = [buf[:rows] for buf in static] + # The layers read the step's batch; the copy's count tensor is the graph's. + # Host-side row counts read during capture cover the whole bucket. + capture_batch = copy.copy(forward_batch) + capture_batch.global_num_token_non_padded_cpu = rows + num_token_non_padded = None + if forward_batch.num_token_non_padded is not None: + num_token_non_padded = torch.full_like( + forward_batch.num_token_non_padded, num_rows + ) + capture_batch.num_token_non_padded = num_token_non_padded + return self._record( + lambda: body(inputs, capture_batch), + rows, + num_token_non_padded, + capture_batch, + start, + free_before, + ) + + def _record( + self, + body, + rows, + num_token_non_padded, + capture_batch, + start, + free_before, + ) -> _Graph: + pool = get_or_create_global_graph_memory_pool(torch.cuda) + set_graph_pool_id(pool) + tp_group = get_parallel().tp_group + graph = BreakableCUDAGraph() + with graph_capture(stream=get_or_create_global_graph_capture_stream()) as ctx: + # The warmups run eagerly; their KV writes repeat the replay's. + for _ in range(2): + torch.cuda.synchronize() + tp_group.barrier() + body() + with ( + graph_pool_capture_scope(), + BreakableCUDAGraphCapture( + cuda_graph=graph, + pool=pool, + stream=ctx.stream, + barrier_fn=tp_group.barrier, + ), + ): + outputs = body() + torch.cuda.current_stream().wait_stream(ctx.stream) + torch.cuda.synchronize() + recorded = _Graph( + graph, + num_token_non_padded, + outputs, + (capture_batch, get_attn_backend().forward_metadata), + ) + seconds = time.perf_counter() - start + used = free_before - torch.cuda.mem_get_info()[0] + self.capture_seconds += seconds + self.capture_bytes += used + if get_parallel().tp_rank == 0: + logger.info( + "%s: captured %d rows (%d segments) in %.2f s, %.3f GiB; " + "%d graphs, %.2f s, %.3f GiB", + self.label, + rows, + len(graph._segments), + seconds, + used / 2**30, + len(self._graphs) + 1, + self.capture_seconds, + self.capture_bytes / 2**30, + ) + return recorded diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 68799da975..bdb56a1bc1 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -172,6 +172,11 @@ from sglang.srt.models.deepseek_v4_platform_stage import ( NonCudaDecoderMixin, forward_layer_fused_boundary, ) +from sglang.srt.models.deepseek_v4_replay_tail import ( + DecoderReplayTail, + bcg_tail_kv_store, + in_replay_tail_graph, +) from sglang.srt.models.deepseek_v41_vit import Aligner, ViT from sglang.srt.models.utils import WeightsMapper from sglang.srt.multimodal.deepseek_v41_image_processing import ( @@ -2305,6 +2310,10 @@ class MQALayer(MqaAttentionBase): positions=global_positions, ) kv = None + elif in_replay_tail_graph(): + assert not fuse_q_rope + bcg_tail_kv_store(self, x_linear, positions, qkv_a) + kv = None else: self._compute_kv_to_cache( x_linear, @@ -2324,7 +2333,10 @@ class MQALayer(MqaAttentionBase): if ( forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph() - and not getattr(attn_backend, "low_ratio_prefill_graph", False) + and ( + not getattr(attn_backend, "low_ratio_prefill_graph", False) + or in_replay_tail_graph() + ) ): bcg_deepseek_v4_low_ratio_sources(self, x, q_lora, positions) else: @@ -3532,6 +3544,8 @@ DeepseekV4DecoderLayer = ( class DeepseekV4Model(nn.Module): fall_back_to_pt_during_load = False + # Class-level default for instances built without running __init__. + decoder_replay_tail: Optional[DecoderReplayTail] = None def __init__( self, @@ -3698,6 +3712,7 @@ class DeepseekV4Model(nn.Module): and config.vision_n_layers > 0 and not getattr(config, "language_model_only", False) ) + self.decoder_replay_tail = DecoderReplayTail.maybe_create(self) def get_input_embeddings(self) -> nn.Module: return self.embed_tokens @@ -3831,6 +3846,13 @@ class DeepseekV4Model(nn.Module): ), in_graph=tail_in_graph, ) + use_tail_graph = ( + self.decoder_replay_tail is not None + and tail is not None + and self.decoder_replay_tail.plan( + tail, num_tokens=input_ids.shape[0], tail_in_graph=tail_in_graph + ) + ) tail_sentinel = positions.new_zeros((1,)) if tail_in_graph else None saved_full = None # The HIP defer_post gate excludes residual readers. CUDA materializes @@ -3890,6 +3912,19 @@ class DeepseekV4Model(nn.Module): positions = tail.positions if hash_ids is not None: hash_ids = tail.rows(hash_ids) + if use_tail_graph: + residual, pre, aux = self.decoder_replay_tail.run( + state=state, + positions=positions, + input_ids=input_ids, + input_ids_global=input_ids_global, + forward_batch=forward_batch, + capture_dspark=capture_dspark, + ) + dspark_aux_hidden_states.extend(aux) + residual_batch.clear(forward_batch) + attn_backend.exit_late_layer_tail(saved_full, forward_batch) + return residual, pre, tail if late_dp_counts is not None and i == self.late_layer_start: saved_dp = enter_late_layer_dp(forward_batch, late_dp_counts) if get_moe_a2a_backend().is_none(): diff --git a/python/sglang/srt/models/deepseek_v4_replay_tail.py b/python/sglang/srt/models/deepseek_v4_replay_tail.py new file mode 100644 index 0000000000..1327615099 --- /dev/null +++ b/python/sglang/srt/models/deepseek_v4_replay_tail.py @@ -0,0 +1,258 @@ +"""DeepSeek-V4.1's decoder-replay tail layers (``late_layer_start`` on), replayed +from `RowBucketGraphRunner` graphs on eager prefill steps. The SWA KV store, the +attention and the low-ratio sources stay eager breaks of those graphs. +""" + +from __future__ import annotations + +from contextlib import contextmanager, nullcontext +from typing import TYPE_CHECKING, List, Optional, Tuple + +import torch + +from sglang.srt.environ import envs +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + check_cuda_graph_backend, +) +from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.model_executor.model_runner_components.layer_setup import ( + compute_attention_and_moe_layers, +) +from sglang.srt.model_executor.runner.row_bucket_graph_runner import ( + RowBucketGraphRunner, +) +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + eager_on_graph, + enable_breakable_cuda_graph, +) +from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.context_manager import ( + get_tc_piecewise_forward_context, + set_tc_piecewise_forward_context, +) +from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode +from sglang.srt.models import deepseek_v4_boundary as hc_boundary +from sglang.srt.models.deepseek_v4_mhc import HcState +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import is_cuda + +if TYPE_CHECKING: + from sglang.srt.layers.attention.deepseek_v4_backend import LateLayerTail + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +_in_replay_tail = False + + +def in_replay_tail_graph() -> bool: + """True while the tail layers capture or replay a decoder tail graph.""" + return _in_replay_tail + + +@contextmanager +def _replay_tail_scope(): + global _in_replay_tail + _in_replay_tail = True + try: + yield + finally: + _in_replay_tail = False + + +def _tail_kv_store(attention, x, positions, qkv_a) -> None: + # The SWA store writes the step's live tail slots, so a tail graph breaks here. + forward_batch = get_tc_piecewise_forward_context().forward_batch + num_rows = forward_batch.global_num_token_non_padded_cpu + if num_rows == 0: + return + attention._compute_kv_to_cache( + x[:num_rows], + positions[:num_rows], + forward_batch, + get_attn_backend(), + qkv_a=None if qkv_a is None else qkv_a[:num_rows], + ) + + +bcg_tail_kv_store = eager_on_graph(True)(_tail_kv_store) + + +def _flatten_state(state: HcState): + """The state's row tensors, whether ``pre`` is among them, and the inverse. + The normed input and mix stats are dropped (the first tail layer rebuilds + them), so every step keys the same graphs.""" + assert isinstance(state.streams, torch.Tensor), "tail graphs are CUDA only" + has_pre = state.pre is not None + leaves = [state.streams, state.pre] if has_pre else [state.streams] + + def rebuild(tensors: List[torch.Tensor]) -> HcState: + return HcState(tensors[0], tensors[1] if has_pre else None) + + return leaves, has_pre, rebuild + + +class DecoderReplayTail: + def __init__( + self, + *, + model: torch.nn.Module, + layer_ids: range, + row_step: int, + max_rows: int, + ) -> None: + self._model = model + self._layer_ids = layer_ids + self._layers = None + self.runner = RowBucketGraphRunner( + row_step=row_step, max_rows=max_rows, label="Decoder tail graph" + ) + + @classmethod + def maybe_create(cls, model: torch.nn.Module) -> Optional[DecoderReplayTail]: + max_rows = envs.SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS.get() + if not ( + model.tail_writes_source_kv + and is_cuda() + and max_rows > 0 + # Borrowed graph-pool storage forbids capturing mid-serving. + and not envs.SGLANG_ENABLE_GRAPH_POOL_BORROW.get() + ): + return None + from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW + + return cls( + model=model, + layer_ids=range(model.late_layer_start, model.end_layer), + row_step=SWA_WINDOW, + max_rows=max_rows, + ) + + def plan( + self, tail: LateLayerTail, *, num_tokens: int, tail_in_graph: bool + ) -> bool: + """Whether this step runs its tail layers from a graph; also reports the + step's token count to the warm gate.""" + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + is_unified_kv_triton, + ) + + self.runner.ready(num_tokens) + return not ( + tail_in_graph + or get_parallel().attn_dp_size > 1 + or get_is_capture_mode() + or is_unified_kv_triton() + or get_global_expert_distribution_recorder().recording + or not self.runner.warm + or tail.cp_metadata is not None + or self.runner.bucket_rows(tail.positions.shape[0]) is None + ) + + def run( + self, + *, + state: HcState, + positions: torch.Tensor, + input_ids: torch.Tensor, + input_ids_global: torch.Tensor, + forward_batch: ForwardBatch, + capture_dspark: bool, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]: + """Returns the tail rows' materialized residual and pre and the DSpark + auxiliary states, as views of graph buffers valid until the next replay.""" + leaves, has_pre, rebuild = _flatten_state(state) + num_state = len(leaves) + + def body(inputs: List[torch.Tensor], batch: ForwardBatch): + residual, pre, aux = self._layer_stack( + rebuild(inputs[:num_state]), + positions=inputs[num_state], + input_ids=inputs[num_state + 1], + input_ids_global=inputs[num_state + 2], + forward_batch=batch, + capture_dspark=capture_dspark, + ) + return [residual, pre, *aux] + + residual, pre, *aux = self.runner.run( + key=(capture_dspark, has_pre), + live=[*leaves, positions, input_ids, input_ids_global], + forward_batch=forward_batch, + body=body, + context=self._break_context, + ) + return residual, pre, aux + + @contextmanager + def _break_context(self, tail_batch: ForwardBatch): + if self._layers is None: + self._layers = compute_attention_and_moe_layers(self._model) + layers = self._layers + with ( + set_tc_piecewise_forward_context( + tail_batch, + layers.attention_layers, + None, + layers.moe_layers, + layers.moe_fusions, + dsa_indexers=layers.dsa_indexers, + mha_companion_layers=layers.mha_companion_layers, + ), + enable_breakable_cuda_graph(), + _replay_tail_scope(), + ): + yield + + def _layer_stack( + self, + state: HcState, + *, + positions: torch.Tensor, + input_ids: torch.Tensor, + input_ids_global: torch.Tensor, + forward_batch: ForwardBatch, + capture_dspark: bool, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]: + """The graphs' body; mirrors the model loop past the tail switch.""" + model = self._model + aux: List[torch.Tensor] = [] + residual_batch.start(forward_batch) + hidden_states = residual_batch.set_written(state, forward_batch) + try: + for i in self._layer_ids: + layer = model.layers[i] + assert layer.engram is None + if capture_dspark and i in model.dspark_layers_to_capture: + call = hc_boundary.LayerCall(input_ids, input_ids_global) + next_boundary = ( + hc_boundary.HcRead(layer, True) + .bind(forward_batch, call) + .next_boundary + ) + hidden_states = state = hc_boundary.materialize( + hidden_states, forward_batch, next=next_boundary + ) + # The draft head reads the attention input of its target layers. + aux.append(state.residual.mean(dim=1)) + ctx = ( + nullcontext() + if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) + else get_global_expert_distribution_recorder().with_current_layer(i) + ) + with ctx: + hidden_states = layer( + positions=positions, + hidden_states=hidden_states, + input_ids=input_ids, + forward_batch=forward_batch, + input_ids_global=input_ids_global, + ) + state = residual_batch.read_output( + hidden_states, forward_batch, hc_boundary.FinalRead(), None + ) + finally: + residual_batch.clear(forward_batch) + state = state.materialized(model.layers[self._layer_ids[-1]].hc_cfg) + return state.residual, state.pre, aux diff --git a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py index 9a0ab7c7d7..dab7f2ef83 100644 --- a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py +++ b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py @@ -89,6 +89,7 @@ class TestDeepseekV41BCGMultimodal(CustomTestCase): pp_group=SimpleNamespace(world_size=1), engram_hasher=lambda *a: torch.zeros((3, 1), dtype=torch.long), late_layer_start=None, + decoder_replay_tail=None, start_layer=0, end_layer=1, layers=[layer],