From 8111841103ca44bec779289c1da8d29cec55cc0a Mon Sep 17 00:00:00 2001 From: Yizheng Jiao Date: Mon, 19 Oct 2026 01:29:14 -0700 Subject: [PATCH] [Perf] deepseek-v4.1: Replay the breakable prefill CUDA graph at any context length (#43733) Co-authored-by: Oasis-Git Co-authored-by: hnyls2002 --- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 26 +- .../srt/layers/attention/base_attn_backend.py | 3 + .../layers/attention/deepseek_v4_backend.py | 226 ++++++++++++++---- .../deepseek_v4_backend_hip_radix.py | 13 +- .../dsv4/v41_indexer/sparse_table.py | 122 +++++++++- .../layers/attention/hybrid_attn_backend.py | 4 + .../srt/layers/attention/tbo_backend.py | 4 + .../runner/prefill_cuda_graph_runner.py | 10 + .../breakable_cuda_graph.py | 8 + .../unittests/dsv4/test_deepseek_v4.py | 4 + .../test_dsv41_late_layer_tail_readers.py | 61 +++++ 11 files changed, 389 insertions(+), 92 deletions(-) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index a78f870550..6a1f66d862 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -16,7 +16,7 @@ from sglang.srt.arg_groups.overrides import ( run_post_process_pass, ) from sglang.srt.environ import envs -from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase +from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import is_gfx95_supported, is_npu @@ -364,30 +364,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: "both servers use the same block size and target/draft KV layout." ) - prefill_graph = cfg.cuda_graph_config.prefill - cp_breakable_prefill = ( - cfg.enable_prefill_cp - and cfg.cp_strategy == "interleave" - and cfg.tp_size > 1 - and prefill_graph.backend == Backend.BREAKABLE - ) - if ( - prefill_graph.backend != Backend.DISABLED - and prefill_graph.max_seq_len is None - and not cp_breakable_prefill - ): - declare_resolution( - server_args, - "validate_deepseek_v41_features", - cuda_graph_config=with_phase( - cfg.cuda_graph_config, Phase.PREFILL, max_seq_len=16 * 1024 - ), - ) - logger.warning( - "Setting cuda_graph_config[prefill].max_seq_len to 16384 for " - "DeepSeek-V4.1; longer contexts run eager prefill." - ) - incompatible = ( ( "the prefill CUDA graph", diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index 03e16a0bc6..af3c9bea07 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -171,6 +171,9 @@ class AttentionBackend(ABC): # True when prefill graph metadata can use ForwardBatch.max_seq_len_override. supports_prefill_cuda_graph_max_context_size: bool = False + # Replayed context limit when the user sets none; None when nothing the backend + # captures depends on the context length. + prefill_cuda_graph_default_max_seq_len: Optional[int] = None dllm_attention = None def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds: diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index aebd30824f..0ac54d0ad8 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -90,6 +90,9 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import ( make_candidate_indexer, make_full_topk_indexer, ) +from sglang.srt.layers.attention.dsv4.v41_indexer.sparse_table import ( + SparseTableBackend, +) from sglang.srt.layers.attention.verify_mask import ( VerifyMask, maybe_create_verify_mask, @@ -114,6 +117,14 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardMode, get_server_return_hidden_states_mode, ) +from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import ( + eager_on_graph, + is_capturing, +) +from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( + get_tc_piecewise_forward_context, +) from sglang.srt.runtime_context import ( get_exec, get_parallel, @@ -262,10 +273,6 @@ def _maybe_precompute_flashmla_sched_meta( def _low_ratio_source_projections( layer, x, q_lora, positions, bufs, run_compressor, run_indexer ): - from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( - get_tc_piecewise_forward_context, - ) - real = ( get_tc_piecewise_forward_context().forward_batch.global_num_token_non_padded_cpu ) @@ -292,25 +299,22 @@ def _low_ratio_source_projections( if score is not None: put("score", score) if run_indexer and layer.indexer is not None: + # Only the captured consumers read these; a full top-k layer projects + # once inside its eager break (_index_q_and_weights), so skip them there. indexer = layer.indexer put("q", indexer.queries(q_lora[:real], layer.freqs_cis[positions[:real]])) put("w", indexer.head_weights(x[:real])) -def _bcg_low_ratio_source_projections(*args): - from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import ( - eager_on_graph, +def _prefill_graph_full_topk(layer, x, q_lora, pos): + forward_batch = get_tc_piecewise_forward_context().forward_batch + return get_attn_backend()._prefill_graph_full_topk_break( + layer, x, q_lora, pos, forward_batch ) - global _bcg_low_ratio_source_projections_fn - if _bcg_low_ratio_source_projections_fn is None: - _bcg_low_ratio_source_projections_fn = eager_on_graph(True)( - _low_ratio_source_projections - ) - return _bcg_low_ratio_source_projections_fn(*args) - -_bcg_low_ratio_source_projections_fn = None +_bcg_prefill_graph_full_topk = eager_on_graph(True)(_prefill_graph_full_topk) +_bcg_low_ratio_source_projections = eager_on_graph(True)(_low_ratio_source_projections) def _as_int_list(values) -> Optional[List[int]]: @@ -1009,11 +1013,16 @@ def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor: return torch.nn.functional.pad(t, (0, num_rows - t.shape[0])) -# Rows per logits chunk for the ratio-1/2 indexer inside the prefill CUDA graph; -# its width is the graph's max_seq_len, and longer contexts replay eagerly. +# Rows per logits chunk of the ratio-1/2 indexer metadata under the prefill CUDA +# graph; its width is _prefill_graph_context_width. _PREFILL_GRAPH_INDEXER_ROW_CHUNK = 2048 +# The candidate window (2,048 blocks of 8 tokens): inside it the captured full +# top-k equals the two-level selection. +_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024 + + def _prefill_graph_max_seq_len() -> Optional[int]: from sglang.srt.runtime_context import get_exec @@ -1071,17 +1080,30 @@ class DSV4Metadata: self.sparse_prefill_cache = None self.prefill_shared_reads_snapshotted = False - def refresh_for_breakable_cuda_graph_replay_(self, static_metadata: DSV4Metadata): + def refresh_for_breakable_cuda_graph_replay_( + self, + static_metadata: DSV4Metadata, + *, + two_level: bool, + ): self.core_attn_metadata.refresh_for_breakable_cuda_graph_replay_( static_metadata.core_attn_metadata ) maybe_copy_inplace(self.indexer_metadata, src=static_metadata.indexer_metadata) - maybe_copy_inplace( - self.c1_indexer_metadata, src=static_metadata.c1_indexer_metadata - ) - maybe_copy_inplace( - self.c2_indexer_metadata, src=static_metadata.c2_indexer_metadata - ) + if not two_level: + # The captured full top-k fallback reads the ratio-1/2 lengths and + # page tables by address: refresh them in place. + maybe_copy_inplace( + self.c1_indexer_metadata, src=static_metadata.c1_indexer_metadata + ) + maybe_copy_inplace( + self.c2_indexer_metadata, src=static_metadata.c2_indexer_metadata + ) + else: + # Two-level candidates: only the eager full top-k breaks read these, so + # they take the live objects; captured consumers read the bridge tensors. + self.c1_indexer_metadata = static_metadata.c1_indexer_metadata + self.c2_indexer_metadata = static_metadata.c2_indexer_metadata maybe_copy_inplace( self.low_ratio_req_indices, src=static_metadata.low_ratio_req_indices ) @@ -1263,6 +1285,8 @@ class DeepseekV4AttnBackend( self._bcg_tail_buffers: Optional[tuple] = None self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {} self._bcg_tail_num_token_non_padded: Optional[torch.Tensor] = None + # Prefill CUDA graph source projection buffers per ratio, newest first. + self._source_proj_bufs: Dict[int, list] = {} self.dsa_topk_backend: DSATopKBackend = DSATopKBackend.resolve(model_runner) self.dsv4_prefill_backend = getattr(kernel, "dsv4_prefill_backend", "auto") if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend): @@ -1601,7 +1625,7 @@ class DeepseekV4AttnBackend( self, core: DSV4AttnMetadata, compress_ratio: int ) -> PagedIndexerMetadata: num_pages = core.page_table.shape[1] - max_seq_len = _prefill_graph_max_seq_len() + max_seq_len = self._prefill_graph_seq_limit() if max_seq_len is not None: num_pages = min(max_seq_len // self.page_size, num_pages) index_page_size = self.token_to_kv_pool.get_index_k_page_size(compress_ratio) @@ -1636,11 +1660,11 @@ class DeepseekV4AttnBackend( ) def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool: - max_seq_len = _prefill_graph_max_seq_len() + max_seq_len = self._prefill_graph_seq_limit() seq_lens_cpu = forward_batch.seq_lens_cpu - if max_seq_len is None or seq_lens_cpu is None or seq_lens_cpu.numel() == 0: + if seq_lens_cpu is None or seq_lens_cpu.numel() == 0: return True - if int(seq_lens_cpu.max().item()) > max_seq_len: + if max_seq_len is not None and int(seq_lens_cpu.max().item()) > max_seq_len: return False if self.enable_decoder_swa_bounded_replay: extend_lens_cpu = forward_batch.extend_seq_lens_cpu @@ -2950,11 +2974,7 @@ class DeepseekV4AttnBackend( def init_forward_metadata_for_breakable_cuda_graph_capture( self, forward_batch: ForwardBatch ): - max_seq_len = forward_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE - if self.low_ratios == (1, 2): - graph_context_limit = _prefill_graph_max_seq_len() - if graph_context_limit is not None: - max_seq_len = min(max_seq_len, graph_context_limit) + max_seq_len = self._prefill_graph_context_width(forward_batch) self.forward_metadata = self._build_forward_metadata( forward_batch, max_seq_len_override=max_seq_len, @@ -2980,6 +3000,19 @@ class DeepseekV4AttnBackend( ) return self.forward_metadata + def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int: + """One context width for the full rows' and the late-layer tail's metadata.""" + if self._prefill_graph_replays_any_context(): + seq_lens_cpu = forward_batch.seq_lens_cpu + if seq_lens_cpu is not None and len(seq_lens_cpu) > 0: + return int(seq_lens_cpu.max().item()) + max_seq_len = forward_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE + if self.low_ratios == (1, 2): + graph_context_limit = self._prefill_graph_seq_limit() + if graph_context_limit is not None: + max_seq_len = min(max_seq_len, graph_context_limit) + return max_seq_len + def _share_bcg_low_ratio_index_buffers(self, core) -> None: # BCG buckets replay serially. Share only full-metadata output buffers; # both ratios must survive the later full-to-tail metadata transition. @@ -3002,7 +3035,7 @@ class DeepseekV4AttnBackend( def _source_projection_buffers(self, num_tokens: int, ratio: int) -> dict: cfg = self.model_runner.model_config.hf_text_config heads, dim = int(cfg.index_n_heads), int(cfg.index_head_dim) - sets = getattr(self, "_source_proj_bufs", None) or {} + sets = self._source_proj_bufs have = sets.get(ratio) if have is None or have[0]["q"].shape[0] < num_tokens: latent = self.model_runner.model_config.head_dim @@ -3022,7 +3055,6 @@ class DeepseekV4AttnBackend( bufs["score"] = zeros(num_tokens, latent, dtype=torch.float32) # Keep the previous allocations alive: captured graphs still hold them. sets[ratio] = [bufs] + (have or []) - self._source_proj_bufs = sets bufs = sets[ratio][0] return {name: buf[:num_tokens] for name, buf in bufs.items()} @@ -3038,20 +3070,17 @@ class DeepseekV4AttnBackend( metadata_batch = ( static_forward_batch if static_forward_batch is not None else forward_batch ) - max_seq_len = ( - metadata_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE - ) - if self.low_ratios == (1, 2): - graph_context_limit = _prefill_graph_max_seq_len() - if graph_context_limit is not None: - max_seq_len = min(max_seq_len, graph_context_limit) + max_seq_len = self._prefill_graph_context_width(metadata_batch) static_metadata = self._build_forward_metadata( metadata_batch, max_seq_len_override=max_seq_len, use_prefill_cuda_graph=True, ) assert isinstance(capture_metadata, DSV4Metadata) - capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata) + capture_metadata.refresh_for_breakable_cuda_graph_replay_( + static_metadata, + two_level=self._prefill_graph_two_level(), + ) self.forward_metadata = capture_metadata self._activate_request_window() self.tail_forward_metadata = ( @@ -3075,7 +3104,10 @@ class DeepseekV4AttnBackend( if held is None: self._bcg_tail_metadata[rows] = fresh return fresh - held.refresh_for_breakable_cuda_graph_replay_(fresh) + held.refresh_for_breakable_cuda_graph_replay_( + fresh, + two_level=self._prefill_graph_two_level(), + ) held.core_attn_metadata.swa_out_cache_loc = ( fresh.core_attn_metadata.swa_out_cache_loc ) @@ -3212,15 +3244,32 @@ class DeepseekV4AttnBackend( and self._low_ratio_in_prefill_graph() ): bufs = self._source_projection_buffers(x.shape[0], layer.compress_ratio) + two_level = self._prefill_graph_two_level() + captured_index = ( + run_indexer + and layer.indexer is not None + and (layer.indexer.uses_candidates or not two_level) + ) _bcg_low_ratio_source_projections( - layer, x, q_lora, pos, bufs, run_compressor, run_indexer + layer, x, q_lora, pos, bufs, run_compressor, captured_index ) if run_compressor and layer.compressor is not None: self._low_ratio_compress_torch( layer, x, req, pos, projected=(bufs["kv"], bufs.get("score")) ) if run_indexer and layer.indexer is not None: - self._low_ratio_index_topk_captured(layer, bufs["q"], bufs["w"]) + if captured_index: + # Captured: sparse scores of the published blocks only, so + # nothing here depends on the context length. + self._low_ratio_index_topk_captured(layer, bufs["q"], bufs["w"]) + else: + # Eager break: dense scoring over the live context; the + # candidate source hands its block table to the consumers. + out = _bcg_prefill_graph_full_topk(layer, x, q_lora, pos) + if layer.indexer.is_candidate_source: + self.forward_metadata.candidate_metadata = ( + self.prefill_candidates.bind_prefill_graph_table(out) + ) return if run_compressor and layer.compressor is not None: self._low_ratio_compress(layer, x, req, pos, forward_batch) @@ -3660,12 +3709,6 @@ class DeepseekV4AttnBackend( else self.forward_metadata.c2_indexer_metadata ) assert metadata is not None, f"no prefill graph indexer metadata for {ratio = }" - if indexer.uses_candidates or indexer.is_candidate_source: - # Inside the window every block is a candidate: the full top-k applies. - width = metadata.max_compressed_seq_len - assert ( - width <= indexer.candidate_topk_blocks * indexer.candidate_block_size - ), f"prefill graph indexer width {width} exceeds the candidate window" inputs = CapturedPrefillInputs( indexer=indexer, layer_id=layer.layer_id, @@ -3675,6 +3718,19 @@ class DeepseekV4AttnBackend( out_page_indices=core.sparse_page_indices(ratio), out_raw_indices=core.sparse_raw_indices(ratio), ) + # At any width: inside the candidate window the published table is the + # identity, and past it the full top-k would be wrong. + if indexer.uses_candidates and self._prefill_graph_two_level(): + self.prefill_candidates.consume_prefill_captured( + inputs, self.forward_metadata.candidate_metadata + ) + return + if indexer.uses_candidates or indexer.is_candidate_source: + # Inside the window every block is a candidate: the full top-k applies. + width = metadata.max_compressed_seq_len + assert ( + width <= indexer.candidate_topk_blocks * indexer.candidate_block_size + ), f"prefill graph indexer width {width} exceeds the candidate window" self.full_topk_indexer.topk_prefill_captured(inputs) def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode): @@ -3763,6 +3819,72 @@ class DeepseekV4AttnBackend( return self.forward_metadata.candidate_metadata = published + def _prefill_graph_two_level(self) -> bool: + """Whether the full top-k layers score at eager breaks and the consumers score + the published candidate blocks inside the prefill graph.""" + return isinstance(self.prefill_candidates, SparseTableBackend) + + def _prefill_graph_replays_any_context(self) -> bool: + """Whether a prefill graph replay admits any context length: with two-level + candidates only c4/c128 compress metadata would be captured at a width.""" + return ( + self.low_ratio_prefill_graph + and self._prefill_graph_two_level() + and not self.has_c4 + and not self.has_c128 + ) + + @property + def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]: + """DeepSeek-V4.1's captured indexer width; prefill context parallelism builds + its own metadata and needs none.""" + if ( + self.low_ratios != (1, 2) + or get_parallel().attn_cp_size > 1 + or self._prefill_graph_replays_any_context() + ): + return None + return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN + + def _prefill_graph_seq_limit(self) -> Optional[int]: + """The replayed context limit: cuda_graph_config[prefill].max_seq_len when + the user sets it, else this backend's default.""" + max_seq_len = _prefill_graph_max_seq_len() + if max_seq_len is not None: + return max_seq_len + return self.prefill_cuda_graph_default_max_seq_len + + def _prefill_graph_full_topk_break(self, layer, x, q_lora, pos, forward_batch): + """One full top-k layer at an eager break of the prefill graph, over the live + batch at any context length: writes the sparse indices the attention break + reads, and the candidate source returns its table for the captured consumers.""" + indexer = layer.indexer + real = forward_batch.global_num_token_non_padded_cpu + if real is None: + real = x.shape[0] + if real == 0: + # An idle DP-attention rank replays on one fabricated request of the + # padded token count (ForwardBatch idle materialization); nothing reads it. + real = forward_batch.extend_num_tokens + # The metadata in use holds the rows x has: the late-layer tail's when the + # source is the tail's first layer, which are not the extend's leading rows. + req = self.forward_metadata.low_ratio_req_indices[:real] + inputs = self._make_low_ratio_prefill_indexer_inputs( + layer, + x[:real], + q_lora[:real], + req, + pos[:real], + forward_batch, + None, + ) + if not indexer.is_candidate_source: + self.full_topk_indexer.topk_prefill(inputs) + return () + return self.prefill_candidates.publish_prefill_graph( + inputs, x.shape[0], shared=is_capturing() + ) + def get_swa_out_cache_loc(self, forward_batch: ForwardBatch) -> torch.Tensor: """Idle always re-translates at store time: its metadata may be stale, and translating the zero-padded out_cache_loc writes to the dummy slot.""" diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 275b5125f7..ee6082a94f 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -66,7 +66,6 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4AttnBackend, LateLayerTail, _as_int_list, - _prefill_graph_max_seq_len, _tail_rows, late_layer_num_tokens, late_layer_whole_extend, @@ -890,6 +889,13 @@ class DeepseekV4HipRadixBackend( # This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py. # PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager. can_run_prefill_cuda_graph = DeepseekV4AttnBackend.can_run_prefill_cuda_graph + prefill_cuda_graph_default_max_seq_len = ( + DeepseekV4AttnBackend.prefill_cuda_graph_default_max_seq_len + ) + _prefill_graph_replays_any_context = ( + DeepseekV4AttnBackend._prefill_graph_replays_any_context + ) + _prefill_graph_seq_limit = DeepseekV4AttnBackend._prefill_graph_seq_limit _low_ratio_in_prefill_graph = DeepseekV4AttnBackend._low_ratio_in_prefill_graph _low_ratio_compress = DeepseekV4AttnBackend._low_ratio_compress _low_ratio_compress_decode = DeepseekV4AttnBackend._low_ratio_compress_decode @@ -2392,14 +2398,13 @@ class DeepseekV4HipRadixBackend( seq_lens_cpu=seq_lens_cpu, ) - @staticmethod - def _graph_metadata_max_seq_len(forward_batch: ForwardBatch) -> Optional[int]: + def _graph_metadata_max_seq_len(self, forward_batch: ForwardBatch) -> Optional[int]: """ Capture and replay build the metadata at this fixed context length, so a replay's metadata is never larger than the captured one. Longer contexts run eager. """ - return forward_batch.max_seq_len_override or _prefill_graph_max_seq_len() + return forward_batch.max_seq_len_override or self._prefill_graph_seq_limit() def init_forward_metadata_for_breakable_cuda_graph_capture( self, forward_batch: ForwardBatch diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py index efc7ede22c..cbdca3e415 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py @@ -41,11 +41,13 @@ from .scoring import ( get_deep_gemm_decode_data, get_deep_gemm_prefill_data, get_index_k_cache, + quantize_index_q, score_tiles, topk_ragged, ) from .types import ( CandidateMetadata, + CapturedPrefillInputs, DecodeInputs, PrefillInputs, get_tail_row_indices, @@ -80,14 +82,25 @@ class _SparsePrefillTable(_SparseTable): request_ids: torch.Tensor # [rows] int32 row-pair ids, see _row_pair_ids q_dtype: torch.dtype page_size: int # index-K pool page size (slots) - rows_per_request: List[int] # rows of each request, in row order + # rows of each request, in row order; None for rows gathered without them + rows_per_request: Optional[List[int]] def tail(self, rows_per_request: List[int]) -> _SparsePrefillTable: + assert self.rows_per_request is not None, ( + "a table of gathered rows has no requests" + ) rows = get_tail_row_indices( full_rows_per_request=self.rows_per_request, tail_rows_per_request=rows_per_request, device=self.blocks.device, ) + return self.gather_rows(rows, rows_per_request) + + def gather_rows( + self, rows: torch.Tensor, rows_per_request: Optional[List[int]] = None + ) -> _SparsePrefillTable: + """The table restricted to ``rows``: a gather, graph-safe for a persistent + index buffer.""" compress_lens = self.compress_lens[rows].contiguous() _, valid_lens = candidate_row_lens(compress_lens, self.blocks.shape[1]) return _build_prefill_table( @@ -121,6 +134,8 @@ class SparseTableBackend: self._cached_row_ids: Optional[torch.Tensor] = None # captured graphs keep reading the buffers they saw self._retired_cached_row_ids: list = [] + # per layer: the table tensors every captured prefill bucket reads + self._graph_table_bufs: dict = {} def publish_prefill(self, inputs: PrefillInputs): inputs.reset_outputs() @@ -172,6 +187,80 @@ class SparseTableBackend: ) data.write_page_indices(inputs) + def consume_prefill_captured( + self, inputs: CapturedPrefillInputs, published: Optional[_SparseTable] + ) -> None: + """A consumer under the prefill CUDA graph: sparse scores of the published + blocks, top-k as pool slots and as compressed positions.""" + assert published is not None, ( + "prefill graph candidate table missing: the candidate source layer " + "must run under the same graph before its consumers" + ) + indexer = inputs.indexer + q_fp4, q_sf = quantize_index_q(inputs.q) + rows = q_fp4.shape[0] + k_cache = get_index_k_cache( + token_to_kv_pool=self.token_to_kv_pool, + layer_id=inputs.layer_id, + page_size=inputs.paged_metadata.compressed_page_size, + ) + logits = _prefill_table_logits(published, q_fp4, q_sf, inputs.weights, k_cache) + topk = min(indexer.index_topk, published.blocks.shape[1] * CANDIDATE_BLOCK_SIZE) + + def transform(blocks: torch.Tensor, out: torch.Tensor) -> None: + target = out[:rows, :topk] + if target.is_contiguous(): + topk_transform_sparse(logits, published.valid_lens, blocks, target) + return + tmp = torch.empty((rows, topk), dtype=torch.int32, device=out.device) + topk_transform_sparse(logits, published.valid_lens, blocks, tmp) + target.copy_(tmp) + + if inputs.out_page_indices is not None: + transform(published.phys_blocks, inputs.out_page_indices) + transform(published.blocks, inputs.out_raw_indices) + + def publish_prefill_graph( + self, inputs: PrefillInputs, num_rows: int, *, shared: bool + ) -> tuple: + """The candidate source's table over the graph's ``num_rows`` rows. Buckets may + share one buffer: graphs replay one at a time, and the shapes, DeepGEMM's + schedule included, depend only on the row and block counts.""" + published = self.publish_prefill(inputs) + assert published is not None, "candidate source published no sparse table" + live = published.blocks.shape[0] + rows = torch.arange(num_rows, device=published.blocks.device).clamp_max( + live - 1 + ) + table = published.gather_rows(rows) + out = (table.blocks, table.schedule, table.phys_blocks, table.valid_lens) + return self._shared_graph_table(inputs.layer_id, out) if shared else out + + def bind_prefill_graph_table(self, tensors: tuple) -> _SparseTable: + """Capture time: the table the captured consumers read, over the tensors the + candidate source's break refreshes at every replay.""" + blocks, schedule, phys_blocks, valid_lens = tensors + ready = torch.cuda.Event() + ready.record() + return _SparseTable( + blocks=blocks, + schedule=schedule, + phys_blocks=phys_blocks, + valid_lens=valid_lens, + ready=ready, + ) + + def _shared_graph_table(self, layer_id: int, tensors: tuple) -> tuple: + bufs = self._graph_table_bufs.setdefault(layer_id, [None] * len(tensors)) + views = [] + for i, t in enumerate(tensors): + buf = bufs[i] + if buf is None or buf.dtype != t.dtype or buf.numel() < t.numel(): + # Graphs captured on a smaller buffer keep it alive themselves. + buf = bufs[i] = torch.empty(t.numel(), dtype=t.dtype, device=t.device) + views.append(buf[: t.numel()].view(t.shape).copy_(t)) + return tuple(views) + def publish_decode(self, inputs: DecodeInputs): data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool) metadata = inputs.paged_metadata @@ -352,7 +441,7 @@ def _build_prefill_table( page_table: torch.Tensor, page_size: int, request_ids: torch.Tensor, - rows_per_request: List[int], + rows_per_request: Optional[List[int]], q_dtype: torch.dtype, valid_lens: torch.Tensor, ) -> _SparsePrefillTable: @@ -446,6 +535,25 @@ def publish_prefill_table( ) +def _prefill_table_logits( + table: _SparseTable, + q_fp4: torch.Tensor, + q_sf: torch.Tensor, + weights: torch.Tensor, + k_cache: torch.Tensor, +) -> torch.Tensor: + """Sparse logits of prefill rows (one query row each) over a table's blocks.""" + rows, heads = q_sf.shape[0], q_sf.shape[1] + return sparse_logits( + q_fp4.view(rows, 1, heads, 64), + q_sf.view(rows, 1, heads), + k_cache, + weights.to(torch.bfloat16), + table.schedule, + table.blocks.shape[1], + ) + + def select_prefill_table( *, table: _SparsePrefillTable, @@ -453,14 +561,6 @@ def select_prefill_table( k_cache: torch.Tensor, out_positions: torch.Tensor, ) -> None: - rows, heads = data.q_sf.shape - logits = sparse_logits( - data.q_fp4.view(rows, 1, heads, 64), - data.q_sf.view(rows, 1, heads), - k_cache, - data.weights.to(torch.bfloat16), - table.schedule, - table.blocks.shape[1], - ) + logits = _prefill_table_logits(table, data.q_fp4, data.q_sf, data.weights, k_cache) # request-relative compressed positions, -1 padded topk_transform_sparse(logits, table.valid_lens, table.blocks, out_positions) diff --git a/python/sglang/srt/layers/attention/hybrid_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_attn_backend.py index 4ebb4846d1..e3900a0603 100644 --- a/python/sglang/srt/layers/attention/hybrid_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_attn_backend.py @@ -107,6 +107,10 @@ class HybridAttnBackend(AttentionBackend): def supports_prefill_cuda_graph_max_context_size(self) -> bool: return self.prefill_backend.supports_prefill_cuda_graph_max_context_size + @property + def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]: + return self.prefill_backend.prefill_cuda_graph_default_max_seq_len + def prepare_full_cuda_graph_chunked_prefix( self, forward_batch: ForwardBatch, diff --git a/python/sglang/srt/layers/attention/tbo_backend.py b/python/sglang/srt/layers/attention/tbo_backend.py index 5c6ac3dac4..76252d6ed6 100644 --- a/python/sglang/srt/layers/attention/tbo_backend.py +++ b/python/sglang/srt/layers/attention/tbo_backend.py @@ -32,6 +32,10 @@ class TboAttnBackend(AttentionBackend): def supports_prefill_cuda_graph_max_context_size(self) -> bool: return self.primary.supports_prefill_cuda_graph_max_context_size + @property + def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]: + return self.primary.prefill_cuda_graph_default_max_seq_len + @property def dllm_attention(self): return self.primary.dllm_attention diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 89c5f5816c..3b2c75ab52 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -375,6 +375,16 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self.max_bs = model_runner.req_to_token_pool.size self.max_context_size = prefill_config.max_context_size self.max_seq_len = prefill_config.max_seq_len + if self.max_seq_len is None: + self.max_seq_len = ( + model_runner.attn_backend.prefill_cuda_graph_default_max_seq_len + ) + if self.max_seq_len is not None: + logger.info( + "Prefill CUDA graph replays contexts up to %d tokens, the " + "attention backend's default; longer contexts run eager prefill.", + self.max_seq_len, + ) self._validate_max_context_capacity( max_context_size=self.max_context_size, table_width=model_runner.req_to_token_pool.req_to_token.shape[1], diff --git a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py index f33b96018a..d3f16a4f59 100644 --- a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py +++ b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py @@ -47,6 +47,7 @@ __all__ = [ "BreakableCUDAGraph", "BreakableCUDAGraphCapture", "break_graph", + "is_capturing", ] @@ -67,6 +68,13 @@ _current_capture_var: ContextVar["BreakableCUDAGraphCapture | None"] = ContextVa _current_stream_var: ContextVar[torch.Stream | None] = ContextVar( "current_stream", default=None ) + + +def is_capturing() -> bool: + """Whether a breakable CUDA graph is being captured; False during replay.""" + return _current_capture_var.get() is not None + + _forked_streams_var: ContextVar[set[torch.Stream] | None] = ContextVar( "forked_streams", default=None ) diff --git a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py index cbd8638372..7d31fe3d6a 100644 --- a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py +++ b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py @@ -684,6 +684,9 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): DeepseekV4AttnBackend, DSV4Metadata, ) + from sglang.srt.layers.attention.dsv4.v41_indexer.dense_blocks import ( + DenseBlocksBackend, + ) capture_metadata = DSV4Metadata( self._make_core_metadata(0), indexer_metadata=None @@ -695,6 +698,7 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): ) backend = object.__new__(DeepseekV4AttnBackend) backend.MAX_SEQ_LEN_FOR_CAPTURE = 4096 + backend.prefill_candidates = object.__new__(DenseBlocksBackend) backend.token_to_kv_pool = SimpleNamespace(request_window=None) backend.low_ratios = () calls = [] diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py index 2d81d6bdcd..5ebe1273bb 100644 --- a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -14,8 +14,10 @@ from sglang.srt.layers.attention import deepseek_v4_backend from sglang.srt.layers.attention.deepseek_v4_backend import ( SWA_WINDOW, DeepseekV4AttnBackend, + DSV4Metadata, late_layer_whole_extend, ) +from sglang.srt.layers.attention.dsv4.v41_indexer.sparse_table import SparseTableBackend from sglang.srt.model_executor import forward_batch_info from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig from sglang.srt.model_executor.forward_batch_info import ( @@ -271,5 +273,64 @@ class TestLateLayerRowsUnderDpAttention(CustomTestCase): self.assertEqual(counts, [SWA_WINDOW, 0]) +def _prefill_graph_replay(*, any_context, seq_len): + """The backend's default replayed-context limit and the widths a replay gives + the full-row and tail metadata, for one request of ``seq_len`` tokens.""" + seen = {} + + def build_full(forward_batch, *, max_seq_len_override, use_prefill_cuda_graph): + seen["full"] = max_seq_len_override + + def build_tail(forward_batch, max_seq_len): + seen["tail"] = max_seq_len + + backend = object.__new__(DeepseekV4AttnBackend) + backend.MAX_SEQ_LEN_FOR_CAPTURE = 4096 + backend.low_ratios = (1, 2) + backend.prefill_candidates = object.__new__(SparseTableBackend) + backend._build_forward_metadata = build_full + backend._prefill_graph_tail_metadata = build_tail + backend._activate_request_window = lambda: None + backend._prefill_graph_replays_any_context = lambda: any_context + capture = object.__new__(DSV4Metadata) + capture.prefill_graph_tail = True + with ( + patch.object( + deepseek_v4_backend, + "get_parallel", + return_value=SimpleNamespace(attn_cp_size=1), + ), + patch.object( + deepseek_v4_backend, "_prefill_graph_max_seq_len", return_value=None + ), + patch.object(DSV4Metadata, "refresh_for_breakable_cuda_graph_replay_"), + ): + seen["limit"] = backend.prefill_cuda_graph_default_max_seq_len + DeepseekV4AttnBackend.prepare_forward_metadata_for_breakable_cuda_graph_replay( + backend, + capture, + SimpleNamespace( + seq_lens_cpu=torch.tensor([seq_len]), max_seq_len_override=1 << 20 + ), + ) + return seen + + +class TestPrefillGraphReplayContext(CustomTestCase): + """DeepSeek-V4.1's prefill graph replays past its captured indexer width only + when the replay is exact, and then its metadata spans the live context.""" + + def test_limit_and_metadata_width(self): + for any_context, seq_len, expected in ( + (True, 155648 + 1024, {"limit": None, "full": 156672, "tail": 156672}), + (False, 8192 + 1024, {"limit": 16384, "full": 16384, "tail": 16384}), + ): + with self.subTest(any_context=any_context): + self.assertEqual( + _prefill_graph_replay(any_context=any_context, seq_len=seq_len), + expected, + ) + + if __name__ == "__main__": unittest.main()