From 6c844eb12ee203c977e030390dc64ce3ee99062f Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 22 Oct 2026 23:47:17 -0700 Subject: [PATCH] [Perf] dsv4: Replay the V4.1 prefill graph past 16K contexts on SM90 (#45941) --- .../layers/attention/deepseek_v4_backend.py | 27 ++------------ .../deepseek_v4_backend_hip_radix.py | 15 +++++--- .../unittests/dsv4/test_deepseek_v4.py | 1 + .../test_dsv41_late_layer_tail_readers.py | 37 ++++++------------- 4 files changed, 25 insertions(+), 55 deletions(-) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 8895de7f51..6fbaa06e29 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1022,11 +1022,6 @@ def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor: return torch.nn.functional.pad(t, (0, num_rows - t.shape[0])) -# The replayed context limit when the prefill graph's metadata is built at a fixed -# width (no two-level candidates); the candidate window, 2,048 blocks of 8 tokens. -_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024 - - def _prefill_graph_max_seq_len() -> Optional[int]: from sglang.srt.runtime_context import get_exec @@ -3167,8 +3162,9 @@ class DeepseekV4AttnBackend( return self.forward_metadata def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int: - """One context width for the full rows' and the late-layer tail's metadata.""" - if self._prefill_graph_replays_any_context(): + """One context width for the full rows' and the late-layer tail's metadata: + the live context, unless c4/c128 compress metadata is captured at a width.""" + if not self.has_c4 and not self.has_c128: seq_lens_cpu = forward_batch.seq_lens_cpu if seq_lens_cpu is not None and len(seq_lens_cpu) > 0: return int(seq_lens_cpu.max().item()) @@ -3975,23 +3971,6 @@ class DeepseekV4AttnBackend( return self.forward_metadata.candidate_metadata = published - def _prefill_graph_replays_any_context(self) -> bool: - """Whether a prefill graph replay admits any context length: with two-level - candidates only c4/c128 compress metadata would be captured at a width.""" - return self.low_ratio_prefill_graph and not self.has_c4 and not self.has_c128 - - @property - def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]: - """DeepSeek-V4.1's fixed metadata width without two-level candidates; prefill - context parallelism builds its own metadata and needs none.""" - if ( - self.low_ratios != (1, 2) - or get_parallel().attn_cp_size > 1 - or self._prefill_graph_replays_any_context() - ): - return None - return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN - def _prefill_graph_seq_limit(self) -> Optional[int]: """The replayed context limit: cuda_graph_config[prefill].max_seq_len when the user sets it, else this backend's default.""" diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index b8d053e197..f663e80a2d 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -157,6 +157,9 @@ _OPUS_PREFILL_MIN_TOKENS = 1024 _OPUS_PREFILL_MAX_ROWS_PER_TOKEN = {1: 256, 2: 128} # One ratio's bf16 workspace stays within 1 GiB. _OPUS_PREFILL_MAX_WORKSPACE_ROWS = 1 << 20 +# DeepSeek-V4.1's prefill graph metadata width when the user sets no +# cuda_graph_config[prefill].max_seq_len; longer contexts run eager. +_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024 @dataclass @@ -893,12 +896,6 @@ class DeepseekV4HipRadixBackend( # This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py. # PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager. can_run_prefill_cuda_graph = DeepseekV4AttnBackend.can_run_prefill_cuda_graph - prefill_cuda_graph_default_max_seq_len = ( - DeepseekV4AttnBackend.prefill_cuda_graph_default_max_seq_len - ) - _prefill_graph_replays_any_context = ( - DeepseekV4AttnBackend._prefill_graph_replays_any_context - ) _prefill_graph_seq_limit = DeepseekV4AttnBackend._prefill_graph_seq_limit _low_ratio_in_prefill_graph = DeepseekV4AttnBackend._low_ratio_in_prefill_graph _low_ratio_compress = DeepseekV4AttnBackend._low_ratio_compress @@ -2365,6 +2362,12 @@ class DeepseekV4HipRadixBackend( seq_lens_cpu=seq_lens_cpu, ) + @property + def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]: + if self.low_ratios != (1, 2) or get_parallel().attn_cp_size > 1: + return None + return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN + def _graph_metadata_max_seq_len(self, forward_batch: ForwardBatch) -> Optional[int]: """ Capture and replay build the metadata at this fixed context length, so a diff --git a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py index 1dee10e2fc..7656299504 100644 --- a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py +++ b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py @@ -661,6 +661,7 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): backend.prefill_candidates = object.__new__(DenseBlocksBackend) backend.token_to_kv_pool = SimpleNamespace(request_window=None) backend.low_ratios = () + backend.has_c4 = backend.has_c128 = True calls = [] def fake_build_forward_metadata( diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py index ff7848126a..f89d0c1c69 100644 --- a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -17,7 +17,6 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( DSV4Metadata, late_layer_whole_extend, ) -from sglang.srt.layers.attention.dsv4.v41_indexer.sparse_table import SparseTableBackend from sglang.srt.model_executor import forward_batch_info from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig from sglang.srt.model_executor.forward_batch_info import ( @@ -273,7 +272,7 @@ class TestLateLayerRowsUnderDpAttention(CustomTestCase): self.assertEqual(counts, [SWA_WINDOW, 0]) -def _prefill_graph_replay(*, any_context, seq_len): +def _prefill_graph_replay(*, ratios, seq_len): """The backend's default replayed-context limit and the widths a replay gives the full-row and tail metadata, for one request of ``seq_len`` tokens.""" seen = {} @@ -286,25 +285,15 @@ def _prefill_graph_replay(*, any_context, seq_len): backend = object.__new__(DeepseekV4AttnBackend) backend.MAX_SEQ_LEN_FOR_CAPTURE = 4096 - backend.low_ratios = (1, 2) - backend.prefill_candidates = object.__new__(SparseTableBackend) + backend.low_ratios = tuple(ratio for ratio in ratios if ratio in (1, 2)) + backend.has_c4 = 4 in ratios + backend.has_c128 = 128 in ratios backend._build_forward_metadata = build_full backend._prefill_graph_tail_metadata = build_tail backend._activate_request_window = lambda: None - backend._prefill_graph_replays_any_context = lambda: any_context capture = object.__new__(DSV4Metadata) capture.prefill_graph_tail = True - with ( - patch.object( - deepseek_v4_backend, - "get_parallel", - return_value=SimpleNamespace(attn_cp_size=1), - ), - patch.object( - deepseek_v4_backend, "_prefill_graph_max_seq_len", return_value=None - ), - patch.object(DSV4Metadata, "refresh_for_breakable_cuda_graph_replay_"), - ): + with patch.object(DSV4Metadata, "refresh_for_breakable_cuda_graph_replay_"): seen["limit"] = backend.prefill_cuda_graph_default_max_seq_len DeepseekV4AttnBackend.prepare_forward_metadata_for_breakable_cuda_graph_replay( backend, @@ -317,18 +306,16 @@ def _prefill_graph_replay(*, any_context, seq_len): class TestPrefillGraphReplayContext(CustomTestCase): - """DeepSeek-V4.1's prefill graph replays past its captured indexer width only - when the replay is exact, and then its metadata spans the live context.""" + """DeepSeek-V4.1's prefill graph replays any context, its metadata spanning the + live context; with c4/c128 compress metadata the width stays fixed.""" def test_limit_and_metadata_width(self): - for any_context, seq_len, expected in ( - (True, 155648 + 1024, {"limit": None, "full": 156672, "tail": 156672}), - (False, 8192 + 1024, {"limit": 16384, "full": 16384, "tail": 16384}), - ): - with self.subTest(any_context=any_context): + seq_len = 155648 + 1024 + for ratios, width in (((1, 2), seq_len), ((4, 128), 1 << 20)): + with self.subTest(ratios=ratios): self.assertEqual( - _prefill_graph_replay(any_context=any_context, seq_len=seq_len), - expected, + _prefill_graph_replay(ratios=ratios, seq_len=seq_len), + {"limit": None, "full": width, "tail": width}, )