[Perf] dsv4: Replay the V4.1 prefill graph past 16K contexts on SM90 (#45941)

This commit is contained in:
Liangsheng Yin
2026-10-22 23:47:17 -07:00
committed by GitHub
parent 6dd0dfabf0
commit 6c844eb12e
4 changed files with 25 additions and 55 deletions
@@ -1022,11 +1022,6 @@ def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
return torch.nn.functional.pad(t, (0, num_rows - t.shape[0]))
# The replayed context limit when the prefill graph's metadata is built at a fixed
# width (no two-level candidates); the candidate window, 2,048 blocks of 8 tokens.
_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024
def _prefill_graph_max_seq_len() -> Optional[int]:
from sglang.srt.runtime_context import get_exec
@@ -3167,8 +3162,9 @@ class DeepseekV4AttnBackend(
return self.forward_metadata
def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int:
"""One context width for the full rows' and the late-layer tail's metadata."""
if self._prefill_graph_replays_any_context():
"""One context width for the full rows' and the late-layer tail's metadata:
the live context, unless c4/c128 compress metadata is captured at a width."""
if not self.has_c4 and not self.has_c128:
seq_lens_cpu = forward_batch.seq_lens_cpu
if seq_lens_cpu is not None and len(seq_lens_cpu) > 0:
return int(seq_lens_cpu.max().item())
@@ -3975,23 +3971,6 @@ class DeepseekV4AttnBackend(
return
self.forward_metadata.candidate_metadata = published
def _prefill_graph_replays_any_context(self) -> bool:
"""Whether a prefill graph replay admits any context length: with two-level
candidates only c4/c128 compress metadata would be captured at a width."""
return self.low_ratio_prefill_graph and not self.has_c4 and not self.has_c128
@property
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
"""DeepSeek-V4.1's fixed metadata width without two-level candidates; prefill
context parallelism builds its own metadata and needs none."""
if (
self.low_ratios != (1, 2)
or get_parallel().attn_cp_size > 1
or self._prefill_graph_replays_any_context()
):
return None
return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN
def _prefill_graph_seq_limit(self) -> Optional[int]:
"""The replayed context limit: cuda_graph_config[prefill].max_seq_len when
the user sets it, else this backend's default."""
@@ -157,6 +157,9 @@ _OPUS_PREFILL_MIN_TOKENS = 1024
_OPUS_PREFILL_MAX_ROWS_PER_TOKEN = {1: 256, 2: 128}
# One ratio's bf16 workspace stays within 1 GiB.
_OPUS_PREFILL_MAX_WORKSPACE_ROWS = 1 << 20
# DeepSeek-V4.1's prefill graph metadata width when the user sets no
# cuda_graph_config[prefill].max_seq_len; longer contexts run eager.
_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024
@dataclass
@@ -893,12 +896,6 @@ class DeepseekV4HipRadixBackend(
# This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py.
# PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager.
can_run_prefill_cuda_graph = DeepseekV4AttnBackend.can_run_prefill_cuda_graph
prefill_cuda_graph_default_max_seq_len = (
DeepseekV4AttnBackend.prefill_cuda_graph_default_max_seq_len
)
_prefill_graph_replays_any_context = (
DeepseekV4AttnBackend._prefill_graph_replays_any_context
)
_prefill_graph_seq_limit = DeepseekV4AttnBackend._prefill_graph_seq_limit
_low_ratio_in_prefill_graph = DeepseekV4AttnBackend._low_ratio_in_prefill_graph
_low_ratio_compress = DeepseekV4AttnBackend._low_ratio_compress
@@ -2365,6 +2362,12 @@ class DeepseekV4HipRadixBackend(
seq_lens_cpu=seq_lens_cpu,
)
@property
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
if self.low_ratios != (1, 2) or get_parallel().attn_cp_size > 1:
return None
return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN
def _graph_metadata_max_seq_len(self, forward_batch: ForwardBatch) -> Optional[int]:
"""
Capture and replay build the metadata at this fixed context length, so a