[Perf] dsv4: Replay the V4.1 prefill graph past 16K contexts on SM90 (#45941)
This commit is contained in:
@@ -1022,11 +1022,6 @@ def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
|
||||
return torch.nn.functional.pad(t, (0, num_rows - t.shape[0]))
|
||||
|
||||
|
||||
# The replayed context limit when the prefill graph's metadata is built at a fixed
|
||||
# width (no two-level candidates); the candidate window, 2,048 blocks of 8 tokens.
|
||||
_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024
|
||||
|
||||
|
||||
def _prefill_graph_max_seq_len() -> Optional[int]:
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
@@ -3167,8 +3162,9 @@ class DeepseekV4AttnBackend(
|
||||
return self.forward_metadata
|
||||
|
||||
def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int:
|
||||
"""One context width for the full rows' and the late-layer tail's metadata."""
|
||||
if self._prefill_graph_replays_any_context():
|
||||
"""One context width for the full rows' and the late-layer tail's metadata:
|
||||
the live context, unless c4/c128 compress metadata is captured at a width."""
|
||||
if not self.has_c4 and not self.has_c128:
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
if seq_lens_cpu is not None and len(seq_lens_cpu) > 0:
|
||||
return int(seq_lens_cpu.max().item())
|
||||
@@ -3975,23 +3971,6 @@ class DeepseekV4AttnBackend(
|
||||
return
|
||||
self.forward_metadata.candidate_metadata = published
|
||||
|
||||
def _prefill_graph_replays_any_context(self) -> bool:
|
||||
"""Whether a prefill graph replay admits any context length: with two-level
|
||||
candidates only c4/c128 compress metadata would be captured at a width."""
|
||||
return self.low_ratio_prefill_graph and not self.has_c4 and not self.has_c128
|
||||
|
||||
@property
|
||||
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
|
||||
"""DeepSeek-V4.1's fixed metadata width without two-level candidates; prefill
|
||||
context parallelism builds its own metadata and needs none."""
|
||||
if (
|
||||
self.low_ratios != (1, 2)
|
||||
or get_parallel().attn_cp_size > 1
|
||||
or self._prefill_graph_replays_any_context()
|
||||
):
|
||||
return None
|
||||
return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN
|
||||
|
||||
def _prefill_graph_seq_limit(self) -> Optional[int]:
|
||||
"""The replayed context limit: cuda_graph_config[prefill].max_seq_len when
|
||||
the user sets it, else this backend's default."""
|
||||
|
||||
@@ -157,6 +157,9 @@ _OPUS_PREFILL_MIN_TOKENS = 1024
|
||||
_OPUS_PREFILL_MAX_ROWS_PER_TOKEN = {1: 256, 2: 128}
|
||||
# One ratio's bf16 workspace stays within 1 GiB.
|
||||
_OPUS_PREFILL_MAX_WORKSPACE_ROWS = 1 << 20
|
||||
# DeepSeek-V4.1's prefill graph metadata width when the user sets no
|
||||
# cuda_graph_config[prefill].max_seq_len; longer contexts run eager.
|
||||
_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -893,12 +896,6 @@ class DeepseekV4HipRadixBackend(
|
||||
# This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py.
|
||||
# PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager.
|
||||
can_run_prefill_cuda_graph = DeepseekV4AttnBackend.can_run_prefill_cuda_graph
|
||||
prefill_cuda_graph_default_max_seq_len = (
|
||||
DeepseekV4AttnBackend.prefill_cuda_graph_default_max_seq_len
|
||||
)
|
||||
_prefill_graph_replays_any_context = (
|
||||
DeepseekV4AttnBackend._prefill_graph_replays_any_context
|
||||
)
|
||||
_prefill_graph_seq_limit = DeepseekV4AttnBackend._prefill_graph_seq_limit
|
||||
_low_ratio_in_prefill_graph = DeepseekV4AttnBackend._low_ratio_in_prefill_graph
|
||||
_low_ratio_compress = DeepseekV4AttnBackend._low_ratio_compress
|
||||
@@ -2365,6 +2362,12 @@ class DeepseekV4HipRadixBackend(
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
)
|
||||
|
||||
@property
|
||||
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
|
||||
if self.low_ratios != (1, 2) or get_parallel().attn_cp_size > 1:
|
||||
return None
|
||||
return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN
|
||||
|
||||
def _graph_metadata_max_seq_len(self, forward_batch: ForwardBatch) -> Optional[int]:
|
||||
"""
|
||||
Capture and replay build the metadata at this fixed context length, so a
|
||||
|
||||
Reference in New Issue
Block a user