[DSv4.1] Support TRT-LLM decoder bounded replay with BCG (#44326)

This commit is contained in:
Yuwei An
2026-10-14 17:39:52 -07:00
committed by GitHub
parent c891c5c7ca
commit 64a0a08f0a
11 changed files with 118 additions and 23 deletions
@@ -215,10 +215,12 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
return
if (
cfg.dsv4_attn_backend == "trtllm"
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
and cfg.cuda_graph_config.prefill.backend
not in (Backend.DISABLED, Backend.BREAKABLE)
):
raise ValueError(
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill "
"disabled or breakable"
)
if cfg.enable_encoder_swa_bounded_replay:
incompatible = (
@@ -266,12 +268,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
("HiSparse", cfg.enable_hisparse),
("the unified KV layout", is_unified_kv_triton()),
(
"TRT-LLM with SWA bounded replay",
cfg.dsv4_attn_backend == "trtllm"
and (
cfg.enable_encoder_swa_bounded_replay
or cfg.enable_decoder_swa_bounded_replay
),
"TRT-LLM with encoder SWA bounded replay",
cfg.dsv4_attn_backend == "trtllm" and cfg.enable_encoder_swa_bounded_replay,
),
("two-batch overlap", cfg.enable_two_batch_overlap),
("pipeline parallelism", cfg.pp_size > 1),
@@ -382,11 +382,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
extra_indices: Optional[torch.Tensor],
extra_topk_lengths: Optional[torch.Tensor],
) -> torch.Tensor:
"""Drive the decode kernel as varlen prefill with one table row per token.
``seq_lens`` includes cached prefixes; the kernel derives causal SWA
validity from it. This path runs eagerly.
"""
"""Run varlen prefill eagerly, including at BCG breaks."""
assert q.ndim == 3, f"{q.shape=}"
num_qo_padded, num_heads, head_dim = q.shape
@@ -395,7 +391,12 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
# Build VarSeq metadata from the same extend lengths as the sparse tables.
core = self.forward_metadata.core_attn_metadata
if core.trtllm_prefill_qmeta is None:
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
tail = self.forward_metadata.late_layer_tail
extend_seq_lens_cpu = (
tail.extend_seq_lens_cpu
if tail is not None
else forward_batch.extend_seq_lens_cpu
)
assert extend_seq_lens_cpu is not None and len(extend_seq_lens_cpu) > 0
batch_size = len(extend_seq_lens_cpu)
cum_lens = [0] * (batch_size + 1)
@@ -404,7 +405,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
sum_q = cum_lens[-1]
max_q_len = max(int(x) for x in extend_seq_lens_cpu)
assert 0 < sum_q <= num_qo_padded, f"{sum_q=} {num_qo_padded=}"
seq_lens_i32 = forward_batch.seq_lens.to(torch.int32)
# SWA masking uses causal lengths, not -1 sentinels: make them
# tail-relative while keeping compressed KV indices full-context.
seq_lens_i32 = (
tail.extend_seq_lens if tail is not None else forward_batch.seq_lens
).to(torch.int32)
assert seq_lens_i32.shape == (batch_size,), f"{seq_lens_i32.shape=}"
core.trtllm_prefill_qmeta = (
self._move_to_device(cum_lens),
@@ -436,7 +436,10 @@ def _local_prefill_cuda_graph_vote(
is_mixed=mode == ForwardMode.MIXED,
batch_max_context_len=(
int(local_batch.seq_lens_cpu.max().item())
if prefill_graph_runner.max_context_size is not None
if (
prefill_graph_runner.max_context_size is not None
or prefill_graph_runner.max_seq_len is not None
)
and local_batch.seq_lens_cpu is not None
and local_batch.seq_lens_cpu.numel() > 0
else None
@@ -357,6 +357,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self.max_num_tokens = max(self.capture_num_tokens)
self.max_bs = model_runner.req_to_token_pool.size
self.max_context_size = prefill_config.max_context_size
self.max_seq_len = prefill_config.max_seq_len
self._validate_max_context_capacity(
max_context_size=self.max_context_size,
table_width=model_runner.req_to_token_pool.req_to_token.shape[1],
@@ -1345,10 +1346,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return False
if return_logprob and not self._uses_eager_prefill_tail():
return False
if self.max_context_size is not None:
if (
batch_max_context_len is None
or batch_max_context_len > self.max_context_size
# Vote on both bounds so DP ranks agree on graph versus eager.
for context_limit in (self.max_context_size, self.max_seq_len):
if context_limit is not None and (
batch_max_context_len is None or batch_max_context_len > context_limit
):
return False
if num_tokens is None:
@@ -1386,7 +1387,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# Non-DP local check (sole decision for tp-only).
batch_max_context_len = (
self._batch_max_context_len(forward_batch)
if self.max_context_size is not None
if self.max_context_size is not None or self.max_seq_len is not None
else None
)
if not self.can_replay_locally(
+2
View File
@@ -2466,6 +2466,8 @@ class MQALayer(MqaAttentionBase):
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
o = attn_q.new_empty(
(*attn_q.shape[:-1], self.attn_mqa.v_head_dim),
# TRT-LLM returns BF16 even with FP8 queries.
dtype=x.dtype,
)
bcg_deepseek_v4_attention_with_output(
attn_q,
@@ -1,6 +1,7 @@
"""DeepSeek-V4.1-Flash accuracy with DSPARK on four GB300 GPUs.
Run GSM8K and MMLU in TP4+EP4 and DP-attention4 + EP4 configurations.
Run GSM8K and MMLU in TP4+EP4 and DP-attention4 + EP4 configurations,
plus TRT-LLM attention with decoder bounded replay and breakable prefill.
The DP+EP recipe is intentionally exercised rather than skipped: startup or
accuracy failures must be visible in CI while this path is being validated.
"""
@@ -19,7 +20,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=1100, stage="base-c", runner_config="4-gpu-gb300")
register_cuda_ci(est_time=1650, stage="base-c", runner_config="4-gpu-gb300")
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
@@ -122,5 +123,27 @@ class TestDSV41FlashDP4EP4DSpark(DSV41FlashAccuracyMixin, CustomTestCase):
]
class TestDSV41FlashTrtllmBoundedBCG(DSV41FlashAccuracyMixin, CustomTestCase):
"""TRT-LLM varlen attention through the full and bounded-tail graph breaks."""
parallel_args = [
"--attn-dp-size",
"1",
"--ep-size",
"4",
"--dsv4-attn-backend",
"trtllm",
"--enable-decoder-swa-bounded-replay",
"--cuda-graph-backend-prefill",
"breakable",
"--cuda-graph-max-bs-prefill",
"4096",
"--cuda-graph-max-seq-len-prefill",
"16384",
"--chunked-prefill-size",
"4096",
]
if __name__ == "__main__":
unittest.main()
@@ -44,6 +44,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
runner.capture_hidden_mode = CaptureHiddenMode.NULL
runner.capture_num_tokens = [4, 16]
runner.max_context_size = None
runner.max_seq_len = None
runner.max_num_tokens = 16
return runner
@@ -405,6 +405,7 @@ class TestDecodeToExtendConversionVote(CustomTestCase):
runner = Mock(spec=dp_attn.PrefillCudaGraphRunner)
runner.enable_lora = False
runner.max_context_size = None
runner.max_seq_len = None
runner.can_replay_locally.return_value = True
batch = SimpleNamespace(
forward_mode=ForwardMode.DECODE,
@@ -458,6 +459,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
runner = Mock(spec=runner_cls)
runner.enable_lora = False
runner.max_context_size = None
runner.max_seq_len = None
runner._qwen_bcg_hc_sidechannel = True
runner.can_replay_locally.side_effect = lambda **kwargs: (
not kwargs["contains_mm_inputs"]
@@ -505,6 +507,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
runner = Mock(spec=runner_cls)
runner.enable_lora = False
runner.max_context_size = None
runner.max_seq_len = None
runner._qwen_bcg_hc_sidechannel = False
runner._is_full_backend = False
runner._captured_attn_metadata_max_bs = 2
@@ -74,6 +74,7 @@ class TestHiddenStateGraphRecapture(CustomTestCase):
runner.capture_hidden_mode = capture_hidden_mode
runner.capture_num_tokens = [4]
runner.max_context_size = None
runner.max_seq_len = None
runner.max_num_tokens = 4
return runner
@@ -6,6 +6,9 @@ import torch
import sglang.srt.model_executor.runner.prefill_cuda_graph_runner as runner_module
from sglang.srt.layers.moe.utils import MoeA2ABackend
from sglang.srt.managers.scheduler_components.dp_attn import (
_local_prefill_cuda_graph_vote,
)
from sglang.srt.model_executor import forward_batch_info
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_info import (
@@ -36,6 +39,7 @@ class TestPrefillCudaGraphPadding(CustomTestCase):
runner.capture_hidden_mode = CaptureHiddenMode.NULL
runner.capture_num_tokens = [4, 16]
runner.max_context_size = None
runner.max_seq_len = None
runner.max_num_tokens = 16
return runner
@@ -238,6 +242,56 @@ class TestPrefillCudaGraphPadding(CustomTestCase):
uncovered.seq_lens_cpu.fill_(701)
self.assertFalse(runner.can_run_graph(uncovered))
def test_sequence_bound_votes_eager_for_active_and_idle_dp_ranks(self):
runner = self._make_runner()
runner.prefill_backend_name = Backend.BREAKABLE
runner.max_seq_len = 16384
for context_len, expected in ((16384, True), (17000, False)):
with self.subTest(context_len=context_len):
active = self._make_forward_batch(4)
active.seq_lens_cpu.fill_(context_len)
schedule_batch = SimpleNamespace(
forward_mode=ForwardMode.EXTEND,
batch_size=lambda: 1,
extend_num_tokens=4,
input_embeds=None,
replace_embeds=None,
prefix_lens=[context_len - 4],
return_logprob=False,
multimodal_inputs=None,
seq_lens_cpu=active.seq_lens_cpu,
)
votes = [
_local_prefill_cuda_graph_vote(
local_batch=batch,
prefill_graph_runner=runner,
coordinated_prefill=True,
breakable_prefill=True,
spec_algorithm=None,
model_config=None,
)
for batch in (schedule_batch, None)
]
self.assertEqual(votes, [expected, True])
# The padded idle request fits; it must still honor the vote.
idle = self._make_forward_batch(4)
for batch in (active, idle):
batch.global_num_tokens_cpu = [4, 4]
batch.can_run_dp_prefill_cuda_graph = all(votes)
self.assertEqual(runner.can_run_graph(batch), expected)
def test_context_and_sequence_bounds_both_apply(self):
runner = self._make_runner()
for context_limit, sequence_limit in ((700, 900), (900, 700)):
with self.subTest(context=context_limit, sequence=sequence_limit):
runner.max_context_size = context_limit
runner.max_seq_len = sequence_limit
batch = self._make_forward_batch(4)
batch.seq_lens_cpu.fill_(700)
self.assertTrue(runner.can_run_graph(batch))
batch.seq_lens_cpu.fill_(701)
self.assertFalse(runner.can_run_graph(batch))
def test_unsupported_path_ignores_max_context_size(self):
runner = self._make_runner()
runner.max_context_size = 1024
@@ -215,6 +215,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
runner.device = torch.device("cpu")
runner.prefill_backend_name = prefill_backend_name
runner.max_context_size = None
runner.max_seq_len = None
runner._capture_req_slots = capture_req_slots
runner.max_bs = 4
runner._prefill_static_buffers = None
@@ -384,6 +385,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner)
runner.capture_num_tokens = [4]
runner.max_context_size = None
runner.max_seq_len = None
runner._capture_chunked_prefix = False
runner.buffer_registry = _FakeBatchRegistry()
translator = _static_pool_translator()
@@ -632,6 +634,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
runner.max_num_tokens = 4
runner.capture_num_tokens = [4]
runner.max_context_size = None
runner.max_seq_len = None
runner._capture_chunked_prefix = False
runner.has_mha_companion_layers = False
@@ -698,6 +701,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
runner.max_num_tokens = 32
runner.capture_num_tokens = [4]
runner.max_context_size = None
runner.max_seq_len = None
runner.backend = SimpleNamespace()
runner.prefill_backend_name = Backend.FULL
runner.has_mha_companion_layers = False