[DSv4.1] Support TRT-LLM decoder bounded replay with BCG (#44326)

This commit is contained in:
Yuwei An
2026-10-14 17:39:52 -07:00
committed by GitHub
parent c891c5c7ca
commit 64a0a08f0a
11 changed files with 118 additions and 23 deletions
@@ -215,10 +215,12 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
return
if (
cfg.dsv4_attn_backend == "trtllm"
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
and cfg.cuda_graph_config.prefill.backend
not in (Backend.DISABLED, Backend.BREAKABLE)
):
raise ValueError(
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill "
"disabled or breakable"
)
if cfg.enable_encoder_swa_bounded_replay:
incompatible = (
@@ -266,12 +268,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
("HiSparse", cfg.enable_hisparse),
("the unified KV layout", is_unified_kv_triton()),
(
"TRT-LLM with SWA bounded replay",
cfg.dsv4_attn_backend == "trtllm"
and (
cfg.enable_encoder_swa_bounded_replay
or cfg.enable_decoder_swa_bounded_replay
),
"TRT-LLM with encoder SWA bounded replay",
cfg.dsv4_attn_backend == "trtllm" and cfg.enable_encoder_swa_bounded_replay,
),
("two-batch overlap", cfg.enable_two_batch_overlap),
("pipeline parallelism", cfg.pp_size > 1),
@@ -382,11 +382,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
extra_indices: Optional[torch.Tensor],
extra_topk_lengths: Optional[torch.Tensor],
) -> torch.Tensor:
"""Drive the decode kernel as varlen prefill with one table row per token.
``seq_lens`` includes cached prefixes; the kernel derives causal SWA
validity from it. This path runs eagerly.
"""
"""Run varlen prefill eagerly, including at BCG breaks."""
assert q.ndim == 3, f"{q.shape=}"
num_qo_padded, num_heads, head_dim = q.shape
@@ -395,7 +391,12 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
# Build VarSeq metadata from the same extend lengths as the sparse tables.
core = self.forward_metadata.core_attn_metadata
if core.trtllm_prefill_qmeta is None:
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
tail = self.forward_metadata.late_layer_tail
extend_seq_lens_cpu = (
tail.extend_seq_lens_cpu
if tail is not None
else forward_batch.extend_seq_lens_cpu
)
assert extend_seq_lens_cpu is not None and len(extend_seq_lens_cpu) > 0
batch_size = len(extend_seq_lens_cpu)
cum_lens = [0] * (batch_size + 1)
@@ -404,7 +405,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
sum_q = cum_lens[-1]
max_q_len = max(int(x) for x in extend_seq_lens_cpu)
assert 0 < sum_q <= num_qo_padded, f"{sum_q=} {num_qo_padded=}"
seq_lens_i32 = forward_batch.seq_lens.to(torch.int32)
# SWA masking uses causal lengths, not -1 sentinels: make them
# tail-relative while keeping compressed KV indices full-context.
seq_lens_i32 = (
tail.extend_seq_lens if tail is not None else forward_batch.seq_lens
).to(torch.int32)
assert seq_lens_i32.shape == (batch_size,), f"{seq_lens_i32.shape=}"
core.trtllm_prefill_qmeta = (
self._move_to_device(cum_lens),
@@ -436,7 +436,10 @@ def _local_prefill_cuda_graph_vote(
is_mixed=mode == ForwardMode.MIXED,
batch_max_context_len=(
int(local_batch.seq_lens_cpu.max().item())
if prefill_graph_runner.max_context_size is not None
if (
prefill_graph_runner.max_context_size is not None
or prefill_graph_runner.max_seq_len is not None
)
and local_batch.seq_lens_cpu is not None
and local_batch.seq_lens_cpu.numel() > 0
else None
@@ -357,6 +357,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self.max_num_tokens = max(self.capture_num_tokens)
self.max_bs = model_runner.req_to_token_pool.size
self.max_context_size = prefill_config.max_context_size
self.max_seq_len = prefill_config.max_seq_len
self._validate_max_context_capacity(
max_context_size=self.max_context_size,
table_width=model_runner.req_to_token_pool.req_to_token.shape[1],
@@ -1345,10 +1346,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return False
if return_logprob and not self._uses_eager_prefill_tail():
return False
if self.max_context_size is not None:
if (
batch_max_context_len is None
or batch_max_context_len > self.max_context_size
# Vote on both bounds so DP ranks agree on graph versus eager.
for context_limit in (self.max_context_size, self.max_seq_len):
if context_limit is not None and (
batch_max_context_len is None or batch_max_context_len > context_limit
):
return False
if num_tokens is None:
@@ -1386,7 +1387,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# Non-DP local check (sole decision for tp-only).
batch_max_context_len = (
self._batch_max_context_len(forward_batch)
if self.max_context_size is not None
if self.max_context_size is not None or self.max_seq_len is not None
else None
)
if not self.can_replay_locally(
+2
View File
@@ -2466,6 +2466,8 @@ class MQALayer(MqaAttentionBase):
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
o = attn_q.new_empty(
(*attn_q.shape[:-1], self.attn_mqa.v_head_dim),
# TRT-LLM returns BF16 even with FP8 queries.
dtype=x.dtype,
)
bcg_deepseek_v4_attention_with_output(
attn_q,