[DSv4.1] Support TRT-LLM decoder bounded replay with BCG (#44326)
This commit is contained in:
@@ -215,10 +215,12 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
return
|
||||
if (
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
and cfg.cuda_graph_config.prefill.backend
|
||||
not in (Backend.DISABLED, Backend.BREAKABLE)
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill "
|
||||
"disabled or breakable"
|
||||
)
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
incompatible = (
|
||||
@@ -266,12 +268,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
("HiSparse", cfg.enable_hisparse),
|
||||
("the unified KV layout", is_unified_kv_triton()),
|
||||
(
|
||||
"TRT-LLM with SWA bounded replay",
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and (
|
||||
cfg.enable_encoder_swa_bounded_replay
|
||||
or cfg.enable_decoder_swa_bounded_replay
|
||||
),
|
||||
"TRT-LLM with encoder SWA bounded replay",
|
||||
cfg.dsv4_attn_backend == "trtllm" and cfg.enable_encoder_swa_bounded_replay,
|
||||
),
|
||||
("two-batch overlap", cfg.enable_two_batch_overlap),
|
||||
("pipeline parallelism", cfg.pp_size > 1),
|
||||
|
||||
@@ -382,11 +382,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
extra_indices: Optional[torch.Tensor],
|
||||
extra_topk_lengths: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
"""Drive the decode kernel as varlen prefill with one table row per token.
|
||||
|
||||
``seq_lens`` includes cached prefixes; the kernel derives causal SWA
|
||||
validity from it. This path runs eagerly.
|
||||
"""
|
||||
"""Run varlen prefill eagerly, including at BCG breaks."""
|
||||
|
||||
assert q.ndim == 3, f"{q.shape=}"
|
||||
num_qo_padded, num_heads, head_dim = q.shape
|
||||
@@ -395,7 +391,12 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
# Build VarSeq metadata from the same extend lengths as the sparse tables.
|
||||
core = self.forward_metadata.core_attn_metadata
|
||||
if core.trtllm_prefill_qmeta is None:
|
||||
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
tail = self.forward_metadata.late_layer_tail
|
||||
extend_seq_lens_cpu = (
|
||||
tail.extend_seq_lens_cpu
|
||||
if tail is not None
|
||||
else forward_batch.extend_seq_lens_cpu
|
||||
)
|
||||
assert extend_seq_lens_cpu is not None and len(extend_seq_lens_cpu) > 0
|
||||
batch_size = len(extend_seq_lens_cpu)
|
||||
cum_lens = [0] * (batch_size + 1)
|
||||
@@ -404,7 +405,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
sum_q = cum_lens[-1]
|
||||
max_q_len = max(int(x) for x in extend_seq_lens_cpu)
|
||||
assert 0 < sum_q <= num_qo_padded, f"{sum_q=} {num_qo_padded=}"
|
||||
seq_lens_i32 = forward_batch.seq_lens.to(torch.int32)
|
||||
# SWA masking uses causal lengths, not -1 sentinels: make them
|
||||
# tail-relative while keeping compressed KV indices full-context.
|
||||
seq_lens_i32 = (
|
||||
tail.extend_seq_lens if tail is not None else forward_batch.seq_lens
|
||||
).to(torch.int32)
|
||||
assert seq_lens_i32.shape == (batch_size,), f"{seq_lens_i32.shape=}"
|
||||
core.trtllm_prefill_qmeta = (
|
||||
self._move_to_device(cum_lens),
|
||||
|
||||
@@ -436,7 +436,10 @@ def _local_prefill_cuda_graph_vote(
|
||||
is_mixed=mode == ForwardMode.MIXED,
|
||||
batch_max_context_len=(
|
||||
int(local_batch.seq_lens_cpu.max().item())
|
||||
if prefill_graph_runner.max_context_size is not None
|
||||
if (
|
||||
prefill_graph_runner.max_context_size is not None
|
||||
or prefill_graph_runner.max_seq_len is not None
|
||||
)
|
||||
and local_batch.seq_lens_cpu is not None
|
||||
and local_batch.seq_lens_cpu.numel() > 0
|
||||
else None
|
||||
|
||||
@@ -357,6 +357,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.max_num_tokens = max(self.capture_num_tokens)
|
||||
self.max_bs = model_runner.req_to_token_pool.size
|
||||
self.max_context_size = prefill_config.max_context_size
|
||||
self.max_seq_len = prefill_config.max_seq_len
|
||||
self._validate_max_context_capacity(
|
||||
max_context_size=self.max_context_size,
|
||||
table_width=model_runner.req_to_token_pool.req_to_token.shape[1],
|
||||
@@ -1345,10 +1346,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return False
|
||||
if return_logprob and not self._uses_eager_prefill_tail():
|
||||
return False
|
||||
if self.max_context_size is not None:
|
||||
if (
|
||||
batch_max_context_len is None
|
||||
or batch_max_context_len > self.max_context_size
|
||||
# Vote on both bounds so DP ranks agree on graph versus eager.
|
||||
for context_limit in (self.max_context_size, self.max_seq_len):
|
||||
if context_limit is not None and (
|
||||
batch_max_context_len is None or batch_max_context_len > context_limit
|
||||
):
|
||||
return False
|
||||
if num_tokens is None:
|
||||
@@ -1386,7 +1387,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# Non-DP local check (sole decision for tp-only).
|
||||
batch_max_context_len = (
|
||||
self._batch_max_context_len(forward_batch)
|
||||
if self.max_context_size is not None
|
||||
if self.max_context_size is not None or self.max_seq_len is not None
|
||||
else None
|
||||
)
|
||||
if not self.can_replay_locally(
|
||||
|
||||
@@ -2466,6 +2466,8 @@ class MQALayer(MqaAttentionBase):
|
||||
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
|
||||
o = attn_q.new_empty(
|
||||
(*attn_q.shape[:-1], self.attn_mqa.v_head_dim),
|
||||
# TRT-LLM returns BF16 even with FP8 queries.
|
||||
dtype=x.dtype,
|
||||
)
|
||||
bcg_deepseek_v4_attention_with_output(
|
||||
attn_q,
|
||||
|
||||
Reference in New Issue
Block a user