[DSv4.1] Support TRT-LLM decoder bounded replay with BCG (#44326)
This commit is contained in:
@@ -215,10 +215,12 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
return
|
||||
if (
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
and cfg.cuda_graph_config.prefill.backend
|
||||
not in (Backend.DISABLED, Backend.BREAKABLE)
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill "
|
||||
"disabled or breakable"
|
||||
)
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
incompatible = (
|
||||
@@ -266,12 +268,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
("HiSparse", cfg.enable_hisparse),
|
||||
("the unified KV layout", is_unified_kv_triton()),
|
||||
(
|
||||
"TRT-LLM with SWA bounded replay",
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and (
|
||||
cfg.enable_encoder_swa_bounded_replay
|
||||
or cfg.enable_decoder_swa_bounded_replay
|
||||
),
|
||||
"TRT-LLM with encoder SWA bounded replay",
|
||||
cfg.dsv4_attn_backend == "trtllm" and cfg.enable_encoder_swa_bounded_replay,
|
||||
),
|
||||
("two-batch overlap", cfg.enable_two_batch_overlap),
|
||||
("pipeline parallelism", cfg.pp_size > 1),
|
||||
|
||||
@@ -382,11 +382,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
extra_indices: Optional[torch.Tensor],
|
||||
extra_topk_lengths: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
"""Drive the decode kernel as varlen prefill with one table row per token.
|
||||
|
||||
``seq_lens`` includes cached prefixes; the kernel derives causal SWA
|
||||
validity from it. This path runs eagerly.
|
||||
"""
|
||||
"""Run varlen prefill eagerly, including at BCG breaks."""
|
||||
|
||||
assert q.ndim == 3, f"{q.shape=}"
|
||||
num_qo_padded, num_heads, head_dim = q.shape
|
||||
@@ -395,7 +391,12 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
# Build VarSeq metadata from the same extend lengths as the sparse tables.
|
||||
core = self.forward_metadata.core_attn_metadata
|
||||
if core.trtllm_prefill_qmeta is None:
|
||||
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
tail = self.forward_metadata.late_layer_tail
|
||||
extend_seq_lens_cpu = (
|
||||
tail.extend_seq_lens_cpu
|
||||
if tail is not None
|
||||
else forward_batch.extend_seq_lens_cpu
|
||||
)
|
||||
assert extend_seq_lens_cpu is not None and len(extend_seq_lens_cpu) > 0
|
||||
batch_size = len(extend_seq_lens_cpu)
|
||||
cum_lens = [0] * (batch_size + 1)
|
||||
@@ -404,7 +405,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
sum_q = cum_lens[-1]
|
||||
max_q_len = max(int(x) for x in extend_seq_lens_cpu)
|
||||
assert 0 < sum_q <= num_qo_padded, f"{sum_q=} {num_qo_padded=}"
|
||||
seq_lens_i32 = forward_batch.seq_lens.to(torch.int32)
|
||||
# SWA masking uses causal lengths, not -1 sentinels: make them
|
||||
# tail-relative while keeping compressed KV indices full-context.
|
||||
seq_lens_i32 = (
|
||||
tail.extend_seq_lens if tail is not None else forward_batch.seq_lens
|
||||
).to(torch.int32)
|
||||
assert seq_lens_i32.shape == (batch_size,), f"{seq_lens_i32.shape=}"
|
||||
core.trtllm_prefill_qmeta = (
|
||||
self._move_to_device(cum_lens),
|
||||
|
||||
@@ -436,7 +436,10 @@ def _local_prefill_cuda_graph_vote(
|
||||
is_mixed=mode == ForwardMode.MIXED,
|
||||
batch_max_context_len=(
|
||||
int(local_batch.seq_lens_cpu.max().item())
|
||||
if prefill_graph_runner.max_context_size is not None
|
||||
if (
|
||||
prefill_graph_runner.max_context_size is not None
|
||||
or prefill_graph_runner.max_seq_len is not None
|
||||
)
|
||||
and local_batch.seq_lens_cpu is not None
|
||||
and local_batch.seq_lens_cpu.numel() > 0
|
||||
else None
|
||||
|
||||
@@ -357,6 +357,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.max_num_tokens = max(self.capture_num_tokens)
|
||||
self.max_bs = model_runner.req_to_token_pool.size
|
||||
self.max_context_size = prefill_config.max_context_size
|
||||
self.max_seq_len = prefill_config.max_seq_len
|
||||
self._validate_max_context_capacity(
|
||||
max_context_size=self.max_context_size,
|
||||
table_width=model_runner.req_to_token_pool.req_to_token.shape[1],
|
||||
@@ -1345,10 +1346,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return False
|
||||
if return_logprob and not self._uses_eager_prefill_tail():
|
||||
return False
|
||||
if self.max_context_size is not None:
|
||||
if (
|
||||
batch_max_context_len is None
|
||||
or batch_max_context_len > self.max_context_size
|
||||
# Vote on both bounds so DP ranks agree on graph versus eager.
|
||||
for context_limit in (self.max_context_size, self.max_seq_len):
|
||||
if context_limit is not None and (
|
||||
batch_max_context_len is None or batch_max_context_len > context_limit
|
||||
):
|
||||
return False
|
||||
if num_tokens is None:
|
||||
@@ -1386,7 +1387,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# Non-DP local check (sole decision for tp-only).
|
||||
batch_max_context_len = (
|
||||
self._batch_max_context_len(forward_batch)
|
||||
if self.max_context_size is not None
|
||||
if self.max_context_size is not None or self.max_seq_len is not None
|
||||
else None
|
||||
)
|
||||
if not self.can_replay_locally(
|
||||
|
||||
@@ -2466,6 +2466,8 @@ class MQALayer(MqaAttentionBase):
|
||||
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
|
||||
o = attn_q.new_empty(
|
||||
(*attn_q.shape[:-1], self.attn_mqa.v_head_dim),
|
||||
# TRT-LLM returns BF16 even with FP8 queries.
|
||||
dtype=x.dtype,
|
||||
)
|
||||
bcg_deepseek_v4_attention_with_output(
|
||||
attn_q,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""DeepSeek-V4.1-Flash accuracy with DSPARK on four GB300 GPUs.
|
||||
|
||||
Run GSM8K and MMLU in TP4+EP4 and DP-attention4 + EP4 configurations.
|
||||
Run GSM8K and MMLU in TP4+EP4 and DP-attention4 + EP4 configurations,
|
||||
plus TRT-LLM attention with decoder bounded replay and breakable prefill.
|
||||
The DP+EP recipe is intentionally exercised rather than skipped: startup or
|
||||
accuracy failures must be visible in CI while this path is being validated.
|
||||
"""
|
||||
@@ -19,7 +20,7 @@ from sglang.test.test_utils import (
|
||||
try_cached_model,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=1100, stage="base-c", runner_config="4-gpu-gb300")
|
||||
register_cuda_ci(est_time=1650, stage="base-c", runner_config="4-gpu-gb300")
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
|
||||
|
||||
@@ -122,5 +123,27 @@ class TestDSV41FlashDP4EP4DSpark(DSV41FlashAccuracyMixin, CustomTestCase):
|
||||
]
|
||||
|
||||
|
||||
class TestDSV41FlashTrtllmBoundedBCG(DSV41FlashAccuracyMixin, CustomTestCase):
|
||||
"""TRT-LLM varlen attention through the full and bounded-tail graph breaks."""
|
||||
|
||||
parallel_args = [
|
||||
"--attn-dp-size",
|
||||
"1",
|
||||
"--ep-size",
|
||||
"4",
|
||||
"--dsv4-attn-backend",
|
||||
"trtllm",
|
||||
"--enable-decoder-swa-bounded-replay",
|
||||
"--cuda-graph-backend-prefill",
|
||||
"breakable",
|
||||
"--cuda-graph-max-bs-prefill",
|
||||
"4096",
|
||||
"--cuda-graph-max-seq-len-prefill",
|
||||
"16384",
|
||||
"--chunked-prefill-size",
|
||||
"4096",
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -44,6 +44,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
runner.capture_hidden_mode = CaptureHiddenMode.NULL
|
||||
runner.capture_num_tokens = [4, 16]
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner.max_num_tokens = 16
|
||||
return runner
|
||||
|
||||
|
||||
@@ -405,6 +405,7 @@ class TestDecodeToExtendConversionVote(CustomTestCase):
|
||||
runner = Mock(spec=dp_attn.PrefillCudaGraphRunner)
|
||||
runner.enable_lora = False
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner.can_replay_locally.return_value = True
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
@@ -458,6 +459,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
|
||||
runner = Mock(spec=runner_cls)
|
||||
runner.enable_lora = False
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner._qwen_bcg_hc_sidechannel = True
|
||||
runner.can_replay_locally.side_effect = lambda **kwargs: (
|
||||
not kwargs["contains_mm_inputs"]
|
||||
@@ -505,6 +507,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
|
||||
runner = Mock(spec=runner_cls)
|
||||
runner.enable_lora = False
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner._qwen_bcg_hc_sidechannel = False
|
||||
runner._is_full_backend = False
|
||||
runner._captured_attn_metadata_max_bs = 2
|
||||
|
||||
@@ -74,6 +74,7 @@ class TestHiddenStateGraphRecapture(CustomTestCase):
|
||||
runner.capture_hidden_mode = capture_hidden_mode
|
||||
runner.capture_num_tokens = [4]
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner.max_num_tokens = 4
|
||||
return runner
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ import torch
|
||||
|
||||
import sglang.srt.model_executor.runner.prefill_cuda_graph_runner as runner_module
|
||||
from sglang.srt.layers.moe.utils import MoeA2ABackend
|
||||
from sglang.srt.managers.scheduler_components.dp_attn import (
|
||||
_local_prefill_cuda_graph_vote,
|
||||
)
|
||||
from sglang.srt.model_executor import forward_batch_info
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -36,6 +39,7 @@ class TestPrefillCudaGraphPadding(CustomTestCase):
|
||||
runner.capture_hidden_mode = CaptureHiddenMode.NULL
|
||||
runner.capture_num_tokens = [4, 16]
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner.max_num_tokens = 16
|
||||
return runner
|
||||
|
||||
@@ -238,6 +242,56 @@ class TestPrefillCudaGraphPadding(CustomTestCase):
|
||||
uncovered.seq_lens_cpu.fill_(701)
|
||||
self.assertFalse(runner.can_run_graph(uncovered))
|
||||
|
||||
def test_sequence_bound_votes_eager_for_active_and_idle_dp_ranks(self):
|
||||
runner = self._make_runner()
|
||||
runner.prefill_backend_name = Backend.BREAKABLE
|
||||
runner.max_seq_len = 16384
|
||||
for context_len, expected in ((16384, True), (17000, False)):
|
||||
with self.subTest(context_len=context_len):
|
||||
active = self._make_forward_batch(4)
|
||||
active.seq_lens_cpu.fill_(context_len)
|
||||
schedule_batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
batch_size=lambda: 1,
|
||||
extend_num_tokens=4,
|
||||
input_embeds=None,
|
||||
replace_embeds=None,
|
||||
prefix_lens=[context_len - 4],
|
||||
return_logprob=False,
|
||||
multimodal_inputs=None,
|
||||
seq_lens_cpu=active.seq_lens_cpu,
|
||||
)
|
||||
votes = [
|
||||
_local_prefill_cuda_graph_vote(
|
||||
local_batch=batch,
|
||||
prefill_graph_runner=runner,
|
||||
coordinated_prefill=True,
|
||||
breakable_prefill=True,
|
||||
spec_algorithm=None,
|
||||
model_config=None,
|
||||
)
|
||||
for batch in (schedule_batch, None)
|
||||
]
|
||||
self.assertEqual(votes, [expected, True])
|
||||
# The padded idle request fits; it must still honor the vote.
|
||||
idle = self._make_forward_batch(4)
|
||||
for batch in (active, idle):
|
||||
batch.global_num_tokens_cpu = [4, 4]
|
||||
batch.can_run_dp_prefill_cuda_graph = all(votes)
|
||||
self.assertEqual(runner.can_run_graph(batch), expected)
|
||||
|
||||
def test_context_and_sequence_bounds_both_apply(self):
|
||||
runner = self._make_runner()
|
||||
for context_limit, sequence_limit in ((700, 900), (900, 700)):
|
||||
with self.subTest(context=context_limit, sequence=sequence_limit):
|
||||
runner.max_context_size = context_limit
|
||||
runner.max_seq_len = sequence_limit
|
||||
batch = self._make_forward_batch(4)
|
||||
batch.seq_lens_cpu.fill_(700)
|
||||
self.assertTrue(runner.can_run_graph(batch))
|
||||
batch.seq_lens_cpu.fill_(701)
|
||||
self.assertFalse(runner.can_run_graph(batch))
|
||||
|
||||
def test_unsupported_path_ignores_max_context_size(self):
|
||||
runner = self._make_runner()
|
||||
runner.max_context_size = 1024
|
||||
|
||||
@@ -215,6 +215,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
runner.device = torch.device("cpu")
|
||||
runner.prefill_backend_name = prefill_backend_name
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner._capture_req_slots = capture_req_slots
|
||||
runner.max_bs = 4
|
||||
runner._prefill_static_buffers = None
|
||||
@@ -384,6 +385,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner)
|
||||
runner.capture_num_tokens = [4]
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner._capture_chunked_prefix = False
|
||||
runner.buffer_registry = _FakeBatchRegistry()
|
||||
translator = _static_pool_translator()
|
||||
@@ -632,6 +634,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
runner.max_num_tokens = 4
|
||||
runner.capture_num_tokens = [4]
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner._capture_chunked_prefix = False
|
||||
runner.has_mha_companion_layers = False
|
||||
|
||||
@@ -698,6 +701,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
runner.max_num_tokens = 32
|
||||
runner.capture_num_tokens = [4]
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner.backend = SimpleNamespace()
|
||||
runner.prefill_backend_name = Backend.FULL
|
||||
runner.has_mha_companion_layers = False
|
||||
|
||||
Reference in New Issue
Block a user