From 64a0a08f0a5d5166cf78bf44c569fd349a67d8f4 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Wed, 14 Oct 2026 17:39:52 -0700 Subject: [PATCH] [DSv4.1] Support TRT-LLM decoder bounded replay with BCG (#44326) --- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 14 +++-- .../attention/deepseek_v4_trtllm_backend.py | 19 ++++--- .../managers/scheduler_components/dp_attn.py | 5 +- .../runner/prefill_cuda_graph_runner.py | 11 ++-- python/sglang/srt/models/deepseek_v4.py | 2 + .../models/test_deepseek_v41_flash_gb300.py | 27 +++++++++- .../test_multimodal_piecewise_cuda_graph.py | 1 + .../scheduler_components/test_dp_attn.py | 3 ++ .../test_hidden_state_graph_recapture.py | 1 + .../runner/test_prefill_cuda_graph_padding.py | 54 +++++++++++++++++++ .../test_prefill_cuda_graph_runner.py | 4 ++ 11 files changed, 118 insertions(+), 23 deletions(-) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index bdb1e76522..37a22e5ef1 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -215,10 +215,12 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: return if ( cfg.dsv4_attn_backend == "trtllm" - and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED + and cfg.cuda_graph_config.prefill.backend + not in (Backend.DISABLED, Backend.BREAKABLE) ): raise ValueError( - "DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled" + "DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill " + "disabled or breakable" ) if cfg.enable_encoder_swa_bounded_replay: incompatible = ( @@ -266,12 +268,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: ("HiSparse", cfg.enable_hisparse), ("the unified KV layout", is_unified_kv_triton()), ( - "TRT-LLM with SWA bounded replay", - cfg.dsv4_attn_backend == "trtllm" - and ( - cfg.enable_encoder_swa_bounded_replay - or cfg.enable_decoder_swa_bounded_replay - ), + "TRT-LLM with encoder SWA bounded replay", + cfg.dsv4_attn_backend == "trtllm" and cfg.enable_encoder_swa_bounded_replay, ), ("two-batch overlap", cfg.enable_two_batch_overlap), ("pipeline parallelism", cfg.pp_size > 1), diff --git a/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py index b98d89159e..37919fb479 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py @@ -382,11 +382,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): extra_indices: Optional[torch.Tensor], extra_topk_lengths: Optional[torch.Tensor], ) -> torch.Tensor: - """Drive the decode kernel as varlen prefill with one table row per token. - - ``seq_lens`` includes cached prefixes; the kernel derives causal SWA - validity from it. This path runs eagerly. - """ + """Run varlen prefill eagerly, including at BCG breaks.""" assert q.ndim == 3, f"{q.shape=}" num_qo_padded, num_heads, head_dim = q.shape @@ -395,7 +391,12 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): # Build VarSeq metadata from the same extend lengths as the sparse tables. core = self.forward_metadata.core_attn_metadata if core.trtllm_prefill_qmeta is None: - extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu + tail = self.forward_metadata.late_layer_tail + extend_seq_lens_cpu = ( + tail.extend_seq_lens_cpu + if tail is not None + else forward_batch.extend_seq_lens_cpu + ) assert extend_seq_lens_cpu is not None and len(extend_seq_lens_cpu) > 0 batch_size = len(extend_seq_lens_cpu) cum_lens = [0] * (batch_size + 1) @@ -404,7 +405,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): sum_q = cum_lens[-1] max_q_len = max(int(x) for x in extend_seq_lens_cpu) assert 0 < sum_q <= num_qo_padded, f"{sum_q=} {num_qo_padded=}" - seq_lens_i32 = forward_batch.seq_lens.to(torch.int32) + # SWA masking uses causal lengths, not -1 sentinels: make them + # tail-relative while keeping compressed KV indices full-context. + seq_lens_i32 = ( + tail.extend_seq_lens if tail is not None else forward_batch.seq_lens + ).to(torch.int32) assert seq_lens_i32.shape == (batch_size,), f"{seq_lens_i32.shape=}" core.trtllm_prefill_qmeta = ( self._move_to_device(cum_lens), diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 765ad679b6..8f069bdcaf 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -436,7 +436,10 @@ def _local_prefill_cuda_graph_vote( is_mixed=mode == ForwardMode.MIXED, batch_max_context_len=( int(local_batch.seq_lens_cpu.max().item()) - if prefill_graph_runner.max_context_size is not None + if ( + prefill_graph_runner.max_context_size is not None + or prefill_graph_runner.max_seq_len is not None + ) and local_batch.seq_lens_cpu is not None and local_batch.seq_lens_cpu.numel() > 0 else None diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index b4c1fccd91..4fa433e689 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -357,6 +357,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self.max_num_tokens = max(self.capture_num_tokens) self.max_bs = model_runner.req_to_token_pool.size self.max_context_size = prefill_config.max_context_size + self.max_seq_len = prefill_config.max_seq_len self._validate_max_context_capacity( max_context_size=self.max_context_size, table_width=model_runner.req_to_token_pool.req_to_token.shape[1], @@ -1345,10 +1346,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): return False if return_logprob and not self._uses_eager_prefill_tail(): return False - if self.max_context_size is not None: - if ( - batch_max_context_len is None - or batch_max_context_len > self.max_context_size + # Vote on both bounds so DP ranks agree on graph versus eager. + for context_limit in (self.max_context_size, self.max_seq_len): + if context_limit is not None and ( + batch_max_context_len is None or batch_max_context_len > context_limit ): return False if num_tokens is None: @@ -1386,7 +1387,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # Non-DP local check (sole decision for tp-only). batch_max_context_len = ( self._batch_max_context_len(forward_batch) - if self.max_context_size is not None + if self.max_context_size is not None or self.max_seq_len is not None else None ) if not self.can_replay_locally( diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index eaafd01966..c23474abe1 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -2466,6 +2466,8 @@ class MQALayer(MqaAttentionBase): if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph(): o = attn_q.new_empty( (*attn_q.shape[:-1], self.attn_mqa.v_head_dim), + # TRT-LLM returns BF16 even with FP8 queries. + dtype=x.dtype, ) bcg_deepseek_v4_attention_with_output( attn_q, diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py index c7f8b409fa..2173ae71d5 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py @@ -1,6 +1,7 @@ """DeepSeek-V4.1-Flash accuracy with DSPARK on four GB300 GPUs. -Run GSM8K and MMLU in TP4+EP4 and DP-attention4 + EP4 configurations. +Run GSM8K and MMLU in TP4+EP4 and DP-attention4 + EP4 configurations, +plus TRT-LLM attention with decoder bounded replay and breakable prefill. The DP+EP recipe is intentionally exercised rather than skipped: startup or accuracy failures must be visible in CI while this path is being validated. """ @@ -19,7 +20,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=1100, stage="base-c", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=1650, stage="base-c", runner_config="4-gpu-gb300") MODEL = "deepseek-ai/DeepSeek-V4.1-Flash" @@ -122,5 +123,27 @@ class TestDSV41FlashDP4EP4DSpark(DSV41FlashAccuracyMixin, CustomTestCase): ] +class TestDSV41FlashTrtllmBoundedBCG(DSV41FlashAccuracyMixin, CustomTestCase): + """TRT-LLM varlen attention through the full and bounded-tail graph breaks.""" + + parallel_args = [ + "--attn-dp-size", + "1", + "--ep-size", + "4", + "--dsv4-attn-backend", + "trtllm", + "--enable-decoder-swa-bounded-replay", + "--cuda-graph-backend-prefill", + "breakable", + "--cuda-graph-max-bs-prefill", + "4096", + "--cuda-graph-max-seq-len-prefill", + "16384", + "--chunked-prefill-size", + "4096", + ] + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index bc2324999e..b08cb4c66f 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -44,6 +44,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): runner.capture_hidden_mode = CaptureHiddenMode.NULL runner.capture_num_tokens = [4, 16] runner.max_context_size = None + runner.max_seq_len = None runner.max_num_tokens = 16 return runner diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index 039be3466d..d317ad50fd 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -405,6 +405,7 @@ class TestDecodeToExtendConversionVote(CustomTestCase): runner = Mock(spec=dp_attn.PrefillCudaGraphRunner) runner.enable_lora = False runner.max_context_size = None + runner.max_seq_len = None runner.can_replay_locally.return_value = True batch = SimpleNamespace( forward_mode=ForwardMode.DECODE, @@ -458,6 +459,7 @@ class TestPrefillCudaGraphVote(CustomTestCase): runner = Mock(spec=runner_cls) runner.enable_lora = False runner.max_context_size = None + runner.max_seq_len = None runner._qwen_bcg_hc_sidechannel = True runner.can_replay_locally.side_effect = lambda **kwargs: ( not kwargs["contains_mm_inputs"] @@ -505,6 +507,7 @@ class TestPrefillCudaGraphVote(CustomTestCase): runner = Mock(spec=runner_cls) runner.enable_lora = False runner.max_context_size = None + runner.max_seq_len = None runner._qwen_bcg_hc_sidechannel = False runner._is_full_backend = False runner._captured_attn_metadata_max_bs = 2 diff --git a/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py b/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py index 19356b12d6..803eef4db2 100644 --- a/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py +++ b/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py @@ -74,6 +74,7 @@ class TestHiddenStateGraphRecapture(CustomTestCase): runner.capture_hidden_mode = capture_hidden_mode runner.capture_num_tokens = [4] runner.max_context_size = None + runner.max_seq_len = None runner.max_num_tokens = 4 return runner diff --git a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py index 6293e301a8..97127c191a 100644 --- a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py +++ b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py @@ -6,6 +6,9 @@ import torch import sglang.srt.model_executor.runner.prefill_cuda_graph_runner as runner_module from sglang.srt.layers.moe.utils import MoeA2ABackend +from sglang.srt.managers.scheduler_components.dp_attn import ( + _local_prefill_cuda_graph_vote, +) from sglang.srt.model_executor import forward_batch_info from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.forward_batch_info import ( @@ -36,6 +39,7 @@ class TestPrefillCudaGraphPadding(CustomTestCase): runner.capture_hidden_mode = CaptureHiddenMode.NULL runner.capture_num_tokens = [4, 16] runner.max_context_size = None + runner.max_seq_len = None runner.max_num_tokens = 16 return runner @@ -238,6 +242,56 @@ class TestPrefillCudaGraphPadding(CustomTestCase): uncovered.seq_lens_cpu.fill_(701) self.assertFalse(runner.can_run_graph(uncovered)) + def test_sequence_bound_votes_eager_for_active_and_idle_dp_ranks(self): + runner = self._make_runner() + runner.prefill_backend_name = Backend.BREAKABLE + runner.max_seq_len = 16384 + for context_len, expected in ((16384, True), (17000, False)): + with self.subTest(context_len=context_len): + active = self._make_forward_batch(4) + active.seq_lens_cpu.fill_(context_len) + schedule_batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + batch_size=lambda: 1, + extend_num_tokens=4, + input_embeds=None, + replace_embeds=None, + prefix_lens=[context_len - 4], + return_logprob=False, + multimodal_inputs=None, + seq_lens_cpu=active.seq_lens_cpu, + ) + votes = [ + _local_prefill_cuda_graph_vote( + local_batch=batch, + prefill_graph_runner=runner, + coordinated_prefill=True, + breakable_prefill=True, + spec_algorithm=None, + model_config=None, + ) + for batch in (schedule_batch, None) + ] + self.assertEqual(votes, [expected, True]) + # The padded idle request fits; it must still honor the vote. + idle = self._make_forward_batch(4) + for batch in (active, idle): + batch.global_num_tokens_cpu = [4, 4] + batch.can_run_dp_prefill_cuda_graph = all(votes) + self.assertEqual(runner.can_run_graph(batch), expected) + + def test_context_and_sequence_bounds_both_apply(self): + runner = self._make_runner() + for context_limit, sequence_limit in ((700, 900), (900, 700)): + with self.subTest(context=context_limit, sequence=sequence_limit): + runner.max_context_size = context_limit + runner.max_seq_len = sequence_limit + batch = self._make_forward_batch(4) + batch.seq_lens_cpu.fill_(700) + self.assertTrue(runner.can_run_graph(batch)) + batch.seq_lens_cpu.fill_(701) + self.assertFalse(runner.can_run_graph(batch)) + def test_unsupported_path_ignores_max_context_size(self): runner = self._make_runner() runner.max_context_size = 1024 diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index e1aa38c653..929f8dd75c 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -215,6 +215,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): runner.device = torch.device("cpu") runner.prefill_backend_name = prefill_backend_name runner.max_context_size = None + runner.max_seq_len = None runner._capture_req_slots = capture_req_slots runner.max_bs = 4 runner._prefill_static_buffers = None @@ -384,6 +385,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner.capture_num_tokens = [4] runner.max_context_size = None + runner.max_seq_len = None runner._capture_chunked_prefix = False runner.buffer_registry = _FakeBatchRegistry() translator = _static_pool_translator() @@ -632,6 +634,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): runner.max_num_tokens = 4 runner.capture_num_tokens = [4] runner.max_context_size = None + runner.max_seq_len = None runner._capture_chunked_prefix = False runner.has_mha_companion_layers = False @@ -698,6 +701,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): runner.max_num_tokens = 32 runner.capture_num_tokens = [4] runner.max_context_size = None + runner.max_seq_len = None runner.backend = SimpleNamespace() runner.prefill_backend_name = Backend.FULL runner.has_mha_companion_layers = False