From aaf5f397a5a6565ad2cbcfe64bab56bdebbe4b75 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Fri, 16 Oct 2026 15:38:33 -0700 Subject: [PATCH] [Feat] dsv4.1: Support encoder SWA bounded replay with CUDA DP attention (#43813) Co-authored-by: Claude Fable 5.1 --- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 26 ++++++++++++------- python/sglang/srt/arg_groups/model_hook.py | 7 +++++ .../managers/scheduler_components/dp_attn.py | 6 +++++ .../srt/model_executor/encoder_swa_replay.py | 13 ++++++++-- .../model_executor/test_encoder_swa_replay.py | 20 +++++++++++--- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 4c0b31094b..2a02dc2146 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -234,6 +234,22 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: ) +def validate_deepseek_v41_encoder_replay_budget(server_args: ServerArgs) -> None: + cfg = resolving_view(server_args) + if not cfg.enable_encoder_swa_bounded_replay: + return + if ( + cfg.max_running_requests is None + or cfg.max_running_requests <= 0 + or not cfg.chunked_prefill_size + or cfg.chunked_prefill_size < 128 + ): + raise ValueError( + "encoder SWA replay requires --max-running-requests > 0 and at least " + "128 chunked prefill tokens per attention DP rank" + ) + + def validate_deepseek_v41_features(server_args: ServerArgs) -> None: from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( is_unified_kv_triton, @@ -268,7 +284,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: "prefill CUDA graphs", cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, ), - ("DP attention", attn_dp_enabled_of(cfg)), ("context parallelism", cfg.attn_cp_size > 1), ("external cache linker", cfg.enable_unified_cache_external_linker), ("unified memory", cfg.enable_unified_memory), @@ -282,15 +297,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: raise ValueError( f"--enable-encoder-swa-bounded-replay does not support {feature} yet" ) - if ( - cfg.max_running_requests is None - or cfg.max_running_requests <= 0 - or not cfg.chunked_prefill_size - or cfg.chunked_prefill_size < 128 - ): - raise ValueError( - "encoder SWA replay requires explicit --max-running-requests and --chunked-prefill-size >= 128" - ) unsupported = ( ( diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index b71a4cc535..15babcaf42 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -1020,6 +1020,13 @@ def handle_model_capability_adjustments(server_args: Any): f"{hf_config.model_type}" ) + if getattr(hf_config, "model_type", None) == "deepseek_v41": + from sglang.srt.arg_groups.deepseek_v4_hook import ( + validate_deepseek_v41_encoder_replay_budget, + ) + + validate_deepseek_v41_encoder_replay_budget(server_args) + def handle_mamba_radix_cache(server_args: Any, hf_config: Any): # Resolution moved to the resolution pipeline (arg_groups/overrides.py: diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 6ac13f63ec..aeb21bbb09 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -32,6 +32,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( check_cuda_graph_backend, cuda_graph_fully_disabled, ) +from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.runner import PrefillCudaGraphRunner from sglang.srt.observability.metrics_collector import ( @@ -534,6 +535,11 @@ def prepare_mlp_sync_batch_raw( num_tokens_for_logprob = num_tokens else: num_tokens = local_batch.extend_num_tokens + if ( + local_batch.encoder_swa_reset is not None + and model_runner.attn_backend.folds_encoder_swa_replay + ): + num_tokens += folded_replay_rows(local_batch) num_tokens_for_logprob = sum( # We should have at least 1 token for sample in every case. max(extend_len - logprob_start_len, 1) diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index 4110047e97..cf0585354c 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -29,6 +29,16 @@ def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]: return _fold_batch(batch=batch, runner=runner, rows=rows) +def folded_replay_rows(batch) -> int: + if not batch.forward_mode.is_extend_without_speculative(): + return 0 + return sum( + min(end, 128) + for reset, end in zip(batch.encoder_swa_reset, batch.prefix_lens) + if reset + ) + + def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None: forward_batch.encoder_swa_row_floor = folded.row_floor forward_batch.encoder_swa_compress_skip = folded.compress_skip @@ -216,7 +226,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend: folded.engram_history = _engram_history( reqs=batch.reqs, starts=pre, runner=runner ) - _set_execution_counts(batch=folded, scheduled=batch) floor = torch.tensor( [p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64 ) @@ -243,7 +252,7 @@ def _set_execution_counts(*, batch, scheduled) -> None: """MLP-sync counts of a batch copied from `scheduled` with different rows.""" if scheduled.global_num_tokens is None: return - # The flag rejects attention DP, so the gather holds only this rank's count. + # Attention DP is admitted only with the fold, so the gather holds one count. if ( len(scheduled.global_num_tokens) != 1 or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index ddb3c046a5..0d953d241e 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -23,6 +23,7 @@ from sglang.srt.model_executor.encoder_swa_replay import ( _build_replay_batch, _fold_batch, drop_folded_rows, + folded_replay_rows, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import get_context, get_parallel @@ -47,6 +48,7 @@ def _tok(slot, pos): # a 64-token prefix (replay from 0) and its whole folded extend is its tail. REQS = [(1, 0, 40), (2, 512, 30), (3, 64, 20)] HITS = [(1, 384, 512), (2, 0, 64)] # (batch index, replay start, prefix) +REPLAY_ROWS = 128 + 64 class _Fixture: @@ -67,7 +69,9 @@ class _Fixture: rows = [(s, p, p + n) for s, p, n in REQS] self.extend_num_tokens = sum(n for _, _, n in REQS) self.batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, reqs=reqs, + encoder_swa_reset=[True] * len(REQS), req_pool_indices=torch.tensor([s for s, _, _ in REQS]), prefix_lens=[p for _, p, _ in REQS], extend_lens=[n for _, _, n in REQS], @@ -82,7 +86,7 @@ class _Fixture: # Slot 3 asks for input logprobs from its 6th new token on. extend_logprob_start_lens=[40, 30, 5], engram_history=None, - global_num_tokens=[self.extend_num_tokens], + global_num_tokens=[self.extend_num_tokens + REPLAY_ROWS], global_num_tokens_for_logprob=[1 + 1 + 15], can_run_decode_cuda_graph=False, can_run_dp_draft_cuda_graph=False, @@ -116,6 +120,7 @@ class TestFoldedExtendRows(CustomTestCase): self.assertEqual(fb.prefix_lens, starts) self.assertEqual(fb.extend_lens, [40, 158, 84]) self.assertEqual(fb.extend_num_tokens, 282) + self.assertEqual(folded_replay_rows(fx.batch), REPLAY_ROWS) spans = [(s, a, p + n) for (s, p, n), a in zip(REQS, starts)] torch.testing.assert_close( fb.input_ids, @@ -149,6 +154,14 @@ class TestFoldedExtendRows(CustomTestCase): out.hidden_states, fx.batch.input_ids[:, None].float() ) + def test_replay_rows_follow_window_resets_only(self): + """A request that keeps its window (a later chunk) adds no replay rows.""" + fx = _Fixture() + fx.batch.encoder_swa_reset = [False, False, True] + self.assertEqual(folded_replay_rows(fx.batch), 64) + fx.batch.forward_mode = ForwardMode.DECODE + self.assertEqual(folded_replay_rows(fx.batch), 0) + class TestFoldWithDecoderSwaTail(CustomTestCase): """--enable-encoder-swa-bounded-replay with --enable-decoder-swa-bounded-replay: @@ -312,9 +325,8 @@ class TestFoldMlpSync(CustomTestCase): fb.prepare_mlp_sync_batch(runner) def test_folded_rows_count_in_mlp_sync_totals(self): - """The scheduler's gathered count covers the scheduled rows only. A folded - batch must sync its folded row count, or the padding to that count - shrinks input_ids (a negative pad); logprob rows keep their count.""" + """The scheduler's gathered count includes the replay rows, so the folded + batch pads to exactly its own rows; logprob rows keep their count.""" fx = _Fixture() fb = self._forward_batch(fx.folded.batch) self._mlp_sync(fb)