[Feat] dsv4.1: Support encoder SWA bounded replay with CUDA DP attention (#43813)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Khoa Pham
2026-10-16 15:38:33 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 8e3f11b219
commit aaf5f397a5
5 changed files with 56 additions and 16 deletions
@@ -234,6 +234,22 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
)
def validate_deepseek_v41_encoder_replay_budget(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if not cfg.enable_encoder_swa_bounded_replay:
return
if (
cfg.max_running_requests is None
or cfg.max_running_requests <= 0
or not cfg.chunked_prefill_size
or cfg.chunked_prefill_size < 128
):
raise ValueError(
"encoder SWA replay requires --max-running-requests > 0 and at least "
"128 chunked prefill tokens per attention DP rank"
)
def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_triton,
@@ -268,7 +284,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"prefill CUDA graphs",
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
("DP attention", attn_dp_enabled_of(cfg)),
("context parallelism", cfg.attn_cp_size > 1),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
@@ -282,15 +297,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
raise ValueError(
f"--enable-encoder-swa-bounded-replay does not support {feature} yet"
)
if (
cfg.max_running_requests is None
or cfg.max_running_requests <= 0
or not cfg.chunked_prefill_size
or cfg.chunked_prefill_size < 128
):
raise ValueError(
"encoder SWA replay requires explicit --max-running-requests and --chunked-prefill-size >= 128"
)
unsupported = (
(
@@ -1020,6 +1020,13 @@ def handle_model_capability_adjustments(server_args: Any):
f"{hf_config.model_type}"
)
if getattr(hf_config, "model_type", None) == "deepseek_v41":
from sglang.srt.arg_groups.deepseek_v4_hook import (
validate_deepseek_v41_encoder_replay_budget,
)
validate_deepseek_v41_encoder_replay_budget(server_args)
def handle_mamba_radix_cache(server_args: Any, hf_config: Any):
# Resolution moved to the resolution pipeline (arg_groups/overrides.py:
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
check_cuda_graph_backend,
cuda_graph_fully_disabled,
)
from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner import PrefillCudaGraphRunner
from sglang.srt.observability.metrics_collector import (
@@ -534,6 +535,11 @@ def prepare_mlp_sync_batch_raw(
num_tokens_for_logprob = num_tokens
else:
num_tokens = local_batch.extend_num_tokens
if (
local_batch.encoder_swa_reset is not None
and model_runner.attn_backend.folds_encoder_swa_replay
):
num_tokens += folded_replay_rows(local_batch)
num_tokens_for_logprob = sum(
# We should have at least 1 token for sample in every case.
max(extend_len - logprob_start_len, 1)
@@ -29,6 +29,16 @@ def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]:
return _fold_batch(batch=batch, runner=runner, rows=rows)
def folded_replay_rows(batch) -> int:
if not batch.forward_mode.is_extend_without_speculative():
return 0
return sum(
min(end, 128)
for reset, end in zip(batch.encoder_swa_reset, batch.prefix_lens)
if reset
)
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_row_floor = folded.row_floor
forward_batch.encoder_swa_compress_skip = folded.compress_skip
@@ -216,7 +226,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
folded.engram_history = _engram_history(
reqs=batch.reqs, starts=pre, runner=runner
)
_set_execution_counts(batch=folded, scheduled=batch)
floor = torch.tensor(
[p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64
)
@@ -243,7 +252,7 @@ def _set_execution_counts(*, batch, scheduled) -> None:
"""MLP-sync counts of a batch copied from `scheduled` with different rows."""
if scheduled.global_num_tokens is None:
return
# The flag rejects attention DP, so the gather holds only this rank's count.
# Attention DP is admitted only with the fold, so the gather holds one count.
if (
len(scheduled.global_num_tokens) != 1
or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens
@@ -23,6 +23,7 @@ from sglang.srt.model_executor.encoder_swa_replay import (
_build_replay_batch,
_fold_batch,
drop_folded_rows,
folded_replay_rows,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import get_context, get_parallel
@@ -47,6 +48,7 @@ def _tok(slot, pos):
# a 64-token prefix (replay from 0) and its whole folded extend is its tail.
REQS = [(1, 0, 40), (2, 512, 30), (3, 64, 20)]
HITS = [(1, 384, 512), (2, 0, 64)] # (batch index, replay start, prefix)
REPLAY_ROWS = 128 + 64
class _Fixture:
@@ -67,7 +69,9 @@ class _Fixture:
rows = [(s, p, p + n) for s, p, n in REQS]
self.extend_num_tokens = sum(n for _, _, n in REQS)
self.batch = SimpleNamespace(
forward_mode=ForwardMode.EXTEND,
reqs=reqs,
encoder_swa_reset=[True] * len(REQS),
req_pool_indices=torch.tensor([s for s, _, _ in REQS]),
prefix_lens=[p for _, p, _ in REQS],
extend_lens=[n for _, _, n in REQS],
@@ -82,7 +86,7 @@ class _Fixture:
# Slot 3 asks for input logprobs from its 6th new token on.
extend_logprob_start_lens=[40, 30, 5],
engram_history=None,
global_num_tokens=[self.extend_num_tokens],
global_num_tokens=[self.extend_num_tokens + REPLAY_ROWS],
global_num_tokens_for_logprob=[1 + 1 + 15],
can_run_decode_cuda_graph=False,
can_run_dp_draft_cuda_graph=False,
@@ -116,6 +120,7 @@ class TestFoldedExtendRows(CustomTestCase):
self.assertEqual(fb.prefix_lens, starts)
self.assertEqual(fb.extend_lens, [40, 158, 84])
self.assertEqual(fb.extend_num_tokens, 282)
self.assertEqual(folded_replay_rows(fx.batch), REPLAY_ROWS)
spans = [(s, a, p + n) for (s, p, n), a in zip(REQS, starts)]
torch.testing.assert_close(
fb.input_ids,
@@ -149,6 +154,14 @@ class TestFoldedExtendRows(CustomTestCase):
out.hidden_states, fx.batch.input_ids[:, None].float()
)
def test_replay_rows_follow_window_resets_only(self):
"""A request that keeps its window (a later chunk) adds no replay rows."""
fx = _Fixture()
fx.batch.encoder_swa_reset = [False, False, True]
self.assertEqual(folded_replay_rows(fx.batch), 64)
fx.batch.forward_mode = ForwardMode.DECODE
self.assertEqual(folded_replay_rows(fx.batch), 0)
class TestFoldWithDecoderSwaTail(CustomTestCase):
"""--enable-encoder-swa-bounded-replay with --enable-decoder-swa-bounded-replay:
@@ -312,9 +325,8 @@ class TestFoldMlpSync(CustomTestCase):
fb.prepare_mlp_sync_batch(runner)
def test_folded_rows_count_in_mlp_sync_totals(self):
"""The scheduler's gathered count covers the scheduled rows only. A folded
batch must sync its folded row count, or the padding to that count
shrinks input_ids (a negative pad); logprob rows keep their count."""
"""The scheduler's gathered count includes the replay rows, so the folded
batch pads to exactly its own rows; logprob rows keep their count."""
fx = _Fixture()
fb = self._forward_batch(fx.folded.batch)
self._mlp_sync(fb)