[Feat] dsv4.1: Support encoder SWA bounded replay with CUDA DP attention (#43813)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
8e3f11b219
commit
aaf5f397a5
@@ -234,6 +234,22 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
|
||||
def validate_deepseek_v41_encoder_replay_budget(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.enable_encoder_swa_bounded_replay:
|
||||
return
|
||||
if (
|
||||
cfg.max_running_requests is None
|
||||
or cfg.max_running_requests <= 0
|
||||
or not cfg.chunked_prefill_size
|
||||
or cfg.chunked_prefill_size < 128
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay requires --max-running-requests > 0 and at least "
|
||||
"128 chunked prefill tokens per attention DP rank"
|
||||
)
|
||||
|
||||
|
||||
def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_triton,
|
||||
@@ -268,7 +284,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"prefill CUDA graphs",
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
),
|
||||
("DP attention", attn_dp_enabled_of(cfg)),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
@@ -282,15 +297,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
raise ValueError(
|
||||
f"--enable-encoder-swa-bounded-replay does not support {feature} yet"
|
||||
)
|
||||
if (
|
||||
cfg.max_running_requests is None
|
||||
or cfg.max_running_requests <= 0
|
||||
or not cfg.chunked_prefill_size
|
||||
or cfg.chunked_prefill_size < 128
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay requires explicit --max-running-requests and --chunked-prefill-size >= 128"
|
||||
)
|
||||
|
||||
unsupported = (
|
||||
(
|
||||
|
||||
@@ -1020,6 +1020,13 @@ def handle_model_capability_adjustments(server_args: Any):
|
||||
f"{hf_config.model_type}"
|
||||
)
|
||||
|
||||
if getattr(hf_config, "model_type", None) == "deepseek_v41":
|
||||
from sglang.srt.arg_groups.deepseek_v4_hook import (
|
||||
validate_deepseek_v41_encoder_replay_budget,
|
||||
)
|
||||
|
||||
validate_deepseek_v41_encoder_replay_budget(server_args)
|
||||
|
||||
|
||||
def handle_mamba_radix_cache(server_args: Any, hf_config: Any):
|
||||
# Resolution moved to the resolution pipeline (arg_groups/overrides.py:
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
check_cuda_graph_backend,
|
||||
cuda_graph_fully_disabled,
|
||||
)
|
||||
from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner import PrefillCudaGraphRunner
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
@@ -534,6 +535,11 @@ def prepare_mlp_sync_batch_raw(
|
||||
num_tokens_for_logprob = num_tokens
|
||||
else:
|
||||
num_tokens = local_batch.extend_num_tokens
|
||||
if (
|
||||
local_batch.encoder_swa_reset is not None
|
||||
and model_runner.attn_backend.folds_encoder_swa_replay
|
||||
):
|
||||
num_tokens += folded_replay_rows(local_batch)
|
||||
num_tokens_for_logprob = sum(
|
||||
# We should have at least 1 token for sample in every case.
|
||||
max(extend_len - logprob_start_len, 1)
|
||||
|
||||
@@ -29,6 +29,16 @@ def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]:
|
||||
return _fold_batch(batch=batch, runner=runner, rows=rows)
|
||||
|
||||
|
||||
def folded_replay_rows(batch) -> int:
|
||||
if not batch.forward_mode.is_extend_without_speculative():
|
||||
return 0
|
||||
return sum(
|
||||
min(end, 128)
|
||||
for reset, end in zip(batch.encoder_swa_reset, batch.prefix_lens)
|
||||
if reset
|
||||
)
|
||||
|
||||
|
||||
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
|
||||
forward_batch.encoder_swa_row_floor = folded.row_floor
|
||||
forward_batch.encoder_swa_compress_skip = folded.compress_skip
|
||||
@@ -216,7 +226,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
|
||||
folded.engram_history = _engram_history(
|
||||
reqs=batch.reqs, starts=pre, runner=runner
|
||||
)
|
||||
_set_execution_counts(batch=folded, scheduled=batch)
|
||||
floor = torch.tensor(
|
||||
[p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64
|
||||
)
|
||||
@@ -243,7 +252,7 @@ def _set_execution_counts(*, batch, scheduled) -> None:
|
||||
"""MLP-sync counts of a batch copied from `scheduled` with different rows."""
|
||||
if scheduled.global_num_tokens is None:
|
||||
return
|
||||
# The flag rejects attention DP, so the gather holds only this rank's count.
|
||||
# Attention DP is admitted only with the fold, so the gather holds one count.
|
||||
if (
|
||||
len(scheduled.global_num_tokens) != 1
|
||||
or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.srt.model_executor.encoder_swa_replay import (
|
||||
_build_replay_batch,
|
||||
_fold_batch,
|
||||
drop_folded_rows,
|
||||
folded_replay_rows,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
@@ -47,6 +48,7 @@ def _tok(slot, pos):
|
||||
# a 64-token prefix (replay from 0) and its whole folded extend is its tail.
|
||||
REQS = [(1, 0, 40), (2, 512, 30), (3, 64, 20)]
|
||||
HITS = [(1, 384, 512), (2, 0, 64)] # (batch index, replay start, prefix)
|
||||
REPLAY_ROWS = 128 + 64
|
||||
|
||||
|
||||
class _Fixture:
|
||||
@@ -67,7 +69,9 @@ class _Fixture:
|
||||
rows = [(s, p, p + n) for s, p, n in REQS]
|
||||
self.extend_num_tokens = sum(n for _, _, n in REQS)
|
||||
self.batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
reqs=reqs,
|
||||
encoder_swa_reset=[True] * len(REQS),
|
||||
req_pool_indices=torch.tensor([s for s, _, _ in REQS]),
|
||||
prefix_lens=[p for _, p, _ in REQS],
|
||||
extend_lens=[n for _, _, n in REQS],
|
||||
@@ -82,7 +86,7 @@ class _Fixture:
|
||||
# Slot 3 asks for input logprobs from its 6th new token on.
|
||||
extend_logprob_start_lens=[40, 30, 5],
|
||||
engram_history=None,
|
||||
global_num_tokens=[self.extend_num_tokens],
|
||||
global_num_tokens=[self.extend_num_tokens + REPLAY_ROWS],
|
||||
global_num_tokens_for_logprob=[1 + 1 + 15],
|
||||
can_run_decode_cuda_graph=False,
|
||||
can_run_dp_draft_cuda_graph=False,
|
||||
@@ -116,6 +120,7 @@ class TestFoldedExtendRows(CustomTestCase):
|
||||
self.assertEqual(fb.prefix_lens, starts)
|
||||
self.assertEqual(fb.extend_lens, [40, 158, 84])
|
||||
self.assertEqual(fb.extend_num_tokens, 282)
|
||||
self.assertEqual(folded_replay_rows(fx.batch), REPLAY_ROWS)
|
||||
spans = [(s, a, p + n) for (s, p, n), a in zip(REQS, starts)]
|
||||
torch.testing.assert_close(
|
||||
fb.input_ids,
|
||||
@@ -149,6 +154,14 @@ class TestFoldedExtendRows(CustomTestCase):
|
||||
out.hidden_states, fx.batch.input_ids[:, None].float()
|
||||
)
|
||||
|
||||
def test_replay_rows_follow_window_resets_only(self):
|
||||
"""A request that keeps its window (a later chunk) adds no replay rows."""
|
||||
fx = _Fixture()
|
||||
fx.batch.encoder_swa_reset = [False, False, True]
|
||||
self.assertEqual(folded_replay_rows(fx.batch), 64)
|
||||
fx.batch.forward_mode = ForwardMode.DECODE
|
||||
self.assertEqual(folded_replay_rows(fx.batch), 0)
|
||||
|
||||
|
||||
class TestFoldWithDecoderSwaTail(CustomTestCase):
|
||||
"""--enable-encoder-swa-bounded-replay with --enable-decoder-swa-bounded-replay:
|
||||
@@ -312,9 +325,8 @@ class TestFoldMlpSync(CustomTestCase):
|
||||
fb.prepare_mlp_sync_batch(runner)
|
||||
|
||||
def test_folded_rows_count_in_mlp_sync_totals(self):
|
||||
"""The scheduler's gathered count covers the scheduled rows only. A folded
|
||||
batch must sync its folded row count, or the padding to that count
|
||||
shrinks input_ids (a negative pad); logprob rows keep their count."""
|
||||
"""The scheduler's gathered count includes the replay rows, so the folded
|
||||
batch pads to exactly its own rows; logprob rows keep their count."""
|
||||
fx = _Fixture()
|
||||
fb = self._forward_batch(fx.folded.batch)
|
||||
self._mlp_sync(fb)
|
||||
|
||||
Reference in New Issue
Block a user