[Feat] dsv4.1: Support encoder SWA bounded replay with CUDA DP attention (#43813)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Khoa Pham
2026-10-16 15:38:33 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 8e3f11b219
commit aaf5f397a5
5 changed files with 56 additions and 16 deletions
@@ -234,6 +234,22 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
)
def validate_deepseek_v41_encoder_replay_budget(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if not cfg.enable_encoder_swa_bounded_replay:
return
if (
cfg.max_running_requests is None
or cfg.max_running_requests <= 0
or not cfg.chunked_prefill_size
or cfg.chunked_prefill_size < 128
):
raise ValueError(
"encoder SWA replay requires --max-running-requests > 0 and at least "
"128 chunked prefill tokens per attention DP rank"
)
def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_triton,
@@ -268,7 +284,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"prefill CUDA graphs",
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
("DP attention", attn_dp_enabled_of(cfg)),
("context parallelism", cfg.attn_cp_size > 1),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
@@ -282,15 +297,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
raise ValueError(
f"--enable-encoder-swa-bounded-replay does not support {feature} yet"
)
if (
cfg.max_running_requests is None
or cfg.max_running_requests <= 0
or not cfg.chunked_prefill_size
or cfg.chunked_prefill_size < 128
):
raise ValueError(
"encoder SWA replay requires explicit --max-running-requests and --chunked-prefill-size >= 128"
)
unsupported = (
(
@@ -1020,6 +1020,13 @@ def handle_model_capability_adjustments(server_args: Any):
f"{hf_config.model_type}"
)
if getattr(hf_config, "model_type", None) == "deepseek_v41":
from sglang.srt.arg_groups.deepseek_v4_hook import (
validate_deepseek_v41_encoder_replay_budget,
)
validate_deepseek_v41_encoder_replay_budget(server_args)
def handle_mamba_radix_cache(server_args: Any, hf_config: Any):
# Resolution moved to the resolution pipeline (arg_groups/overrides.py:
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
check_cuda_graph_backend,
cuda_graph_fully_disabled,
)
from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner import PrefillCudaGraphRunner
from sglang.srt.observability.metrics_collector import (
@@ -534,6 +535,11 @@ def prepare_mlp_sync_batch_raw(
num_tokens_for_logprob = num_tokens
else:
num_tokens = local_batch.extend_num_tokens
if (
local_batch.encoder_swa_reset is not None
and model_runner.attn_backend.folds_encoder_swa_replay
):
num_tokens += folded_replay_rows(local_batch)
num_tokens_for_logprob = sum(
# We should have at least 1 token for sample in every case.
max(extend_len - logprob_start_len, 1)
@@ -29,6 +29,16 @@ def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]:
return _fold_batch(batch=batch, runner=runner, rows=rows)
def folded_replay_rows(batch) -> int:
if not batch.forward_mode.is_extend_without_speculative():
return 0
return sum(
min(end, 128)
for reset, end in zip(batch.encoder_swa_reset, batch.prefix_lens)
if reset
)
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_row_floor = folded.row_floor
forward_batch.encoder_swa_compress_skip = folded.compress_skip
@@ -216,7 +226,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
folded.engram_history = _engram_history(
reqs=batch.reqs, starts=pre, runner=runner
)
_set_execution_counts(batch=folded, scheduled=batch)
floor = torch.tensor(
[p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64
)
@@ -243,7 +252,7 @@ def _set_execution_counts(*, batch, scheduled) -> None:
"""MLP-sync counts of a batch copied from `scheduled` with different rows."""
if scheduled.global_num_tokens is None:
return
# The flag rejects attention DP, so the gather holds only this rank's count.
# Attention DP is admitted only with the fold, so the gather holds one count.
if (
len(scheduled.global_num_tokens) != 1
or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens