[Fix] DSv4.1: Size the encoder-replay SWA pool by the DSpark draft's stages, and keep more prefix tails (#43502)

Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Yuwei An <ayw.sirius19@gmail.com>
This commit is contained in:
YAMY
2026-10-14 22:04:58 -07:00
committed by GitHub
co-authored by Po-Han Huang Po-Han Huang Claude Opus 5.5 Yuwei An
parent e573d91275
commit 1d809bbbbd
5 changed files with 90 additions and 6 deletions
@@ -168,7 +168,9 @@ class Schedule(msgspec.Struct):
"When the SWA KV pool is sized from the request cap (DeepSeek-V4 "
"family), how many radix-cached prefix tails it keeps room for. "
"Each tail is one sliding window plus one page. Default: 4 x "
"max_running_requests per attention-DP rank, 0 when the radix "
"max_running_requests per attention-DP rank, multiplied by "
"num_layers // draft_layers under encoder SWA bounded replay with "
"DSpark (only draft layers are paged there); 0 when the radix "
"cache is disabled."
),
),
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Optional
import msgspec
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.configs.model_config import ModelConfig, is_deepseek_v4
from sglang.srt.runtime_context import (
get_model,
get_parallel,
@@ -44,6 +44,9 @@ class SpecAuxHiddenStateConfig(msgspec.Struct, kw_only=True):
dflash_target_layer_ids: Any = None
# DFLASH draft KV bytes/token; None when unresolved.
dflash_draft_cell_size_per_token: int | None = None
# DeepSeek-V4 DSpark draft stages (DeepseekV4ForCausalLMDSpark.num_stages); a
# bundled head inherits the target's num_hidden_layers, so that overcounts.
dspark_num_stages: Optional[int] = None
# The draft checkpoint read in draft mode, and its KV layer count, for
# every drafting algorithm: the fused-draft placement's input. A private
# EAGLE draft pool is sized by `eagle_draft_num_layers` instead.
@@ -219,6 +222,9 @@ def _resolve_dflash_aux_hidden_state(
)
if dspark_draft_config.target_layer_ids is not None:
target_layer_ids = list(dspark_draft_config.target_layer_ids)
# Dense DSpark drafts build num_hidden_layers stages instead.
if is_deepseek_v4(draft_model_config.hf_config):
config.dspark_num_stages = len(target_layer_ids)
config.dflash_use_aux_hidden_state = True
config.dflash_draft_num_layers = int(draft_num_layers)
@@ -1173,8 +1173,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
self.encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay
self.paged_draft_layers = 0
if self.encoder_replay and kvc.spec_algorithm.is_dspark():
aux = kvc.spec_aux_config
self.paged_draft_layers = int(
kvc.spec_aux_config.dflash_draft_num_layers or 0
aux.dspark_num_stages or aux.dflash_draft_num_layers or 0
)
assert self.paged_draft_layers > 0, "DSpark draft layer count is required"
self.request_window_bytes = 0
@@ -1277,7 +1278,14 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
# Nothing is kept for reuse, so the request cap alone bounds the pool.
return 0
max_running_requests = self.requested_max_running_requests_per_worker
return 4 * max_running_requests if max_running_requests is not None else 0
if max_running_requests is None:
return 0
tails = 4 * max_running_requests
if self.encoder_replay and self.paged_draft_layers:
# A replay tail pays for draft layers only; spend the non-replay
# default's bytes on proportionally more tails.
tails *= max(1, self.num_layers_total // self.paged_draft_layers)
return tails
def _resolve_swa_cap_tokens(self) -> Optional[int]:
"""SWA slots to reserve in cap mode, None to keep ratio sizing. Cap mode