[Fix] DSv4.1: Size the encoder-replay SWA pool by the DSpark draft's stages, and keep more prefix tails (#43502)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Yuwei An <ayw.sirius19@gmail.com>
This commit is contained in:
co-authored by
Po-Han Huang
Po-Han Huang
Claude Opus 5.5
Yuwei An
parent
e573d91275
commit
1d809bbbbd
@@ -168,7 +168,9 @@ class Schedule(msgspec.Struct):
|
||||
"When the SWA KV pool is sized from the request cap (DeepSeek-V4 "
|
||||
"family), how many radix-cached prefix tails it keeps room for. "
|
||||
"Each tail is one sliding window plus one page. Default: 4 x "
|
||||
"max_running_requests per attention-DP rank, 0 when the radix "
|
||||
"max_running_requests per attention-DP rank, multiplied by "
|
||||
"num_layers // draft_layers under encoder SWA bounded replay with "
|
||||
"DSpark (only draft layers are paged there); 0 when the radix "
|
||||
"cache is disabled."
|
||||
),
|
||||
),
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.configs.model_config import ModelConfig, is_deepseek_v4
|
||||
from sglang.srt.runtime_context import (
|
||||
get_model,
|
||||
get_parallel,
|
||||
@@ -44,6 +44,9 @@ class SpecAuxHiddenStateConfig(msgspec.Struct, kw_only=True):
|
||||
dflash_target_layer_ids: Any = None
|
||||
# DFLASH draft KV bytes/token; None when unresolved.
|
||||
dflash_draft_cell_size_per_token: int | None = None
|
||||
# DeepSeek-V4 DSpark draft stages (DeepseekV4ForCausalLMDSpark.num_stages); a
|
||||
# bundled head inherits the target's num_hidden_layers, so that overcounts.
|
||||
dspark_num_stages: Optional[int] = None
|
||||
# The draft checkpoint read in draft mode, and its KV layer count, for
|
||||
# every drafting algorithm: the fused-draft placement's input. A private
|
||||
# EAGLE draft pool is sized by `eagle_draft_num_layers` instead.
|
||||
@@ -219,6 +222,9 @@ def _resolve_dflash_aux_hidden_state(
|
||||
)
|
||||
if dspark_draft_config.target_layer_ids is not None:
|
||||
target_layer_ids = list(dspark_draft_config.target_layer_ids)
|
||||
# Dense DSpark drafts build num_hidden_layers stages instead.
|
||||
if is_deepseek_v4(draft_model_config.hf_config):
|
||||
config.dspark_num_stages = len(target_layer_ids)
|
||||
|
||||
config.dflash_use_aux_hidden_state = True
|
||||
config.dflash_draft_num_layers = int(draft_num_layers)
|
||||
|
||||
@@ -1173,8 +1173,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
self.encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay
|
||||
self.paged_draft_layers = 0
|
||||
if self.encoder_replay and kvc.spec_algorithm.is_dspark():
|
||||
aux = kvc.spec_aux_config
|
||||
self.paged_draft_layers = int(
|
||||
kvc.spec_aux_config.dflash_draft_num_layers or 0
|
||||
aux.dspark_num_stages or aux.dflash_draft_num_layers or 0
|
||||
)
|
||||
assert self.paged_draft_layers > 0, "DSpark draft layer count is required"
|
||||
self.request_window_bytes = 0
|
||||
@@ -1277,7 +1278,14 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
# Nothing is kept for reuse, so the request cap alone bounds the pool.
|
||||
return 0
|
||||
max_running_requests = self.requested_max_running_requests_per_worker
|
||||
return 4 * max_running_requests if max_running_requests is not None else 0
|
||||
if max_running_requests is None:
|
||||
return 0
|
||||
tails = 4 * max_running_requests
|
||||
if self.encoder_replay and self.paged_draft_layers:
|
||||
# A replay tail pays for draft layers only; spend the non-replay
|
||||
# default's bytes on proportionally more tails.
|
||||
tails *= max(1, self.num_layers_total // self.paged_draft_layers)
|
||||
return tails
|
||||
|
||||
def _resolve_swa_cap_tokens(self) -> Optional[int]:
|
||||
"""SWA slots to reserve in cap mode, None to keep ratio sizing. Cap mode
|
||||
|
||||
Reference in New Issue
Block a user