From 1d809bbbbdc64978ad3c115ed10729a5c4b9db68 Mon Sep 17 00:00:00 2001 From: YAMY <74099316+YAMY1234@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:04:58 -0700 Subject: [PATCH] [Fix] DSv4.1: Size the encoder-replay SWA pool by the DSpark draft's stages, and keep more prefix tails (#43502) Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang Co-authored-by: Claude Opus 5.5 Co-authored-by: Yuwei An --- .../sglang/srt/arg_groups/fields/schedule.py | 4 +- .../spec_aux_hidden_state.py | 8 ++- .../srt/model_executor/pool_configurator.py | 12 ++++- .../model_executor/test_pool_configurator.py | 20 ++++++- .../test_spec_aux_hidden_state_resolver.py | 52 ++++++++++++++++++- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/arg_groups/fields/schedule.py b/python/sglang/srt/arg_groups/fields/schedule.py index 8e8fbe5c8c..d84e927be9 100644 --- a/python/sglang/srt/arg_groups/fields/schedule.py +++ b/python/sglang/srt/arg_groups/fields/schedule.py @@ -168,7 +168,9 @@ class Schedule(msgspec.Struct): "When the SWA KV pool is sized from the request cap (DeepSeek-V4 " "family), how many radix-cached prefix tails it keeps room for. " "Each tail is one sliding window plus one page. Default: 4 x " - "max_running_requests per attention-DP rank, 0 when the radix " + "max_running_requests per attention-DP rank, multiplied by " + "num_layers // draft_layers under encoder SWA bounded replay with " + "DSpark (only draft layers are paged there); 0 when the radix " "cache is disabled." ), ), diff --git a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py index eba7d123f0..d8a70ce57e 100644 --- a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py +++ b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Optional import msgspec -from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.configs.model_config import ModelConfig, is_deepseek_v4 from sglang.srt.runtime_context import ( get_model, get_parallel, @@ -44,6 +44,9 @@ class SpecAuxHiddenStateConfig(msgspec.Struct, kw_only=True): dflash_target_layer_ids: Any = None # DFLASH draft KV bytes/token; None when unresolved. dflash_draft_cell_size_per_token: int | None = None + # DeepSeek-V4 DSpark draft stages (DeepseekV4ForCausalLMDSpark.num_stages); a + # bundled head inherits the target's num_hidden_layers, so that overcounts. + dspark_num_stages: Optional[int] = None # The draft checkpoint read in draft mode, and its KV layer count, for # every drafting algorithm: the fused-draft placement's input. A private # EAGLE draft pool is sized by `eagle_draft_num_layers` instead. @@ -219,6 +222,9 @@ def _resolve_dflash_aux_hidden_state( ) if dspark_draft_config.target_layer_ids is not None: target_layer_ids = list(dspark_draft_config.target_layer_ids) + # Dense DSpark drafts build num_hidden_layers stages instead. + if is_deepseek_v4(draft_model_config.hf_config): + config.dspark_num_stages = len(target_layer_ids) config.dflash_use_aux_hidden_state = True config.dflash_draft_num_layers = int(draft_num_layers) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 4efd2fda8a..bebd8277df 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -1173,8 +1173,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): self.encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay self.paged_draft_layers = 0 if self.encoder_replay and kvc.spec_algorithm.is_dspark(): + aux = kvc.spec_aux_config self.paged_draft_layers = int( - kvc.spec_aux_config.dflash_draft_num_layers or 0 + aux.dspark_num_stages or aux.dflash_draft_num_layers or 0 ) assert self.paged_draft_layers > 0, "DSpark draft layer count is required" self.request_window_bytes = 0 @@ -1277,7 +1278,14 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): # Nothing is kept for reuse, so the request cap alone bounds the pool. return 0 max_running_requests = self.requested_max_running_requests_per_worker - return 4 * max_running_requests if max_running_requests is not None else 0 + if max_running_requests is None: + return 0 + tails = 4 * max_running_requests + if self.encoder_replay and self.paged_draft_layers: + # A replay tail pays for draft layers only; spend the non-replay + # default's bytes on proportionally more tails. + tails *= max(1, self.num_layers_total // self.paged_draft_layers) + return tails def _resolve_swa_cap_tokens(self) -> Optional[int]: """SWA slots to reserve in cap mode, None to keep ratio sizing. Cap mode diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 11c125cd4b..ff05a1e256 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1524,7 +1524,9 @@ class TestSWAPoolFloor(CustomTestCase): sliding_window_size=128, page_size=256, spec_algorithm=spec, - spec_aux_config=SimpleNamespace(dflash_draft_num_layers=3), + spec_aux_config=SimpleNamespace( + dflash_draft_num_layers=40, dspark_num_stages=3 + ), ) planner = DSV4PoolConfigurator(kvc) self.assertEqual(planner.bytes_per_swa_token, 3 * 584) @@ -1537,6 +1539,22 @@ class TestSWAPoolFloor(CustomTestCase): budget, ) + def test_dsv4_replay_prefix_tails_scale_with_draft_layers(self): + from sglang.srt.model_executor.pool_configurator import DSV4PoolConfigurator + + _publish_config(self, max_running_requests=64) + cfg = object.__new__(DSV4PoolConfigurator) + cfg.requested_max_running_requests_per_worker = 64 + cfg.num_layers_total = 43 + cfg.encoder_replay = False + cfg.paged_draft_layers = 0 + self.assertEqual(cfg._resolve_swa_prefix_tails(), 256) + cfg.encoder_replay = True + cfg.paged_draft_layers = 3 + self.assertEqual(cfg._resolve_swa_prefix_tails(), 256 * 14) + _publish_config(self, swa_prefix_tails=1024) + self.assertEqual(cfg._resolve_swa_prefix_tails(), 1024) + def test_dsv4_trtllm_kv_bytes(self): from sglang.srt.model_executor.pool_configurator import DSV4PoolConfigurator diff --git a/test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py b/test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py index 84755b272f..b92de5375e 100644 --- a/test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py +++ b/test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py @@ -11,7 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Which config the EAGLE aux-hidden-state resolver sizes the draft KV from. +"""Which config the EAGLE aux-hidden-state resolver sizes the draft KV from, +and the stage count it records for a DSpark draft. python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py -v """ @@ -86,5 +87,54 @@ class TestEagleResolverDraftConfig(unittest.TestCase): self.assertEqual(config.eagle_draft_num_layers, 2) +class TestDSparkResolverStages(unittest.TestCase): + def _resolve(self, architecture): + """Run the DFLASH-family resolver for a DSpark draft that inherits a + 40-layer config but captures three target layers.""" + config = _mod.SpecAuxHiddenStateConfig() + draft = SimpleNamespace( + hf_config=SimpleNamespace( + architectures=[architecture], + num_hidden_layers=40, + dspark_block_size=5, + dspark_markov_rank=1, + dspark_target_layer_ids=[1, 20, 39], + ) + ) + target = SimpleNamespace( + hf_config=SimpleNamespace(architectures=["DeepseekV4ForCausalLM"]), + hf_text_config=SimpleNamespace(num_hidden_layers=43), + ) + spec = SimpleNamespace( + speculative_draft_model_path="/draft", + speculative_draft_model_revision=None, + ) + with ( + patch.object(_mod, "get_spec", return_value=spec), + patch.object(_mod.ModelConfig, "from_server_args", return_value=draft), + patch.object(_mod, "_resolve_dflash_draft_cell_size", return_value=None), + ): + _mod._resolve_dflash_aux_hidden_state( + config=config, + server_args=None, + model_config=target, + spec_algorithm=SimpleNamespace( + is_dflash_family=lambda: True, is_dspark=lambda: True + ), + is_draft_worker=False, + ) + self.assertEqual(config.dflash_draft_num_layers, 40) + self.assertEqual(config.dflash_target_layer_ids, [1, 20, 39]) + return config + + def test_deepseek_v4_draft_counts_target_layer_stages(self): + config = self._resolve("DeepseekV4ForCausalLMDSpark") + self.assertEqual(config.dspark_num_stages, 3) + + def test_dense_draft_leaves_stages_unset(self): + config = self._resolve("Qwen3DSparkModel") + self.assertIsNone(config.dspark_num_stages) + + if __name__ == "__main__": unittest.main()