diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 1ba3b17f37..1e1667f20b 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -137,6 +137,18 @@ def _get_dsv4_compress_state_dtypes() -> tuple[torch.dtype, torch.dtype]: _is_npu = is_npu() +def _pd_prefill_skips_mamba_spec_state() -> bool: + # Prefill never runs TARGET_VERIFY, so the plain hybrid req pool skips the + # verify-only snapshots; role-switch and unified-memory pools still allocate them. + disagg = get_disagg() + return ( + disagg.disaggregation_mode == "prefill" + and not disagg.enable_pd_role_switch + and not get_memory().enable_unified_memory + and not _is_npu + ) + + def unified_fp8_for_dsv4_pool(*, is_draft_worker: bool, spec_algorithm) -> bool: """Per-pool fp8 layout. DSpark draft writers scatter bf16, so that pool stays a bf16 ring; MTP/EAGLE NextN follows the env.""" @@ -1151,12 +1163,9 @@ class KVCacheConfigurator: enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer, enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy, **self._get_ple_req_pool_kwargs(), - # A PD prefill server never runs TARGET_VERIFY, so skip the - # verify-only per-draft-token state snapshots (see the draft-head - # case above: None => the pool skips SpeculativeState). speculative_num_draft_tokens=( None - if get_disagg().disaggregation_mode == "prefill" and not _is_npu + if _pd_prefill_skips_mamba_spec_state() else max_speculative_num_draft_tokens() ), speculative_eagle_topk=get_spec().speculative_eagle_topk, @@ -2483,6 +2492,11 @@ class KVCacheConfigurator: extra_per_slot = replayssm_ring_per_slot + int( self.extra_mamba_cache_bytes_per_req * pp_layer_scale ) + reserves_spec_state = ( + has_spec_dec + and not replayssm_active + and not _pd_prefill_skips_mamba_spec_state() + ) if has_spec_dec: assert get_spec().speculative_num_draft_tokens is not None assert get_schedule().max_running_requests is not None @@ -2496,8 +2510,8 @@ class KVCacheConfigurator: ) # Reserve intermediate memory based on capped max_num_reqs (+1: the # pool's padding slot, see memory_pool.py). Skipped under replayssm - # (no intermediate_ssm allocated). - if has_spec_dec and not replayssm_active: + # and on PD prefill (no intermediate_ssm allocated). + if reserves_spec_state: ratio = self._calculate_mamba_ratio() capped_reqs = min( get_schedule().max_running_requests // self.attn_dp_size, @@ -2520,8 +2534,8 @@ class KVCacheConfigurator: // self.attn_dp_size, ) # Reserve intermediate memory based on capped max_num_reqs (+1: the - # pool's padding slot). Skipped under replayssm. - if has_spec_dec and not replayssm_active: + # pool's padding slot). Skipped under replayssm and on PD prefill. + if reserves_spec_state: intermediate_size = ( stage_per_req * (get_schedule().max_mamba_cache_size + 1) @@ -2542,7 +2556,7 @@ class KVCacheConfigurator: ) mamba_budget_bytes = mamba_budget * (1 << 30) - if has_spec_dec and not replayssm_active: + if reserves_spec_state: ratio = self._calculate_mamba_ratio() D = get_spec().speculative_num_draft_tokens # Joint solve: main_state + intermediate = mamba_budget diff --git a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py index 9e555e6c19..53da60ce86 100644 --- a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py +++ b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py @@ -393,6 +393,44 @@ class TestExtraMambaCacheSizing(unittest.TestCase): self.assertEqual(self._size(0, draft_tokens=2), (3, 88 << 20)) self.assertEqual(self._size(4 << 20, draft_tokens=2), (2, 80 << 20)) + def test_pd_prefill_does_not_reserve_speculative_scratch(self): + """A PD prefill server's hybrid req pool never allocates the per-draft-token + verify snapshots, so none of the three capacity branches may charge them.""" + prefill = dict(disaggregation_mode="prefill") + # Fixed capacity: 8 slots, ratio 2 -> 4 capped reqs; the (4 + 1) * 2 * 4 MiB + # scratch is only charged outside PD prefill. + self.assertEqual( + self._size(0, draft_tokens=2, max_mamba_cache_size=8), (8, 44 << 20) + ) + self.assertEqual( + self._size(0, draft_tokens=2, max_mamba_cache_size=8, **prefill), + (8, 84 << 20), + ) + # Capacity from max_running_requests with the radix cache disabled. + fixed = dict(disable_radix_cache=True, max_running_requests=8) + self.assertEqual(self._size(0, draft_tokens=2, **fixed), (8, 12 << 20)) + self.assertEqual( + self._size(0, draft_tokens=2, **fixed, **prefill), (8, 84 << 20) + ) + # Auto capacity sizes like a non-speculative server. + self.assertEqual(self._size(0, draft_tokens=2, **prefill), self._size(0)) + # Prefill pools that still allocate the snapshots keep the reserve. + for allocates in ( + dict(enable_pd_role_switch=True), + dict(enable_unified_memory=True), + ): + with self.subTest(**allocates): + self.assertEqual( + self._size( + 0, + draft_tokens=2, + max_mamba_cache_size=8, + **prefill, + **allocates, + ), + (8, 44 << 20), + ) + if __name__ == "__main__": unittest.main()