[PD] Stop reserving unallocated mamba spec-verify scratch in prefill KV sizing (#42649)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-06 17:16:22 -07:00
committed by GitHub
co-authored by merrymercy 842974287
parent f497b626b6
commit f1065423d3
2 changed files with 61 additions and 9 deletions
@@ -137,6 +137,18 @@ def _get_dsv4_compress_state_dtypes() -> tuple[torch.dtype, torch.dtype]:
_is_npu = is_npu()
def _pd_prefill_skips_mamba_spec_state() -> bool:
# Prefill never runs TARGET_VERIFY, so the plain hybrid req pool skips the
# verify-only snapshots; role-switch and unified-memory pools still allocate them.
disagg = get_disagg()
return (
disagg.disaggregation_mode == "prefill"
and not disagg.enable_pd_role_switch
and not get_memory().enable_unified_memory
and not _is_npu
)
def unified_fp8_for_dsv4_pool(*, is_draft_worker: bool, spec_algorithm) -> bool:
"""Per-pool fp8 layout. DSpark draft writers scatter bf16, so that pool
stays a bf16 ring; MTP/EAGLE NextN follows the env."""
@@ -1151,12 +1163,9 @@ class KVCacheConfigurator:
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
**self._get_ple_req_pool_kwargs(),
# A PD prefill server never runs TARGET_VERIFY, so skip the
# verify-only per-draft-token state snapshots (see the draft-head
# case above: None => the pool skips SpeculativeState).
speculative_num_draft_tokens=(
None
if get_disagg().disaggregation_mode == "prefill" and not _is_npu
if _pd_prefill_skips_mamba_spec_state()
else max_speculative_num_draft_tokens()
),
speculative_eagle_topk=get_spec().speculative_eagle_topk,
@@ -2483,6 +2492,11 @@ class KVCacheConfigurator:
extra_per_slot = replayssm_ring_per_slot + int(
self.extra_mamba_cache_bytes_per_req * pp_layer_scale
)
reserves_spec_state = (
has_spec_dec
and not replayssm_active
and not _pd_prefill_skips_mamba_spec_state()
)
if has_spec_dec:
assert get_spec().speculative_num_draft_tokens is not None
assert get_schedule().max_running_requests is not None
@@ -2496,8 +2510,8 @@ class KVCacheConfigurator:
)
# Reserve intermediate memory based on capped max_num_reqs (+1: the
# pool's padding slot, see memory_pool.py). Skipped under replayssm
# (no intermediate_ssm allocated).
if has_spec_dec and not replayssm_active:
# and on PD prefill (no intermediate_ssm allocated).
if reserves_spec_state:
ratio = self._calculate_mamba_ratio()
capped_reqs = min(
get_schedule().max_running_requests // self.attn_dp_size,
@@ -2520,8 +2534,8 @@ class KVCacheConfigurator:
// self.attn_dp_size,
)
# Reserve intermediate memory based on capped max_num_reqs (+1: the
# pool's padding slot). Skipped under replayssm.
if has_spec_dec and not replayssm_active:
# pool's padding slot). Skipped under replayssm and on PD prefill.
if reserves_spec_state:
intermediate_size = (
stage_per_req
* (get_schedule().max_mamba_cache_size + 1)
@@ -2542,7 +2556,7 @@ class KVCacheConfigurator:
)
mamba_budget_bytes = mamba_budget * (1 << 30)
if has_spec_dec and not replayssm_active:
if reserves_spec_state:
ratio = self._calculate_mamba_ratio()
D = get_spec().speculative_num_draft_tokens
# Joint solve: main_state + intermediate = mamba_budget
@@ -393,6 +393,44 @@ class TestExtraMambaCacheSizing(unittest.TestCase):
self.assertEqual(self._size(0, draft_tokens=2), (3, 88 << 20))
self.assertEqual(self._size(4 << 20, draft_tokens=2), (2, 80 << 20))
def test_pd_prefill_does_not_reserve_speculative_scratch(self):
"""A PD prefill server's hybrid req pool never allocates the per-draft-token
verify snapshots, so none of the three capacity branches may charge them."""
prefill = dict(disaggregation_mode="prefill")
# Fixed capacity: 8 slots, ratio 2 -> 4 capped reqs; the (4 + 1) * 2 * 4 MiB
# scratch is only charged outside PD prefill.
self.assertEqual(
self._size(0, draft_tokens=2, max_mamba_cache_size=8), (8, 44 << 20)
)
self.assertEqual(
self._size(0, draft_tokens=2, max_mamba_cache_size=8, **prefill),
(8, 84 << 20),
)
# Capacity from max_running_requests with the radix cache disabled.
fixed = dict(disable_radix_cache=True, max_running_requests=8)
self.assertEqual(self._size(0, draft_tokens=2, **fixed), (8, 12 << 20))
self.assertEqual(
self._size(0, draft_tokens=2, **fixed, **prefill), (8, 84 << 20)
)
# Auto capacity sizes like a non-speculative server.
self.assertEqual(self._size(0, draft_tokens=2, **prefill), self._size(0))
# Prefill pools that still allocate the snapshots keep the reserve.
for allocates in (
dict(enable_pd_role_switch=True),
dict(enable_unified_memory=True),
):
with self.subTest(**allocates):
self.assertEqual(
self._size(
0,
draft_tokens=2,
max_mamba_cache_size=8,
**prefill,
**allocates,
),
(8, 44 << 20),
)
if __name__ == "__main__":
unittest.main()