[PD] Stop reserving unallocated mamba spec-verify scratch in prefill KV sizing (#42649)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
842974287
parent
f497b626b6
commit
f1065423d3
@@ -137,6 +137,18 @@ def _get_dsv4_compress_state_dtypes() -> tuple[torch.dtype, torch.dtype]:
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
def _pd_prefill_skips_mamba_spec_state() -> bool:
|
||||
# Prefill never runs TARGET_VERIFY, so the plain hybrid req pool skips the
|
||||
# verify-only snapshots; role-switch and unified-memory pools still allocate them.
|
||||
disagg = get_disagg()
|
||||
return (
|
||||
disagg.disaggregation_mode == "prefill"
|
||||
and not disagg.enable_pd_role_switch
|
||||
and not get_memory().enable_unified_memory
|
||||
and not _is_npu
|
||||
)
|
||||
|
||||
|
||||
def unified_fp8_for_dsv4_pool(*, is_draft_worker: bool, spec_algorithm) -> bool:
|
||||
"""Per-pool fp8 layout. DSpark draft writers scatter bf16, so that pool
|
||||
stays a bf16 ring; MTP/EAGLE NextN follows the env."""
|
||||
@@ -1151,12 +1163,9 @@ class KVCacheConfigurator:
|
||||
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
|
||||
**self._get_ple_req_pool_kwargs(),
|
||||
# A PD prefill server never runs TARGET_VERIFY, so skip the
|
||||
# verify-only per-draft-token state snapshots (see the draft-head
|
||||
# case above: None => the pool skips SpeculativeState).
|
||||
speculative_num_draft_tokens=(
|
||||
None
|
||||
if get_disagg().disaggregation_mode == "prefill" and not _is_npu
|
||||
if _pd_prefill_skips_mamba_spec_state()
|
||||
else max_speculative_num_draft_tokens()
|
||||
),
|
||||
speculative_eagle_topk=get_spec().speculative_eagle_topk,
|
||||
@@ -2483,6 +2492,11 @@ class KVCacheConfigurator:
|
||||
extra_per_slot = replayssm_ring_per_slot + int(
|
||||
self.extra_mamba_cache_bytes_per_req * pp_layer_scale
|
||||
)
|
||||
reserves_spec_state = (
|
||||
has_spec_dec
|
||||
and not replayssm_active
|
||||
and not _pd_prefill_skips_mamba_spec_state()
|
||||
)
|
||||
if has_spec_dec:
|
||||
assert get_spec().speculative_num_draft_tokens is not None
|
||||
assert get_schedule().max_running_requests is not None
|
||||
@@ -2496,8 +2510,8 @@ class KVCacheConfigurator:
|
||||
)
|
||||
# Reserve intermediate memory based on capped max_num_reqs (+1: the
|
||||
# pool's padding slot, see memory_pool.py). Skipped under replayssm
|
||||
# (no intermediate_ssm allocated).
|
||||
if has_spec_dec and not replayssm_active:
|
||||
# and on PD prefill (no intermediate_ssm allocated).
|
||||
if reserves_spec_state:
|
||||
ratio = self._calculate_mamba_ratio()
|
||||
capped_reqs = min(
|
||||
get_schedule().max_running_requests // self.attn_dp_size,
|
||||
@@ -2520,8 +2534,8 @@ class KVCacheConfigurator:
|
||||
// self.attn_dp_size,
|
||||
)
|
||||
# Reserve intermediate memory based on capped max_num_reqs (+1: the
|
||||
# pool's padding slot). Skipped under replayssm.
|
||||
if has_spec_dec and not replayssm_active:
|
||||
# pool's padding slot). Skipped under replayssm and on PD prefill.
|
||||
if reserves_spec_state:
|
||||
intermediate_size = (
|
||||
stage_per_req
|
||||
* (get_schedule().max_mamba_cache_size + 1)
|
||||
@@ -2542,7 +2556,7 @@ class KVCacheConfigurator:
|
||||
)
|
||||
mamba_budget_bytes = mamba_budget * (1 << 30)
|
||||
|
||||
if has_spec_dec and not replayssm_active:
|
||||
if reserves_spec_state:
|
||||
ratio = self._calculate_mamba_ratio()
|
||||
D = get_spec().speculative_num_draft_tokens
|
||||
# Joint solve: main_state + intermediate = mamba_budget
|
||||
|
||||
@@ -393,6 +393,44 @@ class TestExtraMambaCacheSizing(unittest.TestCase):
|
||||
self.assertEqual(self._size(0, draft_tokens=2), (3, 88 << 20))
|
||||
self.assertEqual(self._size(4 << 20, draft_tokens=2), (2, 80 << 20))
|
||||
|
||||
def test_pd_prefill_does_not_reserve_speculative_scratch(self):
|
||||
"""A PD prefill server's hybrid req pool never allocates the per-draft-token
|
||||
verify snapshots, so none of the three capacity branches may charge them."""
|
||||
prefill = dict(disaggregation_mode="prefill")
|
||||
# Fixed capacity: 8 slots, ratio 2 -> 4 capped reqs; the (4 + 1) * 2 * 4 MiB
|
||||
# scratch is only charged outside PD prefill.
|
||||
self.assertEqual(
|
||||
self._size(0, draft_tokens=2, max_mamba_cache_size=8), (8, 44 << 20)
|
||||
)
|
||||
self.assertEqual(
|
||||
self._size(0, draft_tokens=2, max_mamba_cache_size=8, **prefill),
|
||||
(8, 84 << 20),
|
||||
)
|
||||
# Capacity from max_running_requests with the radix cache disabled.
|
||||
fixed = dict(disable_radix_cache=True, max_running_requests=8)
|
||||
self.assertEqual(self._size(0, draft_tokens=2, **fixed), (8, 12 << 20))
|
||||
self.assertEqual(
|
||||
self._size(0, draft_tokens=2, **fixed, **prefill), (8, 84 << 20)
|
||||
)
|
||||
# Auto capacity sizes like a non-speculative server.
|
||||
self.assertEqual(self._size(0, draft_tokens=2, **prefill), self._size(0))
|
||||
# Prefill pools that still allocate the snapshots keep the reserve.
|
||||
for allocates in (
|
||||
dict(enable_pd_role_switch=True),
|
||||
dict(enable_unified_memory=True),
|
||||
):
|
||||
with self.subTest(**allocates):
|
||||
self.assertEqual(
|
||||
self._size(
|
||||
0,
|
||||
draft_tokens=2,
|
||||
max_mamba_cache_size=8,
|
||||
**prefill,
|
||||
**allocates,
|
||||
),
|
||||
(8, 44 << 20),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user