[PD] Stop reserving unallocated mamba spec-verify scratch in prefill KV sizing (#42649)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
842974287
parent
f497b626b6
commit
f1065423d3
@@ -137,6 +137,18 @@ def _get_dsv4_compress_state_dtypes() -> tuple[torch.dtype, torch.dtype]:
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
def _pd_prefill_skips_mamba_spec_state() -> bool:
|
||||
# Prefill never runs TARGET_VERIFY, so the plain hybrid req pool skips the
|
||||
# verify-only snapshots; role-switch and unified-memory pools still allocate them.
|
||||
disagg = get_disagg()
|
||||
return (
|
||||
disagg.disaggregation_mode == "prefill"
|
||||
and not disagg.enable_pd_role_switch
|
||||
and not get_memory().enable_unified_memory
|
||||
and not _is_npu
|
||||
)
|
||||
|
||||
|
||||
def unified_fp8_for_dsv4_pool(*, is_draft_worker: bool, spec_algorithm) -> bool:
|
||||
"""Per-pool fp8 layout. DSpark draft writers scatter bf16, so that pool
|
||||
stays a bf16 ring; MTP/EAGLE NextN follows the env."""
|
||||
@@ -1151,12 +1163,9 @@ class KVCacheConfigurator:
|
||||
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
|
||||
**self._get_ple_req_pool_kwargs(),
|
||||
# A PD prefill server never runs TARGET_VERIFY, so skip the
|
||||
# verify-only per-draft-token state snapshots (see the draft-head
|
||||
# case above: None => the pool skips SpeculativeState).
|
||||
speculative_num_draft_tokens=(
|
||||
None
|
||||
if get_disagg().disaggregation_mode == "prefill" and not _is_npu
|
||||
if _pd_prefill_skips_mamba_spec_state()
|
||||
else max_speculative_num_draft_tokens()
|
||||
),
|
||||
speculative_eagle_topk=get_spec().speculative_eagle_topk,
|
||||
@@ -2483,6 +2492,11 @@ class KVCacheConfigurator:
|
||||
extra_per_slot = replayssm_ring_per_slot + int(
|
||||
self.extra_mamba_cache_bytes_per_req * pp_layer_scale
|
||||
)
|
||||
reserves_spec_state = (
|
||||
has_spec_dec
|
||||
and not replayssm_active
|
||||
and not _pd_prefill_skips_mamba_spec_state()
|
||||
)
|
||||
if has_spec_dec:
|
||||
assert get_spec().speculative_num_draft_tokens is not None
|
||||
assert get_schedule().max_running_requests is not None
|
||||
@@ -2496,8 +2510,8 @@ class KVCacheConfigurator:
|
||||
)
|
||||
# Reserve intermediate memory based on capped max_num_reqs (+1: the
|
||||
# pool's padding slot, see memory_pool.py). Skipped under replayssm
|
||||
# (no intermediate_ssm allocated).
|
||||
if has_spec_dec and not replayssm_active:
|
||||
# and on PD prefill (no intermediate_ssm allocated).
|
||||
if reserves_spec_state:
|
||||
ratio = self._calculate_mamba_ratio()
|
||||
capped_reqs = min(
|
||||
get_schedule().max_running_requests // self.attn_dp_size,
|
||||
@@ -2520,8 +2534,8 @@ class KVCacheConfigurator:
|
||||
// self.attn_dp_size,
|
||||
)
|
||||
# Reserve intermediate memory based on capped max_num_reqs (+1: the
|
||||
# pool's padding slot). Skipped under replayssm.
|
||||
if has_spec_dec and not replayssm_active:
|
||||
# pool's padding slot). Skipped under replayssm and on PD prefill.
|
||||
if reserves_spec_state:
|
||||
intermediate_size = (
|
||||
stage_per_req
|
||||
* (get_schedule().max_mamba_cache_size + 1)
|
||||
@@ -2542,7 +2556,7 @@ class KVCacheConfigurator:
|
||||
)
|
||||
mamba_budget_bytes = mamba_budget * (1 << 30)
|
||||
|
||||
if has_spec_dec and not replayssm_active:
|
||||
if reserves_spec_state:
|
||||
ratio = self._calculate_mamba_ratio()
|
||||
D = get_spec().speculative_num_draft_tokens
|
||||
# Joint solve: main_state + intermediate = mamba_budget
|
||||
|
||||
Reference in New Issue
Block a user