[mem_cache][PD] Fix decode radix cache ownership and allow EAGLE/EAGLE3 with decode radix cache (#40857)
Co-authored-by: alphabetc1 <2508695655@qq.com>
This commit is contained in:
co-authored by
alphabetc1
parent
12dc623102
commit
d3dc14eb38
@@ -84,7 +84,7 @@ class Disagg(msgspec.Struct):
|
||||
] = None
|
||||
disaggregation_decode_enable_radix_cache: A[
|
||||
bool,
|
||||
"Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, speculative decoding, and --disaggregation-transfer-backend fake.",
|
||||
"Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, --disaggregation-transfer-backend fake, and speculative algorithms other than EAGLE, EAGLE3, and DSPARK.",
|
||||
] = False
|
||||
disaggregation_decode_enable_offload_kvcache: A[
|
||||
bool, "Enable async KV cache offloading on decode server (PD mode)."
|
||||
|
||||
@@ -107,13 +107,6 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
"--disaggregation-decode-enable-radix-cache is incompatible "
|
||||
"with --disaggregation-transfer-backend fake"
|
||||
)
|
||||
if cfg.speculative_algorithm not in (None, "DSPARK"):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache is incompatible "
|
||||
"with speculative decoding "
|
||||
f"(--speculative-algorithm {cfg.speculative_algorithm})"
|
||||
)
|
||||
|
||||
if attn_dp_enabled_of(resolved_view(server_args)):
|
||||
logger.warning(
|
||||
"EXPERIMENTAL: Decode radix cache with DP attention. "
|
||||
|
||||
@@ -158,6 +158,19 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
|
||||
),
|
||||
)
|
||||
|
||||
# Check the final algorithm: NEXTN aliases EAGLE, while Gemma4 drafts promote
|
||||
# EAGLE/NEXTN to FROZEN_KV_MTP.
|
||||
if (
|
||||
cfg.disaggregation_mode == "decode"
|
||||
and cfg.disaggregation_decode_enable_radix_cache
|
||||
and cfg.speculative_algorithm not in (None, "DSPARK", "EAGLE", "EAGLE3")
|
||||
):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache is incompatible "
|
||||
"with speculative decoding "
|
||||
f"(--speculative-algorithm {cfg.speculative_algorithm})"
|
||||
)
|
||||
|
||||
if cfg.speculative_use_block_verification:
|
||||
if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DFLASH", "DSPARK"):
|
||||
raise ValueError(
|
||||
|
||||
@@ -1197,19 +1197,17 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
)
|
||||
|
||||
# Match prefix. SWA insertion retains one extra window before the
|
||||
# page-aligned boundary, so the normal match remains safe to repoint.
|
||||
# The tree's own walk: a session slot must not answer for the insert.
|
||||
match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req))
|
||||
new_last_node = match_result.last_device_node
|
||||
# With PD decode radix caching, SWA matching can stop short of the inserted path.
|
||||
# Repoint from that path to protect all tree-owned KV.
|
||||
new_last_node = result.last_device_node
|
||||
new_indices = self.path_device_indices(new_last_node)
|
||||
new_prefix_len = result.prefix_len
|
||||
assert req.kv.cache_protected_len <= len(new_indices) + self.page_size - 1, (
|
||||
f"{req.kv.cache_protected_len=}, {len(new_indices)=}, {page_aligned_len=}"
|
||||
)
|
||||
assert new_prefix_len <= len(new_indices), (
|
||||
f"{new_prefix_len=}, {len(new_indices)=}"
|
||||
)
|
||||
assert (
|
||||
new_prefix_len <= len(new_indices) and len(new_indices) == page_aligned_len
|
||||
), f"{new_prefix_len=}, {len(new_indices)=}, {page_aligned_len=}"
|
||||
self.req_to_token_pool.write(
|
||||
(req.kv.req_pool_idx, slice(req.kv.cache_protected_len, len(new_indices))),
|
||||
new_indices[req.kv.cache_protected_len :],
|
||||
|
||||
Reference in New Issue
Block a user