[mem_cache][PD] Fix decode radix cache ownership and allow EAGLE/EAGLE3 with decode radix cache (#40857)

Co-authored-by: alphabetc1 <2508695655@qq.com>
This commit is contained in:
Charles Chen
2026-10-14 14:19:51 -07:00
committed by GitHub
co-authored by alphabetc1
parent 12dc623102
commit d3dc14eb38
6 changed files with 83 additions and 16 deletions
@@ -84,7 +84,7 @@ class Disagg(msgspec.Struct):
] = None
disaggregation_decode_enable_radix_cache: A[
bool,
"Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, speculative decoding, and --disaggregation-transfer-backend fake.",
"Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, --disaggregation-transfer-backend fake, and speculative algorithms other than EAGLE, EAGLE3, and DSPARK.",
] = False
disaggregation_decode_enable_offload_kvcache: A[
bool, "Enable async KV cache offloading on decode server (PD mode)."
@@ -107,13 +107,6 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
"--disaggregation-decode-enable-radix-cache is incompatible "
"with --disaggregation-transfer-backend fake"
)
if cfg.speculative_algorithm not in (None, "DSPARK"):
raise ValueError(
"--disaggregation-decode-enable-radix-cache is incompatible "
"with speculative decoding "
f"(--speculative-algorithm {cfg.speculative_algorithm})"
)
if attn_dp_enabled_of(resolved_view(server_args)):
logger.warning(
"EXPERIMENTAL: Decode radix cache with DP attention. "
@@ -158,6 +158,19 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
),
)
# Check the final algorithm: NEXTN aliases EAGLE, while Gemma4 drafts promote
# EAGLE/NEXTN to FROZEN_KV_MTP.
if (
cfg.disaggregation_mode == "decode"
and cfg.disaggregation_decode_enable_radix_cache
and cfg.speculative_algorithm not in (None, "DSPARK", "EAGLE", "EAGLE3")
):
raise ValueError(
"--disaggregation-decode-enable-radix-cache is incompatible "
"with speculative decoding "
f"(--speculative-algorithm {cfg.speculative_algorithm})"
)
if cfg.speculative_use_block_verification:
if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DFLASH", "DSPARK"):
raise ValueError(
@@ -1197,19 +1197,17 @@ class UnifiedRadixCache(BasePrefixCache):
)
)
# Match prefix. SWA insertion retains one extra window before the
# page-aligned boundary, so the normal match remains safe to repoint.
# The tree's own walk: a session slot must not answer for the insert.
match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req))
new_last_node = match_result.last_device_node
# With PD decode radix caching, SWA matching can stop short of the inserted path.
# Repoint from that path to protect all tree-owned KV.
new_last_node = result.last_device_node
new_indices = self.path_device_indices(new_last_node)
new_prefix_len = result.prefix_len
assert req.kv.cache_protected_len <= len(new_indices) + self.page_size - 1, (
f"{req.kv.cache_protected_len=}, {len(new_indices)=}, {page_aligned_len=}"
)
assert new_prefix_len <= len(new_indices), (
f"{new_prefix_len=}, {len(new_indices)=}"
)
assert (
new_prefix_len <= len(new_indices) and len(new_indices) == page_aligned_len
), f"{new_prefix_len=}, {len(new_indices)=}, {page_aligned_len=}"
self.req_to_token_pool.write(
(req.kv.req_pool_idx, slice(req.kv.cache_protected_len, len(new_indices))),
new_indices[req.kv.cache_protected_len :],