[mem_cache] Replace is_chunk_cache / is_tree_cache with supports_prefix_sharing (#42362)

This commit is contained in:
Liangsheng Yin
2026-10-03 02:07:55 -07:00
committed by GitHub
parent 8293f9af54
commit 122b59152c
20 changed files with 48 additions and 48 deletions
+2 -5
View File
@@ -484,11 +484,8 @@ class TreeCacheNamespace(SimpleNamespace):
def supports_mamba(self) -> bool:
return False
def is_chunk_cache(self) -> bool:
return False
def is_tree_cache(self) -> bool:
return not self.is_chunk_cache()
def supports_prefix_sharing(self) -> bool:
return True
def evict(self, params: EvictParams):
pass
@@ -295,7 +295,7 @@ class DSV4NPUTokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
if available_pages() >= num_pages:
return True
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return False
while available_pages() < num_pages:
+4 -1
View File
@@ -3962,7 +3962,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
req.last_node, req.lock_receipt
)
req.swa_prefix_lock_released = True
elif self.forward_mode.is_extend() and self.tree_cache.is_chunk_cache():
elif (
self.forward_mode.is_extend()
and not self.tree_cache.supports_prefix_sharing()
):
pre_len = self.prefix_lens[idx]
if self.enable_overlap:
# In chunked prefill case, when the second extend batch is scheduling, the first extend batch is still running, so we cannot evict swa tokens
+1 -1
View File
@@ -2719,7 +2719,7 @@ class Scheduler(
if (
get_exec().moe.elastic_ep_backend is None
or self.disable_radix_cache
or not self.tree_cache.is_tree_cache()
or not self.tree_cache.supports_prefix_sharing()
):
return
@@ -524,7 +524,7 @@ class SchedulerInvariantChecker:
if not envs.SGLANG_ENABLE_TREE_CACHE_SANITY_CHECK.get():
return
if (
self.tree_cache.is_tree_cache()
self.tree_cache.supports_prefix_sharing()
and (self.is_hybrid_swa and self.tree_cache.supports_swa())
or (self.is_hybrid_ssm and self.tree_cache.supports_mamba())
):
@@ -244,7 +244,8 @@ class SchedulerPoolStatsObserver:
def _get_mamba_token_info(self):
is_mamba_radix_cache = (
self.tree_cache.supports_mamba() and self.tree_cache.is_tree_cache()
self.tree_cache.supports_mamba()
and self.tree_cache.supports_prefix_sharing()
)
full_available_size = self.token_to_kv_pool_allocator.available_size()
full_evictable_size = (
@@ -141,7 +141,7 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
from sglang.srt.mem_cache.common import _evict_until_allocatable
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
shortfall = num_tokens - self.available_size()
if shortfall > 0:
+1 -1
View File
@@ -199,7 +199,7 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> None:
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
full_shortfall = max(0, num_tokens - self.full_available_size())
swa_shortfall = max(0, num_tokens - self.swa_available_size())
@@ -1191,7 +1191,7 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
) -> bool | None:
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
reclaim_plan = self.reclaim_plan(
@@ -307,8 +307,8 @@ class MatchResult(NamedTuple):
def zero_match_result(
tree_cache, match_result: MatchResult, extra_key: Optional[str] = None
) -> MatchResult:
if tree_cache.is_chunk_cache():
# Chunk caches' match_prefix already returns a miss; no root_node to walk back to.
if not tree_cache.supports_prefix_sharing():
# match_prefix already returns a miss; no root_node to walk back to.
return match_result
root = tree_cache.root_node_handle(extra_key=extra_key)
return match_result._replace(
@@ -624,7 +624,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
page_size=self.page_size,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
is_chunk_cache=self.is_chunk_cache(),
supports_prefix_sharing=self.supports_prefix_sharing(),
retain_floor=self.swa_retain_floor(req),
eviction_interval=eviction_interval,
)
@@ -666,11 +666,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
return 0
def is_chunk_cache(self) -> bool:
return False
def is_tree_cache(self) -> bool:
return not self.is_chunk_cache()
def supports_prefix_sharing(self) -> bool:
"""Whether a request's prefix stays in the cache for other requests to
share, including after the request finishes."""
return True
def available_and_evictable_str(self) -> str:
available_size = self.token_to_kv_pool_allocator.available_size()
+2 -2
View File
@@ -51,8 +51,8 @@ class ChunkCache(BasePrefixCache):
self.protected_size_ = 0
def is_chunk_cache(self) -> bool:
return True
def supports_prefix_sharing(self) -> bool:
return False
# NOTE (csy): this is to determine if a cache has prefix matching feature.
# Chunk cache always return True to indicate no prefix matching.
+8 -8
View File
@@ -60,7 +60,7 @@ def free_swa_out_of_window_slots(
page_size: int,
req_to_token_pool: ReqToTokenPool,
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
is_chunk_cache: bool = False,
supports_prefix_sharing: bool = True,
retain_floor: int | None = None,
component_type: ComponentType = ComponentType.SWA,
free_segment: Callable[..., None] | None = None,
@@ -85,20 +85,20 @@ def free_swa_out_of_window_slots(
evicted_seqlen = max(evicted_seqlen, dead_lo)
req.kv.set_evicted_seqlen(component_type, evicted_seqlen)
if is_chunk_cache:
# Chunk cache builds no radix tree, so no tombstone-leaf concern; evict
if not supports_prefix_sharing:
# Nothing is inserted into a tree, so no tombstone-leaf concern; evict
# up to the window boundary (the trailing floor keeps it page-aligned).
evict_threshold = pre_len - sliding_window_size
else:
# Radix cache: keep max(window, page). The trailing floor page-aligns the
# Prefix-sharing cache: keep max(window, page). The trailing floor page-aligns the
# frontier, and subtracting at least one page keeps it below the insert
# boundary (page_floor(seq_len)) so the last leaf is never all-tombstone.
# No extra page margin is needed.
evict_threshold = pre_len - max(sliding_window_size, page_size)
if retain_floor is not None and not is_chunk_cache:
if retain_floor is not None and supports_prefix_sharing:
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
# this only promises not to free past it. Chunk cache has no tree, so a
# retained checkpoint could never be matched and holding it is pure cost.
# this only promises not to free past it. Without prefix sharing a retained
# checkpoint could never be matched, so holding it is pure cost.
evict_threshold = min(evict_threshold, retain_floor)
new_evicted_seqlen = max(evicted_seqlen, evict_threshold)
@@ -184,7 +184,7 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
def evict_from_tree_cache(
tree_cache: BasePrefixCache | None, num_tokens: int
) -> bool | None:
if tree_cache is not None and not tree_cache.is_chunk_cache():
if tree_cache is not None and tree_cache.supports_prefix_sharing():
return tree_cache.token_to_kv_pool_allocator.evict_to_free_tokens(
tree_cache, num_tokens
)
@@ -566,8 +566,8 @@ class UnifiedRadixCache(BasePrefixCache):
def supports_fast_match_prefix(self) -> bool:
return self.tree_core.supports_fast_match_prefix()
def is_chunk_cache(self) -> bool:
return self.disable
def supports_prefix_sharing(self) -> bool:
return not self.disable
@rank_consensus(
same_params=["len(params.key)"],