[mem_cache] Replace is_chunk_cache / is_tree_cache with supports_prefix_sharing (#42362)
This commit is contained in:
@@ -295,7 +295,7 @@ class DSV4NPUTokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
|
||||
if available_pages() >= num_pages:
|
||||
return True
|
||||
if tree_cache is None or tree_cache.is_chunk_cache():
|
||||
if tree_cache is None or not tree_cache.supports_prefix_sharing():
|
||||
return False
|
||||
|
||||
while available_pages() < num_pages:
|
||||
|
||||
@@ -3962,7 +3962,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
req.last_node, req.lock_receipt
|
||||
)
|
||||
req.swa_prefix_lock_released = True
|
||||
elif self.forward_mode.is_extend() and self.tree_cache.is_chunk_cache():
|
||||
elif (
|
||||
self.forward_mode.is_extend()
|
||||
and not self.tree_cache.supports_prefix_sharing()
|
||||
):
|
||||
pre_len = self.prefix_lens[idx]
|
||||
if self.enable_overlap:
|
||||
# In chunked prefill case, when the second extend batch is scheduling, the first extend batch is still running, so we cannot evict swa tokens
|
||||
|
||||
@@ -2719,7 +2719,7 @@ class Scheduler(
|
||||
if (
|
||||
get_exec().moe.elastic_ep_backend is None
|
||||
or self.disable_radix_cache
|
||||
or not self.tree_cache.is_tree_cache()
|
||||
or not self.tree_cache.supports_prefix_sharing()
|
||||
):
|
||||
return
|
||||
|
||||
|
||||
@@ -524,7 +524,7 @@ class SchedulerInvariantChecker:
|
||||
if not envs.SGLANG_ENABLE_TREE_CACHE_SANITY_CHECK.get():
|
||||
return
|
||||
if (
|
||||
self.tree_cache.is_tree_cache()
|
||||
self.tree_cache.supports_prefix_sharing()
|
||||
and (self.is_hybrid_swa and self.tree_cache.supports_swa())
|
||||
or (self.is_hybrid_ssm and self.tree_cache.supports_mamba())
|
||||
):
|
||||
|
||||
@@ -244,7 +244,8 @@ class SchedulerPoolStatsObserver:
|
||||
|
||||
def _get_mamba_token_info(self):
|
||||
is_mamba_radix_cache = (
|
||||
self.tree_cache.supports_mamba() and self.tree_cache.is_tree_cache()
|
||||
self.tree_cache.supports_mamba()
|
||||
and self.tree_cache.supports_prefix_sharing()
|
||||
)
|
||||
full_available_size = self.token_to_kv_pool_allocator.available_size()
|
||||
full_evictable_size = (
|
||||
|
||||
@@ -141,7 +141,7 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
|
||||
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
|
||||
from sglang.srt.mem_cache.common import _evict_until_allocatable
|
||||
|
||||
if tree_cache is None or tree_cache.is_chunk_cache():
|
||||
if tree_cache is None or not tree_cache.supports_prefix_sharing():
|
||||
return
|
||||
shortfall = num_tokens - self.available_size()
|
||||
if shortfall > 0:
|
||||
|
||||
@@ -199,7 +199,7 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> None:
|
||||
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
|
||||
|
||||
if tree_cache is None or tree_cache.is_chunk_cache():
|
||||
if tree_cache is None or not tree_cache.supports_prefix_sharing():
|
||||
return
|
||||
full_shortfall = max(0, num_tokens - self.full_available_size())
|
||||
swa_shortfall = max(0, num_tokens - self.swa_available_size())
|
||||
|
||||
@@ -1191,7 +1191,7 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
) -> bool | None:
|
||||
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
|
||||
|
||||
if tree_cache is None or tree_cache.is_chunk_cache():
|
||||
if tree_cache is None or not tree_cache.supports_prefix_sharing():
|
||||
return
|
||||
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
|
||||
reclaim_plan = self.reclaim_plan(
|
||||
|
||||
@@ -307,8 +307,8 @@ class MatchResult(NamedTuple):
|
||||
def zero_match_result(
|
||||
tree_cache, match_result: MatchResult, extra_key: Optional[str] = None
|
||||
) -> MatchResult:
|
||||
if tree_cache.is_chunk_cache():
|
||||
# Chunk caches' match_prefix already returns a miss; no root_node to walk back to.
|
||||
if not tree_cache.supports_prefix_sharing():
|
||||
# match_prefix already returns a miss; no root_node to walk back to.
|
||||
return match_result
|
||||
root = tree_cache.root_node_handle(extra_key=extra_key)
|
||||
return match_result._replace(
|
||||
@@ -624,7 +624,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
page_size=self.page_size,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
is_chunk_cache=self.is_chunk_cache(),
|
||||
supports_prefix_sharing=self.supports_prefix_sharing(),
|
||||
retain_floor=self.swa_retain_floor(req),
|
||||
eviction_interval=eviction_interval,
|
||||
)
|
||||
@@ -666,11 +666,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return 0
|
||||
|
||||
def is_chunk_cache(self) -> bool:
|
||||
return False
|
||||
|
||||
def is_tree_cache(self) -> bool:
|
||||
return not self.is_chunk_cache()
|
||||
def supports_prefix_sharing(self) -> bool:
|
||||
"""Whether a request's prefix stays in the cache for other requests to
|
||||
share, including after the request finishes."""
|
||||
return True
|
||||
|
||||
def available_and_evictable_str(self) -> str:
|
||||
available_size = self.token_to_kv_pool_allocator.available_size()
|
||||
|
||||
@@ -51,8 +51,8 @@ class ChunkCache(BasePrefixCache):
|
||||
|
||||
self.protected_size_ = 0
|
||||
|
||||
def is_chunk_cache(self) -> bool:
|
||||
return True
|
||||
def supports_prefix_sharing(self) -> bool:
|
||||
return False
|
||||
|
||||
# NOTE (csy): this is to determine if a cache has prefix matching feature.
|
||||
# Chunk cache always return True to indicate no prefix matching.
|
||||
|
||||
@@ -60,7 +60,7 @@ def free_swa_out_of_window_slots(
|
||||
page_size: int,
|
||||
req_to_token_pool: ReqToTokenPool,
|
||||
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
||||
is_chunk_cache: bool = False,
|
||||
supports_prefix_sharing: bool = True,
|
||||
retain_floor: int | None = None,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
free_segment: Callable[..., None] | None = None,
|
||||
@@ -85,20 +85,20 @@ def free_swa_out_of_window_slots(
|
||||
evicted_seqlen = max(evicted_seqlen, dead_lo)
|
||||
req.kv.set_evicted_seqlen(component_type, evicted_seqlen)
|
||||
|
||||
if is_chunk_cache:
|
||||
# Chunk cache builds no radix tree, so no tombstone-leaf concern; evict
|
||||
if not supports_prefix_sharing:
|
||||
# Nothing is inserted into a tree, so no tombstone-leaf concern; evict
|
||||
# up to the window boundary (the trailing floor keeps it page-aligned).
|
||||
evict_threshold = pre_len - sliding_window_size
|
||||
else:
|
||||
# Radix cache: keep max(window, page). The trailing floor page-aligns the
|
||||
# Prefix-sharing cache: keep max(window, page). The trailing floor page-aligns the
|
||||
# frontier, and subtracting at least one page keeps it below the insert
|
||||
# boundary (page_floor(seq_len)) so the last leaf is never all-tombstone.
|
||||
# No extra page margin is needed.
|
||||
evict_threshold = pre_len - max(sliding_window_size, page_size)
|
||||
if retain_floor is not None and not is_chunk_cache:
|
||||
if retain_floor is not None and supports_prefix_sharing:
|
||||
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
|
||||
# this only promises not to free past it. Chunk cache has no tree, so a
|
||||
# retained checkpoint could never be matched and holding it is pure cost.
|
||||
# this only promises not to free past it. Without prefix sharing a retained
|
||||
# checkpoint could never be matched, so holding it is pure cost.
|
||||
evict_threshold = min(evict_threshold, retain_floor)
|
||||
|
||||
new_evicted_seqlen = max(evicted_seqlen, evict_threshold)
|
||||
@@ -184,7 +184,7 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
def evict_from_tree_cache(
|
||||
tree_cache: BasePrefixCache | None, num_tokens: int
|
||||
) -> bool | None:
|
||||
if tree_cache is not None and not tree_cache.is_chunk_cache():
|
||||
if tree_cache is not None and tree_cache.supports_prefix_sharing():
|
||||
return tree_cache.token_to_kv_pool_allocator.evict_to_free_tokens(
|
||||
tree_cache, num_tokens
|
||||
)
|
||||
|
||||
@@ -566,8 +566,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def supports_fast_match_prefix(self) -> bool:
|
||||
return self.tree_core.supports_fast_match_prefix()
|
||||
|
||||
def is_chunk_cache(self) -> bool:
|
||||
return self.disable
|
||||
def supports_prefix_sharing(self) -> bool:
|
||||
return not self.disable
|
||||
|
||||
@rank_consensus(
|
||||
same_params=["len(params.key)"],
|
||||
|
||||
Reference in New Issue
Block a user