[mem_cache] Replace is_chunk_cache / is_tree_cache with supports_prefix_sharing (#42362)

This commit is contained in:
Liangsheng Yin
2026-10-03 02:07:55 -07:00
committed by GitHub
parent 8293f9af54
commit 122b59152c
20 changed files with 48 additions and 48 deletions
+2 -5
View File
@@ -484,11 +484,8 @@ class TreeCacheNamespace(SimpleNamespace):
def supports_mamba(self) -> bool:
return False
def is_chunk_cache(self) -> bool:
return False
def is_tree_cache(self) -> bool:
return not self.is_chunk_cache()
def supports_prefix_sharing(self) -> bool:
return True
def evict(self, params: EvictParams):
pass
@@ -295,7 +295,7 @@ class DSV4NPUTokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
if available_pages() >= num_pages:
return True
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return False
while available_pages() < num_pages:
+4 -1
View File
@@ -3962,7 +3962,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
req.last_node, req.lock_receipt
)
req.swa_prefix_lock_released = True
elif self.forward_mode.is_extend() and self.tree_cache.is_chunk_cache():
elif (
self.forward_mode.is_extend()
and not self.tree_cache.supports_prefix_sharing()
):
pre_len = self.prefix_lens[idx]
if self.enable_overlap:
# In chunked prefill case, when the second extend batch is scheduling, the first extend batch is still running, so we cannot evict swa tokens
+1 -1
View File
@@ -2719,7 +2719,7 @@ class Scheduler(
if (
get_exec().moe.elastic_ep_backend is None
or self.disable_radix_cache
or not self.tree_cache.is_tree_cache()
or not self.tree_cache.supports_prefix_sharing()
):
return
@@ -524,7 +524,7 @@ class SchedulerInvariantChecker:
if not envs.SGLANG_ENABLE_TREE_CACHE_SANITY_CHECK.get():
return
if (
self.tree_cache.is_tree_cache()
self.tree_cache.supports_prefix_sharing()
and (self.is_hybrid_swa and self.tree_cache.supports_swa())
or (self.is_hybrid_ssm and self.tree_cache.supports_mamba())
):
@@ -244,7 +244,8 @@ class SchedulerPoolStatsObserver:
def _get_mamba_token_info(self):
is_mamba_radix_cache = (
self.tree_cache.supports_mamba() and self.tree_cache.is_tree_cache()
self.tree_cache.supports_mamba()
and self.tree_cache.supports_prefix_sharing()
)
full_available_size = self.token_to_kv_pool_allocator.available_size()
full_evictable_size = (
@@ -141,7 +141,7 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
from sglang.srt.mem_cache.common import _evict_until_allocatable
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
shortfall = num_tokens - self.available_size()
if shortfall > 0:
+1 -1
View File
@@ -199,7 +199,7 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> None:
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
full_shortfall = max(0, num_tokens - self.full_available_size())
swa_shortfall = max(0, num_tokens - self.swa_available_size())
@@ -1191,7 +1191,7 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
) -> bool | None:
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
if tree_cache is None or tree_cache.is_chunk_cache():
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
reclaim_plan = self.reclaim_plan(
@@ -307,8 +307,8 @@ class MatchResult(NamedTuple):
def zero_match_result(
tree_cache, match_result: MatchResult, extra_key: Optional[str] = None
) -> MatchResult:
if tree_cache.is_chunk_cache():
# Chunk caches' match_prefix already returns a miss; no root_node to walk back to.
if not tree_cache.supports_prefix_sharing():
# match_prefix already returns a miss; no root_node to walk back to.
return match_result
root = tree_cache.root_node_handle(extra_key=extra_key)
return match_result._replace(
@@ -624,7 +624,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
page_size=self.page_size,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
is_chunk_cache=self.is_chunk_cache(),
supports_prefix_sharing=self.supports_prefix_sharing(),
retain_floor=self.swa_retain_floor(req),
eviction_interval=eviction_interval,
)
@@ -666,11 +666,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
return 0
def is_chunk_cache(self) -> bool:
return False
def is_tree_cache(self) -> bool:
return not self.is_chunk_cache()
def supports_prefix_sharing(self) -> bool:
"""Whether a request's prefix stays in the cache for other requests to
share, including after the request finishes."""
return True
def available_and_evictable_str(self) -> str:
available_size = self.token_to_kv_pool_allocator.available_size()
+2 -2
View File
@@ -51,8 +51,8 @@ class ChunkCache(BasePrefixCache):
self.protected_size_ = 0
def is_chunk_cache(self) -> bool:
return True
def supports_prefix_sharing(self) -> bool:
return False
# NOTE (csy): this is to determine if a cache has prefix matching feature.
# Chunk cache always return True to indicate no prefix matching.
+8 -8
View File
@@ -60,7 +60,7 @@ def free_swa_out_of_window_slots(
page_size: int,
req_to_token_pool: ReqToTokenPool,
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
is_chunk_cache: bool = False,
supports_prefix_sharing: bool = True,
retain_floor: int | None = None,
component_type: ComponentType = ComponentType.SWA,
free_segment: Callable[..., None] | None = None,
@@ -85,20 +85,20 @@ def free_swa_out_of_window_slots(
evicted_seqlen = max(evicted_seqlen, dead_lo)
req.kv.set_evicted_seqlen(component_type, evicted_seqlen)
if is_chunk_cache:
# Chunk cache builds no radix tree, so no tombstone-leaf concern; evict
if not supports_prefix_sharing:
# Nothing is inserted into a tree, so no tombstone-leaf concern; evict
# up to the window boundary (the trailing floor keeps it page-aligned).
evict_threshold = pre_len - sliding_window_size
else:
# Radix cache: keep max(window, page). The trailing floor page-aligns the
# Prefix-sharing cache: keep max(window, page). The trailing floor page-aligns the
# frontier, and subtracting at least one page keeps it below the insert
# boundary (page_floor(seq_len)) so the last leaf is never all-tombstone.
# No extra page margin is needed.
evict_threshold = pre_len - max(sliding_window_size, page_size)
if retain_floor is not None and not is_chunk_cache:
if retain_floor is not None and supports_prefix_sharing:
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
# this only promises not to free past it. Chunk cache has no tree, so a
# retained checkpoint could never be matched and holding it is pure cost.
# this only promises not to free past it. Without prefix sharing a retained
# checkpoint could never be matched, so holding it is pure cost.
evict_threshold = min(evict_threshold, retain_floor)
new_evicted_seqlen = max(evicted_seqlen, evict_threshold)
@@ -184,7 +184,7 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
def evict_from_tree_cache(
tree_cache: BasePrefixCache | None, num_tokens: int
) -> bool | None:
if tree_cache is not None and not tree_cache.is_chunk_cache():
if tree_cache is not None and tree_cache.supports_prefix_sharing():
return tree_cache.token_to_kv_pool_allocator.evict_to_free_tokens(
tree_cache, num_tokens
)
@@ -566,8 +566,8 @@ class UnifiedRadixCache(BasePrefixCache):
def supports_fast_match_prefix(self) -> bool:
return self.tree_core.supports_fast_match_prefix()
def is_chunk_cache(self) -> bool:
return self.disable
def supports_prefix_sharing(self) -> bool:
return not self.disable
@rank_consensus(
same_params=["len(params.key)"],
@@ -41,7 +41,7 @@ class TestCheckTreeCacheGate(CustomTestCase):
def _make_checker(self):
tree_cache = MagicMock()
tree_cache.is_tree_cache.return_value = True
tree_cache.supports_prefix_sharing.return_value = True
tree_cache.supports_swa.return_value = True
return SchedulerInvariantChecker(
is_hybrid_swa=True,
@@ -149,7 +149,7 @@ class TestPrefillAdder(CustomTestCase):
def create_shared_adder(self, *, num_mixed_decode_tokens=0):
self.mock_tree_cache.supports_mamba.return_value = False
self.mock_tree_cache.sliding_window_size = 8
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_tree_cache.supports_prefix_sharing.return_value = False
allocator = init_unified_swa_pools(
device="cpu",
kv_cache_dtype=torch.float16,
@@ -197,7 +197,7 @@ class TestPrefillAdder(CustomTestCase):
override.install()
self.addCleanup(override.restore)
self.mock_tree_cache.supports_mamba.return_value = False
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_tree_cache.supports_prefix_sharing.return_value = False
self.mock_token_allocator.available_size.return_value = 32768
return self.create_adder(
self.create_running_batch(), page_size=256, rem_chunk_tokens=chunk_tokens
@@ -286,7 +286,7 @@ class TestPrefillAdder(CustomTestCase):
# A Mamba checkpoint only lands on a page-aligned chunk end, so an
# off-grid chunk leaves the rest of the prompt uncacheable.
self.mock_token_allocator.available_size.return_value = 32768
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_tree_cache.supports_prefix_sharing.return_value = False
for supports_mamba, expected in ((False, 100), (True, 64)):
with (
self.subTest(supports_mamba=supports_mamba),
@@ -825,7 +825,7 @@ class TestPrefillAdder(CustomTestCase):
**kwargs,
):
self.mock_tree_cache.supports_mamba.return_value = False
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_tree_cache.supports_prefix_sharing.return_value = False
if shard_spec is None:
shard_spec = PageShardSpec(
shard_rank=0,
@@ -1155,7 +1155,7 @@ class TestPrefillAdder(CustomTestCase):
self.mock_token_allocator.full_available_size.return_value = 100_000
self.mock_token_allocator.available_size.return_value = 100_000
self.mock_tree_cache.sliding_window_size = WINDOW
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_tree_cache.supports_prefix_sharing.return_value = False
adder = self.create_adder(self.create_running_batch(), page_size=PAGE)
adder.is_hybrid_swa = True
adder.memory_budget = SWAPrefillBudget(
@@ -1204,7 +1204,7 @@ class TestPrefillAdder(CustomTestCase):
self.mock_token_allocator.full_available_size.return_value = 100_000
self.mock_token_allocator.available_size.return_value = 100_000
self.mock_tree_cache.sliding_window_size = WINDOW
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_tree_cache.supports_prefix_sharing.return_value = False
def run(delivered: int, remaining_after_load: int = 100_000):
self.mock_token_allocator.full_available_size.return_value = 100_000
@@ -54,8 +54,8 @@ class _FakeTreeCache:
self.page_size = allocator.page_size
self.token_to_kv_pool_allocator = allocator
def is_chunk_cache(self):
return True
def supports_prefix_sharing(self):
return False
def _make_req(rid, prefix, block_size, *, req_pool_idx=None, reuse=False):
@@ -80,7 +80,7 @@ class TestDSV4DecodeCapacity(CustomTestCase):
),
)
tree_cache = SimpleNamespace(
is_chunk_cache=lambda: False,
supports_prefix_sharing=lambda: True,
evict=MagicMock(side_effect=evict),
)
@@ -66,7 +66,7 @@ def _cache():
sliding_window_size=8,
full_evictable_size=lambda: 0,
swa_evictable_size=lambda: 0,
is_chunk_cache=lambda: False,
supports_prefix_sharing=lambda: True,
)
@@ -186,7 +186,7 @@ class TestSharedPrefillAdmission(unittest.TestCase):
disable=True,
full_evictable_size=lambda: 0,
swa_evictable_size=lambda: 0,
is_chunk_cache=lambda: True,
supports_prefix_sharing=lambda: False,
supports_mamba=lambda: False,
)
adder = PrefillAdder(
@@ -66,8 +66,8 @@ class TestZeroMatchResult(unittest.TestCase):
def test_chunk_cache_is_passthrough(self):
class _StubChunkCache:
def is_chunk_cache(self) -> bool:
return True
def supports_prefix_sharing(self) -> bool:
return False
original = MatchResult(
device_indices=torch.empty((0,), dtype=torch.int64),
@@ -553,7 +553,7 @@ class TestUnifiedRadixAllocationEviction(CustomTestCase):
def test_common_helper_uses_allocation_aware_entry_point(self):
tree_cache = MagicMock()
tree_cache.is_chunk_cache.return_value = False
tree_cache.supports_prefix_sharing.return_value = True
from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator
allocator = object.__new__(TokenToKVPoolAllocator)