diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index 320db074ed..1a390fce78 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -484,11 +484,8 @@ class TreeCacheNamespace(SimpleNamespace): def supports_mamba(self) -> bool: return False - def is_chunk_cache(self) -> bool: - return False - - def is_tree_cache(self) -> bool: - return not self.is_chunk_cache() + def supports_prefix_sharing(self) -> bool: + return True def evict(self, params: EvictParams): pass diff --git a/python/sglang/srt/hardware_backend/npu/dsv4/dsv4_allocator.py b/python/sglang/srt/hardware_backend/npu/dsv4/dsv4_allocator.py index 52196c3052..3e89ca1f92 100644 --- a/python/sglang/srt/hardware_backend/npu/dsv4/dsv4_allocator.py +++ b/python/sglang/srt/hardware_backend/npu/dsv4/dsv4_allocator.py @@ -295,7 +295,7 @@ class DSV4NPUTokenToKVPoolAllocator(SWATokenToKVPoolAllocator): if available_pages() >= num_pages: return True - if tree_cache is None or tree_cache.is_chunk_cache(): + if tree_cache is None or not tree_cache.supports_prefix_sharing(): return False while available_pages() < num_pages: diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 3296b3af29..532695d4e3 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -3962,7 +3962,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): req.last_node, req.lock_receipt ) req.swa_prefix_lock_released = True - elif self.forward_mode.is_extend() and self.tree_cache.is_chunk_cache(): + elif ( + self.forward_mode.is_extend() + and not self.tree_cache.supports_prefix_sharing() + ): pre_len = self.prefix_lens[idx] if self.enable_overlap: # In chunked prefill case, when the second extend batch is scheduling, the first extend batch is still running, so we cannot evict swa tokens diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 5c8dbd76b9..b638db0c6e 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2719,7 +2719,7 @@ class Scheduler( if ( get_exec().moe.elastic_ep_backend is None or self.disable_radix_cache - or not self.tree_cache.is_tree_cache() + or not self.tree_cache.supports_prefix_sharing() ): return diff --git a/python/sglang/srt/managers/scheduler_components/invariant_checker.py b/python/sglang/srt/managers/scheduler_components/invariant_checker.py index 9c902651c8..0001521e11 100644 --- a/python/sglang/srt/managers/scheduler_components/invariant_checker.py +++ b/python/sglang/srt/managers/scheduler_components/invariant_checker.py @@ -524,7 +524,7 @@ class SchedulerInvariantChecker: if not envs.SGLANG_ENABLE_TREE_CACHE_SANITY_CHECK.get(): return if ( - self.tree_cache.is_tree_cache() + self.tree_cache.supports_prefix_sharing() and (self.is_hybrid_swa and self.tree_cache.supports_swa()) or (self.is_hybrid_ssm and self.tree_cache.supports_mamba()) ): diff --git a/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py b/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py index f061506f11..3c6fc964c1 100644 --- a/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py +++ b/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py @@ -244,7 +244,8 @@ class SchedulerPoolStatsObserver: def _get_mamba_token_info(self): is_mamba_radix_cache = ( - self.tree_cache.supports_mamba() and self.tree_cache.is_tree_cache() + self.tree_cache.supports_mamba() + and self.tree_cache.supports_prefix_sharing() ) full_available_size = self.token_to_kv_pool_allocator.available_size() full_evictable_size = ( diff --git a/python/sglang/srt/mem_cache/allocator/base.py b/python/sglang/srt/mem_cache/allocator/base.py index 870013ed03..898973ee89 100644 --- a/python/sglang/srt/mem_cache/allocator/base.py +++ b/python/sglang/srt/mem_cache/allocator/base.py @@ -141,7 +141,7 @@ class BaseTokenToKVPoolAllocator(abc.ABC): from sglang.srt.mem_cache.base_prefix_cache import EvictParams from sglang.srt.mem_cache.common import _evict_until_allocatable - if tree_cache is None or tree_cache.is_chunk_cache(): + if tree_cache is None or not tree_cache.supports_prefix_sharing(): return shortfall = num_tokens - self.available_size() if shortfall > 0: diff --git a/python/sglang/srt/mem_cache/allocator/swa.py b/python/sglang/srt/mem_cache/allocator/swa.py index cee82b1236..aacaf0f743 100644 --- a/python/sglang/srt/mem_cache/allocator/swa.py +++ b/python/sglang/srt/mem_cache/allocator/swa.py @@ -199,7 +199,7 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> None: from sglang.srt.mem_cache.base_prefix_cache import EvictParams - if tree_cache is None or tree_cache.is_chunk_cache(): + if tree_cache is None or not tree_cache.supports_prefix_sharing(): return full_shortfall = max(0, num_tokens - self.full_available_size()) swa_shortfall = max(0, num_tokens - self.swa_available_size()) diff --git a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py index c94f9c56de..e7747f4988 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py +++ b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py @@ -1191,7 +1191,7 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): ) -> bool | None: from sglang.srt.mem_cache.base_prefix_cache import EvictParams - if tree_cache is None or tree_cache.is_chunk_cache(): + if tree_cache is None or not tree_cache.supports_prefix_sharing(): return required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens reclaim_plan = self.reclaim_plan( diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 206f557d3c..eff3b57bda 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -307,8 +307,8 @@ class MatchResult(NamedTuple): def zero_match_result( tree_cache, match_result: MatchResult, extra_key: Optional[str] = None ) -> MatchResult: - if tree_cache.is_chunk_cache(): - # Chunk caches' match_prefix already returns a miss; no root_node to walk back to. + if not tree_cache.supports_prefix_sharing(): + # match_prefix already returns a miss; no root_node to walk back to. return match_result root = tree_cache.root_node_handle(extra_key=extra_key) return match_result._replace( @@ -624,7 +624,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait): page_size=self.page_size, req_to_token_pool=self.req_to_token_pool, token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - is_chunk_cache=self.is_chunk_cache(), + supports_prefix_sharing=self.supports_prefix_sharing(), retain_floor=self.swa_retain_floor(req), eviction_interval=eviction_interval, ) @@ -666,11 +666,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait): def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int: return 0 - def is_chunk_cache(self) -> bool: - return False - - def is_tree_cache(self) -> bool: - return not self.is_chunk_cache() + def supports_prefix_sharing(self) -> bool: + """Whether a request's prefix stays in the cache for other requests to + share, including after the request finishes.""" + return True def available_and_evictable_str(self) -> str: available_size = self.token_to_kv_pool_allocator.available_size() diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py index ae0e50f5d1..fb05edd283 100644 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ b/python/sglang/srt/mem_cache/chunk_cache.py @@ -51,8 +51,8 @@ class ChunkCache(BasePrefixCache): self.protected_size_ = 0 - def is_chunk_cache(self) -> bool: - return True + def supports_prefix_sharing(self) -> bool: + return False # NOTE (csy): this is to determine if a cache has prefix matching feature. # Chunk cache always return True to indicate no prefix matching. diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index c6d048b974..8a8fd5d360 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -60,7 +60,7 @@ def free_swa_out_of_window_slots( page_size: int, req_to_token_pool: ReqToTokenPool, token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator, - is_chunk_cache: bool = False, + supports_prefix_sharing: bool = True, retain_floor: int | None = None, component_type: ComponentType = ComponentType.SWA, free_segment: Callable[..., None] | None = None, @@ -85,20 +85,20 @@ def free_swa_out_of_window_slots( evicted_seqlen = max(evicted_seqlen, dead_lo) req.kv.set_evicted_seqlen(component_type, evicted_seqlen) - if is_chunk_cache: - # Chunk cache builds no radix tree, so no tombstone-leaf concern; evict + if not supports_prefix_sharing: + # Nothing is inserted into a tree, so no tombstone-leaf concern; evict # up to the window boundary (the trailing floor keeps it page-aligned). evict_threshold = pre_len - sliding_window_size else: - # Radix cache: keep max(window, page). The trailing floor page-aligns the + # Prefix-sharing cache: keep max(window, page). The trailing floor page-aligns the # frontier, and subtracting at least one page keeps it below the insert # boundary (page_floor(seq_len)) so the last leaf is never all-tombstone. # No extra page margin is needed. evict_threshold = pre_len - max(sliding_window_size, page_size) - if retain_floor is not None and not is_chunk_cache: + if retain_floor is not None and supports_prefix_sharing: # The caller owns where the floor is (see BasePrefixCache.swa_retain_floor); - # this only promises not to free past it. Chunk cache has no tree, so a - # retained checkpoint could never be matched and holding it is pure cost. + # this only promises not to free past it. Without prefix sharing a retained + # checkpoint could never be matched, so holding it is pure cost. evict_threshold = min(evict_threshold, retain_floor) new_evicted_seqlen = max(evicted_seqlen, evict_threshold) @@ -184,7 +184,7 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None: def evict_from_tree_cache( tree_cache: BasePrefixCache | None, num_tokens: int ) -> bool | None: - if tree_cache is not None and not tree_cache.is_chunk_cache(): + if tree_cache is not None and tree_cache.supports_prefix_sharing(): return tree_cache.token_to_kv_pool_allocator.evict_to_free_tokens( tree_cache, num_tokens ) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 9d4e6a1807..7cc6549d3b 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -566,8 +566,8 @@ class UnifiedRadixCache(BasePrefixCache): def supports_fast_match_prefix(self) -> bool: return self.tree_core.supports_fast_match_prefix() - def is_chunk_cache(self) -> bool: - return self.disable + def supports_prefix_sharing(self) -> bool: + return not self.disable @rank_consensus( same_params=["len(params.key)"], diff --git a/test/registered/unit/managers/scheduler_components/test_invariant_checker.py b/test/registered/unit/managers/scheduler_components/test_invariant_checker.py index 0981b8f3e3..ff61894a76 100644 --- a/test/registered/unit/managers/scheduler_components/test_invariant_checker.py +++ b/test/registered/unit/managers/scheduler_components/test_invariant_checker.py @@ -41,7 +41,7 @@ class TestCheckTreeCacheGate(CustomTestCase): def _make_checker(self): tree_cache = MagicMock() - tree_cache.is_tree_cache.return_value = True + tree_cache.supports_prefix_sharing.return_value = True tree_cache.supports_swa.return_value = True return SchedulerInvariantChecker( is_hybrid_swa=True, diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 0e475d5ea6..3a82090019 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -149,7 +149,7 @@ class TestPrefillAdder(CustomTestCase): def create_shared_adder(self, *, num_mixed_decode_tokens=0): self.mock_tree_cache.supports_mamba.return_value = False self.mock_tree_cache.sliding_window_size = 8 - self.mock_tree_cache.is_tree_cache.return_value = False + self.mock_tree_cache.supports_prefix_sharing.return_value = False allocator = init_unified_swa_pools( device="cpu", kv_cache_dtype=torch.float16, @@ -197,7 +197,7 @@ class TestPrefillAdder(CustomTestCase): override.install() self.addCleanup(override.restore) self.mock_tree_cache.supports_mamba.return_value = False - self.mock_tree_cache.is_tree_cache.return_value = False + self.mock_tree_cache.supports_prefix_sharing.return_value = False self.mock_token_allocator.available_size.return_value = 32768 return self.create_adder( self.create_running_batch(), page_size=256, rem_chunk_tokens=chunk_tokens @@ -286,7 +286,7 @@ class TestPrefillAdder(CustomTestCase): # A Mamba checkpoint only lands on a page-aligned chunk end, so an # off-grid chunk leaves the rest of the prompt uncacheable. self.mock_token_allocator.available_size.return_value = 32768 - self.mock_tree_cache.is_tree_cache.return_value = False + self.mock_tree_cache.supports_prefix_sharing.return_value = False for supports_mamba, expected in ((False, 100), (True, 64)): with ( self.subTest(supports_mamba=supports_mamba), @@ -825,7 +825,7 @@ class TestPrefillAdder(CustomTestCase): **kwargs, ): self.mock_tree_cache.supports_mamba.return_value = False - self.mock_tree_cache.is_tree_cache.return_value = False + self.mock_tree_cache.supports_prefix_sharing.return_value = False if shard_spec is None: shard_spec = PageShardSpec( shard_rank=0, @@ -1155,7 +1155,7 @@ class TestPrefillAdder(CustomTestCase): self.mock_token_allocator.full_available_size.return_value = 100_000 self.mock_token_allocator.available_size.return_value = 100_000 self.mock_tree_cache.sliding_window_size = WINDOW - self.mock_tree_cache.is_tree_cache.return_value = False + self.mock_tree_cache.supports_prefix_sharing.return_value = False adder = self.create_adder(self.create_running_batch(), page_size=PAGE) adder.is_hybrid_swa = True adder.memory_budget = SWAPrefillBudget( @@ -1204,7 +1204,7 @@ class TestPrefillAdder(CustomTestCase): self.mock_token_allocator.full_available_size.return_value = 100_000 self.mock_token_allocator.available_size.return_value = 100_000 self.mock_tree_cache.sliding_window_size = WINDOW - self.mock_tree_cache.is_tree_cache.return_value = False + self.mock_tree_cache.supports_prefix_sharing.return_value = False def run(delivered: int, remaining_after_load: int = 100_000): self.mock_token_allocator.full_available_size.return_value = 100_000 diff --git a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py index 7af5cea53d..8d602dc24d 100644 --- a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py +++ b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py @@ -54,8 +54,8 @@ class _FakeTreeCache: self.page_size = allocator.page_size self.token_to_kv_pool_allocator = allocator - def is_chunk_cache(self): - return True + def supports_prefix_sharing(self): + return False def _make_req(rid, prefix, block_size, *, req_pool_idx=None, reuse=False): diff --git a/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py b/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py index b151a166a0..14163cd2cf 100644 --- a/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py +++ b/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py @@ -80,7 +80,7 @@ class TestDSV4DecodeCapacity(CustomTestCase): ), ) tree_cache = SimpleNamespace( - is_chunk_cache=lambda: False, + supports_prefix_sharing=lambda: True, evict=MagicMock(side_effect=evict), ) diff --git a/test/registered/unit/mem_cache/test_prefill_memory_budget.py b/test/registered/unit/mem_cache/test_prefill_memory_budget.py index 44f7c919ed..c686c377ce 100644 --- a/test/registered/unit/mem_cache/test_prefill_memory_budget.py +++ b/test/registered/unit/mem_cache/test_prefill_memory_budget.py @@ -66,7 +66,7 @@ def _cache(): sliding_window_size=8, full_evictable_size=lambda: 0, swa_evictable_size=lambda: 0, - is_chunk_cache=lambda: False, + supports_prefix_sharing=lambda: True, ) @@ -186,7 +186,7 @@ class TestSharedPrefillAdmission(unittest.TestCase): disable=True, full_evictable_size=lambda: 0, swa_evictable_size=lambda: 0, - is_chunk_cache=lambda: True, + supports_prefix_sharing=lambda: False, supports_mamba=lambda: False, ) adder = PrefillAdder( diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index 38d670712e..8209214179 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -66,8 +66,8 @@ class TestZeroMatchResult(unittest.TestCase): def test_chunk_cache_is_passthrough(self): class _StubChunkCache: - def is_chunk_cache(self) -> bool: - return True + def supports_prefix_sharing(self) -> bool: + return False original = MatchResult( device_indices=torch.empty((0,), dtype=torch.int64), diff --git a/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py b/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py index 5b99273aa8..da017ca902 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py +++ b/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py @@ -553,7 +553,7 @@ class TestUnifiedRadixAllocationEviction(CustomTestCase): def test_common_helper_uses_allocation_aware_entry_point(self): tree_cache = MagicMock() - tree_cache.is_chunk_cache.return_value = False + tree_cache.supports_prefix_sharing.return_value = True from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator allocator = object.__new__(TokenToKVPoolAllocator)