From 01f3ed18a5f330885cf69a69f56f11f244defc4e Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 13 Oct 2026 11:58:24 -0700 Subject: [PATCH] [unified memory pool] Prefer SWA reclaim quotas with a HiCache storage backend or on PD decode (#43919) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: ZYHowell --- .../mem_cache/allocator/unified_hybrid_swa.py | 30 ++++++++-- .../mem_cache/test_multi_ended_allocator.py | 57 ++++++++++++++++++- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py index 278cbd7bc3..cfae1376bf 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py +++ b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py @@ -39,6 +39,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import ( install_move_gate, ) from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool +from sglang.srt.runtime_context import get_disagg, get_memory from sglang.srt.utils.common import get_num_new_pages logger = logging.getLogger(__name__) @@ -908,12 +909,14 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): full_evictable_tokens: int = 0, swa_evictable_tokens: int = 0, empty_pool: bool = False, + prefer_swa: bool = False, ) -> Optional[Tuple[int, int]]: """Return cumulative FULL/SWA eviction targets, or None if impossible. The tree evicts FULL before SWA. Minimize required SWA reclaim with all evictable FULL available, then trim excess FULL reclaim. FULL eviction's actual SWA cascade is counted by the tree's shared eviction tracker. + With ``prefer_swa``, minimize FULL reclaim first and trim SWA instead. """ if full_tokens < 0 or swa_tokens < 0: return None @@ -962,12 +965,20 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): low = mid + 1 return low - swa_reclaim_pages = first_fit( - max_swa_pages, lambda value: fits(max_full_pages, value) - ) - full_reclaim_pages = first_fit( - max_full_pages, lambda value: fits(value, swa_reclaim_pages) - ) + if prefer_swa: + full_reclaim_pages = first_fit( + max_full_pages, lambda value: fits(value, max_swa_pages) + ) + swa_reclaim_pages = first_fit( + max_swa_pages, lambda value: fits(full_reclaim_pages, value) + ) + else: + swa_reclaim_pages = first_fit( + max_swa_pages, lambda value: fits(max_full_pages, value) + ) + full_reclaim_pages = first_fit( + max_full_pages, lambda value: fits(value, swa_reclaim_pages) + ) return ( full_reclaim_pages * page_size, swa_reclaim_pages * page_size, @@ -1174,12 +1185,19 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): if tree_cache is None or not tree_cache.supports_prefix_sharing(): return + # Read per call: storage attach/detach and PD role switches move both leaves. + memory = get_memory() + prefer_swa = ( + memory.enable_hierarchical_cache + and memory.hicache_storage_backend is not None + ) or get_disagg().disaggregation_mode == "decode" required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens reclaim_plan = self.reclaim_plan( num_tokens, required_swa, full_evictable_tokens=tree_cache.full_evictable_size(), swa_evictable_tokens=tree_cache.swa_evictable_size(), + prefer_swa=prefer_swa, ) if reclaim_plan is None: return diff --git a/test/registered/unit/mem_cache/test_multi_ended_allocator.py b/test/registered/unit/mem_cache/test_multi_ended_allocator.py index 10290eee06..3da7ffc22a 100644 --- a/test/registered/unit/mem_cache/test_multi_ended_allocator.py +++ b/test/registered/unit/mem_cache/test_multi_ended_allocator.py @@ -51,7 +51,13 @@ from sglang.srt.mem_cache.unified_memory_pool import ( UnifiedMambaSlotAllocator, ) from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache -from sglang.srt.runtime_context import get_parallel, publish, reset_context +from sglang.srt.runtime_context import ( + get_disagg, + get_memory, + get_parallel, + publish, + reset_context, +) from sglang.srt.server_args import ServerArgs _DEV = "cpu" @@ -686,6 +692,55 @@ class TestUnifiedSWATokenToKVPoolAllocator(unittest.TestCase): ), feasible[0] if feasible else None, ) + # ``prefer_swa`` flips the lexicographic order: + # least FULL reclaim first, then least SWA. + self.assertEqual( + allocator.reclaim_plan( + full_pages * page_size, + swa_pages * page_size, + full_evictable_tokens=full_evictable, + swa_evictable_tokens=swa_evictable, + prefer_swa=True, + ), + min(feasible) if feasible else None, + ) + + def test_evict_to_free_tokens_prefers_swa_reclaim_on_storage_or_decode(self): + """The default quota order spends FULL prefix to keep the sliding + window. With a HiCache storage backend attached, or while serving as + the decode side of a PD pair, the FULL prefix is the one worth keeping, + so the allocator asks the planner for SWA-first quotas. Hierarchical + cache alone (no storage backend) keeps the default. Pins the planner + argument per configuration, so a predicate that degrades to one branch + is caught.""" + _, allocator, _ = self._build(page_size=4) + tree_cache = MagicMock() + tree_cache.supports_prefix_sharing.return_value = True + tree_cache.full_evictable_size.return_value = 8 + tree_cache.swa_evictable_size.return_value = 8 + allocator.reclaim_plan = MagicMock(return_value=(0, 0)) + cases = ( + ({}, None, False), + ({"enable_hierarchical_cache": True}, None, False), + ( + {"enable_hierarchical_cache": True, "hicache_storage_backend": "file"}, + None, + True, + ), + ({}, "decode", True), + ) + for memory, role, prefer_swa in cases: + with self.subTest(memory=memory, role=role): + with contextlib.ExitStack() as stack: + stack.enter_context(get_memory().override(**memory)) + if role is not None: + stack.enter_context( + get_disagg().override(disaggregation_mode=role) + ) + allocator.evict_to_free_tokens(tree_cache, 4) + self.assertEqual( + allocator.reclaim_plan.call_args.kwargs["prefer_swa"], prefer_swa + ) def test_restore_swa_without_allocating_more_full(self): _, allocator, _ = self._build(page_size=4)