[unified memory pool] Prefer SWA reclaim quotas with a HiCache storage backend or on PD decode (#43919)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-13 11:58:24 -07:00
committed by GitHub
co-authored by Hanming Lu ZYHowell
parent 7e3da1b784
commit 01f3ed18a5
2 changed files with 80 additions and 7 deletions
@@ -39,6 +39,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
install_move_gate,
)
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
from sglang.srt.runtime_context import get_disagg, get_memory
from sglang.srt.utils.common import get_num_new_pages
logger = logging.getLogger(__name__)
@@ -908,12 +909,14 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
full_evictable_tokens: int = 0,
swa_evictable_tokens: int = 0,
empty_pool: bool = False,
prefer_swa: bool = False,
) -> Optional[Tuple[int, int]]:
"""Return cumulative FULL/SWA eviction targets, or None if impossible.
The tree evicts FULL before SWA. Minimize required SWA reclaim with all
evictable FULL available, then trim excess FULL reclaim. FULL eviction's
actual SWA cascade is counted by the tree's shared eviction tracker.
With ``prefer_swa``, minimize FULL reclaim first and trim SWA instead.
"""
if full_tokens < 0 or swa_tokens < 0:
return None
@@ -962,12 +965,20 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
low = mid + 1
return low
swa_reclaim_pages = first_fit(
max_swa_pages, lambda value: fits(max_full_pages, value)
)
full_reclaim_pages = first_fit(
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
)
if prefer_swa:
full_reclaim_pages = first_fit(
max_full_pages, lambda value: fits(value, max_swa_pages)
)
swa_reclaim_pages = first_fit(
max_swa_pages, lambda value: fits(full_reclaim_pages, value)
)
else:
swa_reclaim_pages = first_fit(
max_swa_pages, lambda value: fits(max_full_pages, value)
)
full_reclaim_pages = first_fit(
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
)
return (
full_reclaim_pages * page_size,
swa_reclaim_pages * page_size,
@@ -1174,12 +1185,19 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
# Read per call: storage attach/detach and PD role switches move both leaves.
memory = get_memory()
prefer_swa = (
memory.enable_hierarchical_cache
and memory.hicache_storage_backend is not None
) or get_disagg().disaggregation_mode == "decode"
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
reclaim_plan = self.reclaim_plan(
num_tokens,
required_swa,
full_evictable_tokens=tree_cache.full_evictable_size(),
swa_evictable_tokens=tree_cache.swa_evictable_size(),
prefer_swa=prefer_swa,
)
if reclaim_plan is None:
return
@@ -51,7 +51,13 @@ from sglang.srt.mem_cache.unified_memory_pool import (
UnifiedMambaSlotAllocator,
)
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.runtime_context import get_parallel, publish, reset_context
from sglang.srt.runtime_context import (
get_disagg,
get_memory,
get_parallel,
publish,
reset_context,
)
from sglang.srt.server_args import ServerArgs
_DEV = "cpu"
@@ -686,6 +692,55 @@ class TestUnifiedSWATokenToKVPoolAllocator(unittest.TestCase):
),
feasible[0] if feasible else None,
)
# ``prefer_swa`` flips the lexicographic order:
# least FULL reclaim first, then least SWA.
self.assertEqual(
allocator.reclaim_plan(
full_pages * page_size,
swa_pages * page_size,
full_evictable_tokens=full_evictable,
swa_evictable_tokens=swa_evictable,
prefer_swa=True,
),
min(feasible) if feasible else None,
)
def test_evict_to_free_tokens_prefers_swa_reclaim_on_storage_or_decode(self):
"""The default quota order spends FULL prefix to keep the sliding
window. With a HiCache storage backend attached, or while serving as
the decode side of a PD pair, the FULL prefix is the one worth keeping,
so the allocator asks the planner for SWA-first quotas. Hierarchical
cache alone (no storage backend) keeps the default. Pins the planner
argument per configuration, so a predicate that degrades to one branch
is caught."""
_, allocator, _ = self._build(page_size=4)
tree_cache = MagicMock()
tree_cache.supports_prefix_sharing.return_value = True
tree_cache.full_evictable_size.return_value = 8
tree_cache.swa_evictable_size.return_value = 8
allocator.reclaim_plan = MagicMock(return_value=(0, 0))
cases = (
({}, None, False),
({"enable_hierarchical_cache": True}, None, False),
(
{"enable_hierarchical_cache": True, "hicache_storage_backend": "file"},
None,
True,
),
({}, "decode", True),
)
for memory, role, prefer_swa in cases:
with self.subTest(memory=memory, role=role):
with contextlib.ExitStack() as stack:
stack.enter_context(get_memory().override(**memory))
if role is not None:
stack.enter_context(
get_disagg().override(disaggregation_mode=role)
)
allocator.evict_to_free_tokens(tree_cache, 4)
self.assertEqual(
allocator.reclaim_plan.call_args.kwargs["prefer_swa"], prefer_swa
)
def test_restore_swa_without_allocating_more_full(self):
_, allocator, _ = self._build(page_size=4)