[unified memory pool] Prefer SWA reclaim quotas with a HiCache storage backend or on PD decode (#43919)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
ZYHowell
parent
7e3da1b784
commit
01f3ed18a5
@@ -39,6 +39,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
|
||||
install_move_gate,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
|
||||
from sglang.srt.runtime_context import get_disagg, get_memory
|
||||
from sglang.srt.utils.common import get_num_new_pages
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -908,12 +909,14 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
full_evictable_tokens: int = 0,
|
||||
swa_evictable_tokens: int = 0,
|
||||
empty_pool: bool = False,
|
||||
prefer_swa: bool = False,
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Return cumulative FULL/SWA eviction targets, or None if impossible.
|
||||
|
||||
The tree evicts FULL before SWA. Minimize required SWA reclaim with all
|
||||
evictable FULL available, then trim excess FULL reclaim. FULL eviction's
|
||||
actual SWA cascade is counted by the tree's shared eviction tracker.
|
||||
With ``prefer_swa``, minimize FULL reclaim first and trim SWA instead.
|
||||
"""
|
||||
if full_tokens < 0 or swa_tokens < 0:
|
||||
return None
|
||||
@@ -962,12 +965,20 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
low = mid + 1
|
||||
return low
|
||||
|
||||
swa_reclaim_pages = first_fit(
|
||||
max_swa_pages, lambda value: fits(max_full_pages, value)
|
||||
)
|
||||
full_reclaim_pages = first_fit(
|
||||
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
|
||||
)
|
||||
if prefer_swa:
|
||||
full_reclaim_pages = first_fit(
|
||||
max_full_pages, lambda value: fits(value, max_swa_pages)
|
||||
)
|
||||
swa_reclaim_pages = first_fit(
|
||||
max_swa_pages, lambda value: fits(full_reclaim_pages, value)
|
||||
)
|
||||
else:
|
||||
swa_reclaim_pages = first_fit(
|
||||
max_swa_pages, lambda value: fits(max_full_pages, value)
|
||||
)
|
||||
full_reclaim_pages = first_fit(
|
||||
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
|
||||
)
|
||||
return (
|
||||
full_reclaim_pages * page_size,
|
||||
swa_reclaim_pages * page_size,
|
||||
@@ -1174,12 +1185,19 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
|
||||
if tree_cache is None or not tree_cache.supports_prefix_sharing():
|
||||
return
|
||||
# Read per call: storage attach/detach and PD role switches move both leaves.
|
||||
memory = get_memory()
|
||||
prefer_swa = (
|
||||
memory.enable_hierarchical_cache
|
||||
and memory.hicache_storage_backend is not None
|
||||
) or get_disagg().disaggregation_mode == "decode"
|
||||
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
|
||||
reclaim_plan = self.reclaim_plan(
|
||||
num_tokens,
|
||||
required_swa,
|
||||
full_evictable_tokens=tree_cache.full_evictable_size(),
|
||||
swa_evictable_tokens=tree_cache.swa_evictable_size(),
|
||||
prefer_swa=prefer_swa,
|
||||
)
|
||||
if reclaim_plan is None:
|
||||
return
|
||||
|
||||
@@ -51,7 +51,13 @@ from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
UnifiedMambaSlotAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.runtime_context import get_parallel, publish, reset_context
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_memory,
|
||||
get_parallel,
|
||||
publish,
|
||||
reset_context,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
_DEV = "cpu"
|
||||
@@ -686,6 +692,55 @@ class TestUnifiedSWATokenToKVPoolAllocator(unittest.TestCase):
|
||||
),
|
||||
feasible[0] if feasible else None,
|
||||
)
|
||||
# ``prefer_swa`` flips the lexicographic order:
|
||||
# least FULL reclaim first, then least SWA.
|
||||
self.assertEqual(
|
||||
allocator.reclaim_plan(
|
||||
full_pages * page_size,
|
||||
swa_pages * page_size,
|
||||
full_evictable_tokens=full_evictable,
|
||||
swa_evictable_tokens=swa_evictable,
|
||||
prefer_swa=True,
|
||||
),
|
||||
min(feasible) if feasible else None,
|
||||
)
|
||||
|
||||
def test_evict_to_free_tokens_prefers_swa_reclaim_on_storage_or_decode(self):
|
||||
"""The default quota order spends FULL prefix to keep the sliding
|
||||
window. With a HiCache storage backend attached, or while serving as
|
||||
the decode side of a PD pair, the FULL prefix is the one worth keeping,
|
||||
so the allocator asks the planner for SWA-first quotas. Hierarchical
|
||||
cache alone (no storage backend) keeps the default. Pins the planner
|
||||
argument per configuration, so a predicate that degrades to one branch
|
||||
is caught."""
|
||||
_, allocator, _ = self._build(page_size=4)
|
||||
tree_cache = MagicMock()
|
||||
tree_cache.supports_prefix_sharing.return_value = True
|
||||
tree_cache.full_evictable_size.return_value = 8
|
||||
tree_cache.swa_evictable_size.return_value = 8
|
||||
allocator.reclaim_plan = MagicMock(return_value=(0, 0))
|
||||
cases = (
|
||||
({}, None, False),
|
||||
({"enable_hierarchical_cache": True}, None, False),
|
||||
(
|
||||
{"enable_hierarchical_cache": True, "hicache_storage_backend": "file"},
|
||||
None,
|
||||
True,
|
||||
),
|
||||
({}, "decode", True),
|
||||
)
|
||||
for memory, role, prefer_swa in cases:
|
||||
with self.subTest(memory=memory, role=role):
|
||||
with contextlib.ExitStack() as stack:
|
||||
stack.enter_context(get_memory().override(**memory))
|
||||
if role is not None:
|
||||
stack.enter_context(
|
||||
get_disagg().override(disaggregation_mode=role)
|
||||
)
|
||||
allocator.evict_to_free_tokens(tree_cache, 4)
|
||||
self.assertEqual(
|
||||
allocator.reclaim_plan.call_args.kwargs["prefer_swa"], prefer_swa
|
||||
)
|
||||
|
||||
def test_restore_swa_without_allocating_more_full(self):
|
||||
_, allocator, _ = self._build(page_size=4)
|
||||
|
||||
Reference in New Issue
Block a user