[unified memory pool] Prefer SWA reclaim quotas with a HiCache storage backend or on PD decode (#43919)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-13 11:58:24 -07:00
committed by GitHub
co-authored by Hanming Lu ZYHowell
parent 7e3da1b784
commit 01f3ed18a5
2 changed files with 80 additions and 7 deletions
@@ -39,6 +39,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
install_move_gate,
)
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
from sglang.srt.runtime_context import get_disagg, get_memory
from sglang.srt.utils.common import get_num_new_pages
logger = logging.getLogger(__name__)
@@ -908,12 +909,14 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
full_evictable_tokens: int = 0,
swa_evictable_tokens: int = 0,
empty_pool: bool = False,
prefer_swa: bool = False,
) -> Optional[Tuple[int, int]]:
"""Return cumulative FULL/SWA eviction targets, or None if impossible.
The tree evicts FULL before SWA. Minimize required SWA reclaim with all
evictable FULL available, then trim excess FULL reclaim. FULL eviction's
actual SWA cascade is counted by the tree's shared eviction tracker.
With ``prefer_swa``, minimize FULL reclaim first and trim SWA instead.
"""
if full_tokens < 0 or swa_tokens < 0:
return None
@@ -962,12 +965,20 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
low = mid + 1
return low
swa_reclaim_pages = first_fit(
max_swa_pages, lambda value: fits(max_full_pages, value)
)
full_reclaim_pages = first_fit(
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
)
if prefer_swa:
full_reclaim_pages = first_fit(
max_full_pages, lambda value: fits(value, max_swa_pages)
)
swa_reclaim_pages = first_fit(
max_swa_pages, lambda value: fits(full_reclaim_pages, value)
)
else:
swa_reclaim_pages = first_fit(
max_swa_pages, lambda value: fits(max_full_pages, value)
)
full_reclaim_pages = first_fit(
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
)
return (
full_reclaim_pages * page_size,
swa_reclaim_pages * page_size,
@@ -1174,12 +1185,19 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
# Read per call: storage attach/detach and PD role switches move both leaves.
memory = get_memory()
prefer_swa = (
memory.enable_hierarchical_cache
and memory.hicache_storage_backend is not None
) or get_disagg().disaggregation_mode == "decode"
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
reclaim_plan = self.reclaim_plan(
num_tokens,
required_swa,
full_evictable_tokens=tree_cache.full_evictable_size(),
swa_evictable_tokens=tree_cache.swa_evictable_size(),
prefer_swa=prefer_swa,
)
if reclaim_plan is None:
return