[unified memory pool] Prefer SWA reclaim quotas with a HiCache storage backend or on PD decode (#43919)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
ZYHowell
parent
7e3da1b784
commit
01f3ed18a5
@@ -39,6 +39,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
|
||||
install_move_gate,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
|
||||
from sglang.srt.runtime_context import get_disagg, get_memory
|
||||
from sglang.srt.utils.common import get_num_new_pages
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -908,12 +909,14 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
full_evictable_tokens: int = 0,
|
||||
swa_evictable_tokens: int = 0,
|
||||
empty_pool: bool = False,
|
||||
prefer_swa: bool = False,
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Return cumulative FULL/SWA eviction targets, or None if impossible.
|
||||
|
||||
The tree evicts FULL before SWA. Minimize required SWA reclaim with all
|
||||
evictable FULL available, then trim excess FULL reclaim. FULL eviction's
|
||||
actual SWA cascade is counted by the tree's shared eviction tracker.
|
||||
With ``prefer_swa``, minimize FULL reclaim first and trim SWA instead.
|
||||
"""
|
||||
if full_tokens < 0 or swa_tokens < 0:
|
||||
return None
|
||||
@@ -962,12 +965,20 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
low = mid + 1
|
||||
return low
|
||||
|
||||
swa_reclaim_pages = first_fit(
|
||||
max_swa_pages, lambda value: fits(max_full_pages, value)
|
||||
)
|
||||
full_reclaim_pages = first_fit(
|
||||
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
|
||||
)
|
||||
if prefer_swa:
|
||||
full_reclaim_pages = first_fit(
|
||||
max_full_pages, lambda value: fits(value, max_swa_pages)
|
||||
)
|
||||
swa_reclaim_pages = first_fit(
|
||||
max_swa_pages, lambda value: fits(full_reclaim_pages, value)
|
||||
)
|
||||
else:
|
||||
swa_reclaim_pages = first_fit(
|
||||
max_swa_pages, lambda value: fits(max_full_pages, value)
|
||||
)
|
||||
full_reclaim_pages = first_fit(
|
||||
max_full_pages, lambda value: fits(value, swa_reclaim_pages)
|
||||
)
|
||||
return (
|
||||
full_reclaim_pages * page_size,
|
||||
swa_reclaim_pages * page_size,
|
||||
@@ -1174,12 +1185,19 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
|
||||
if tree_cache is None or not tree_cache.supports_prefix_sharing():
|
||||
return
|
||||
# Read per call: storage attach/detach and PD role switches move both leaves.
|
||||
memory = get_memory()
|
||||
prefer_swa = (
|
||||
memory.enable_hierarchical_cache
|
||||
and memory.hicache_storage_backend is not None
|
||||
) or get_disagg().disaggregation_mode == "decode"
|
||||
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
|
||||
reclaim_plan = self.reclaim_plan(
|
||||
num_tokens,
|
||||
required_swa,
|
||||
full_evictable_tokens=tree_cache.full_evictable_size(),
|
||||
swa_evictable_tokens=tree_cache.swa_evictable_size(),
|
||||
prefer_swa=prefer_swa,
|
||||
)
|
||||
if reclaim_plan is None:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user