[unified memory pool] Allocate without transfer waits, pin floating SWA reservations, drop the HiCache startup asserts (#43206)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
ZYHowell
parent
a65f4ffba5
commit
f2e9a2bed1
@@ -659,15 +659,6 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
"preserve unified page-envelope indices and compaction lifetimes. "
|
||||
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
|
||||
)
|
||||
if cfg.enable_hierarchical_cache:
|
||||
assert cfg.pp_size == 1, (
|
||||
"--enable-unified-memory with hierarchical cache does not support "
|
||||
"pipeline parallelism (--pp-size > 1)."
|
||||
)
|
||||
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
|
||||
"--enable-unified-memory with hierarchical cache requires lazy "
|
||||
"compaction so pending H2D physical reservations remain stable."
|
||||
)
|
||||
assert not (
|
||||
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
|
||||
and cfg.disaggregation_decode_retraction_backup == "host_pool"
|
||||
|
||||
@@ -1357,7 +1357,8 @@ class PrefillAdder:
|
||||
):
|
||||
full_load_tokens = req.host_hit_length
|
||||
if (
|
||||
isinstance(self.tree_cache, UnifiedRadixCache)
|
||||
self.memory_budget.load_back_needs_full_tokens
|
||||
and isinstance(self.tree_cache, UnifiedRadixCache)
|
||||
and self.tree_cache.buffer_pipeline is None
|
||||
and not (
|
||||
self.tree_cache.linker is not None
|
||||
|
||||
@@ -335,33 +335,6 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
lazy_compaction=self.lazy_compaction,
|
||||
)
|
||||
|
||||
def bind_swa_for_loaded_rows(
|
||||
self, full_token_ids: torch.Tensor
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Bind SWA pages to resident or newly loaded full-attention virtual IDs.
|
||||
|
||||
Bind before translating: unbound pages translate to the padding sink.
|
||||
Return physical IDs, or None if capacity cannot be reclaimed.
|
||||
"""
|
||||
ids = full_token_ids.to(torch.int64)
|
||||
if ids.numel() == 0:
|
||||
return ids
|
||||
ps = self.page_size
|
||||
pages = torch.unique(ids // ps)
|
||||
# Tombstones (-1) and the padding sink (0) both need a physical binding.
|
||||
unbound = pages[self.swa_attn_allocator.virtual_to_physical[pages] <= 0]
|
||||
need = int(unbound.numel()) * ps
|
||||
if need:
|
||||
if need > self.swa_available_size():
|
||||
return None
|
||||
if (
|
||||
need > self.swa_attn_allocator.available_size()
|
||||
and not _relieve_for_alloc(self.swa_attn_allocator, need)
|
||||
):
|
||||
return None
|
||||
self.swa_attn_allocator.alloc_with_virtual(unbound)
|
||||
return self.translate_loc_from_full_to_swa(ids)
|
||||
|
||||
def set_host_transfer_move_gate(self, gate: Callable[[], bool]) -> None:
|
||||
"""Block page relocation while host transfers use resolved device indices."""
|
||||
install_move_gate(
|
||||
|
||||
@@ -953,8 +953,6 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
with record_function("MultiEndedAlloc._alloc_bind_fast_or_slow"):
|
||||
if N == 0:
|
||||
return torch.empty(0, dtype=torch.int64, device=self.device)
|
||||
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
|
||||
self._wait_hicache_transfers()
|
||||
|
||||
# FAST PATH: eager, or lazy with no current holes.
|
||||
if not self.lazy_compaction or self._free_phys_pages.numel() == 0:
|
||||
@@ -2014,6 +2012,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self._pending_hicache_load_pages = 0
|
||||
|
||||
def _wait_hicache_transfers(self) -> None:
|
||||
"""Order the current stream after outstanding HiCache copies. Only moves
|
||||
and frees need this: a copy's source and target are not free pages until
|
||||
its finish event is observed, so allocation never reuses them."""
|
||||
if not self._hicache_transfer_done_events:
|
||||
return
|
||||
current_stream = torch.cuda.current_stream()
|
||||
@@ -2041,26 +2042,23 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
if need_size > self.available_size():
|
||||
if not _relieve_for_alloc(self, need_size):
|
||||
return None
|
||||
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
|
||||
self._wait_hicache_transfers()
|
||||
physical_pages = self.take_physical_pages(need_size // self.page_size)
|
||||
if physical_pages is None:
|
||||
return None
|
||||
if self.lazy_compaction:
|
||||
self._pending_hicache_load_pages += int(physical_pages.shape[0])
|
||||
# Blocks page moves (`moves_blocked`) until the H2D is queued or cancelled.
|
||||
self._pending_hicache_load_pages += int(physical_pages.shape[0])
|
||||
return self._expand_pages_to_tokens(physical_pages)
|
||||
|
||||
def cancel_physical_reservation(self, free_index: torch.Tensor) -> None:
|
||||
"""Roll back a HiCache physical allocation before its H2D is submitted."""
|
||||
if free_index is None or free_index.numel() == 0:
|
||||
return
|
||||
if self.lazy_compaction:
|
||||
num_pages = free_index.numel() // self.page_size
|
||||
assert num_pages <= self._pending_hicache_load_pages, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
|
||||
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
|
||||
)
|
||||
self._pending_hicache_load_pages -= num_pages
|
||||
num_pages = free_index.numel() // self.page_size
|
||||
assert num_pages <= self._pending_hicache_load_pages, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
|
||||
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
|
||||
)
|
||||
self._pending_hicache_load_pages -= num_pages
|
||||
self.free_physical(free_index)
|
||||
|
||||
def free_physical(self, free_index: torch.Tensor) -> None:
|
||||
|
||||
@@ -28,7 +28,6 @@ from __future__ import annotations
|
||||
import logging
|
||||
from array import array
|
||||
from collections import deque
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
@@ -1532,51 +1531,6 @@ class BufferModePipeline:
|
||||
),
|
||||
0,
|
||||
)
|
||||
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
|
||||
binds_swa_to_full = (
|
||||
swa_entry is not None
|
||||
and swa_entry.device_indices_from_anchor_fn is not None
|
||||
)
|
||||
repair_ranges = []
|
||||
if binds_swa_to_full and staged_swa:
|
||||
window_start = span_end - staged_swa
|
||||
repair_end = min(splice_base, span_end)
|
||||
if window_start < repair_end:
|
||||
repair_ranges = cache.tree_core.swa_tombstone_ranges(
|
||||
key, window_start, repair_end
|
||||
)
|
||||
# Only missing SWA rows belong to this load. Existing bindings may
|
||||
# be in use by another request and must survive allocation rollback.
|
||||
anchor_parts = []
|
||||
host_parts = []
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
anchor_parts.append(prefix_indices[repair_start:repair_end_])
|
||||
host_parts.append(
|
||||
slice(repair_start - window_start, repair_end_ - window_start)
|
||||
)
|
||||
tail_start = max(splice_base, window_start)
|
||||
if tail_start < span_end:
|
||||
anchor_parts.append(
|
||||
slice(tail_start - splice_base, span_end - splice_base)
|
||||
)
|
||||
host_parts.append(
|
||||
slice(tail_start - window_start, span_end - window_start)
|
||||
)
|
||||
for i, transfer in enumerate(load_xfers):
|
||||
if transfer.name != PoolName.SWA:
|
||||
continue
|
||||
# Keep the original complete host bounce for ack/drop release.
|
||||
load_xfers[i] = replace(
|
||||
transfer,
|
||||
host_indices=(
|
||||
torch.cat([transfer.host_indices[part] for part in host_parts])
|
||||
if host_parts
|
||||
else transfer.host_indices[:0]
|
||||
),
|
||||
anchor_index_parts=anchor_parts,
|
||||
)
|
||||
if not anchor_parts:
|
||||
load_xfers = [t for t in load_xfers if t.name != PoolName.SWA]
|
||||
|
||||
device_indices = cc.load(
|
||||
host_indices=f.host_indices[trim_tokens:],
|
||||
@@ -1605,21 +1559,10 @@ class BufferModePipeline:
|
||||
None,
|
||||
)
|
||||
aux_device_releases: list[tuple[PoolName, torch.Tensor]] = []
|
||||
if swa_dev is not None and binds_swa_to_full:
|
||||
# Binding already updated the allocator's virtual-to-physical table.
|
||||
# The tree owns virtual FULL rows, not the H2D kernel-facing IDs.
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
for action in cache.tree_core.attach_swa_window(
|
||||
key,
|
||||
repair_start,
|
||||
repair_end_,
|
||||
prefix_indices[repair_start:repair_end_],
|
||||
):
|
||||
cache._apply_cache_action(action)
|
||||
elif swa_dev is not None:
|
||||
# Register the window's FULL->SWA translation now (attention reads
|
||||
# through it). Keep SWA slots another request may still hold; their
|
||||
# redundant H2D destinations are reclaimed at the transfer ack.
|
||||
if swa_dev is not None:
|
||||
# SWA loads into fresh reservations. Register the window's FULL->SWA
|
||||
# translation now (attention reads through it); slots another request
|
||||
# still holds are kept, and their redundant destinations freed at ack.
|
||||
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
|
||||
|
||||
@@ -115,9 +115,6 @@ class PoolTransfer:
|
||||
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
|
||||
nodes_to_load: Optional[List[Any]] = None
|
||||
indices_from_pool: Optional[PoolName] = None
|
||||
# Full IDs backing a dependent device allocation: resident tensors or
|
||||
# slices of the full rows allocated by this load, in transfer order.
|
||||
anchor_index_parts: Optional[List[torch.Tensor | slice]] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -1719,7 +1719,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
return None
|
||||
newly_allocated: list[tuple[PoolTransfer, Callable, torch.Tensor]] = []
|
||||
derived_transfers: list[PoolTransfer] = []
|
||||
anchor_transfers = []
|
||||
|
||||
def rollback_allocated() -> None:
|
||||
for prev_pool, prev_free_fn, prev_indices in reversed(newly_allocated):
|
||||
@@ -1735,11 +1734,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
continue
|
||||
if pool.device_indices is not None or pool.host_indices is None:
|
||||
continue
|
||||
if entry.device_indices_from_anchor_fn is not None:
|
||||
# Allocate independent pools first: their allocation/eviction
|
||||
# can compact SWA before its physical IDs are captured.
|
||||
anchor_transfers.append((pool, entry))
|
||||
continue
|
||||
# device_alloc_fn / device_free_fn override entry.device_pool's
|
||||
# methods for pools whose device_pool is a raw KV pool (layout)
|
||||
# rather than an allocator (e.g. SWA).
|
||||
@@ -1758,28 +1752,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool.device_indices = indices
|
||||
newly_allocated.append((pool, free_fn, indices))
|
||||
|
||||
for pool, entry in anchor_transfers:
|
||||
if kv_device_indices is None or not pool.anchor_index_parts:
|
||||
rollback_allocated()
|
||||
return None
|
||||
anchor_indices = torch.cat(
|
||||
[
|
||||
kv_device_indices[part] if isinstance(part, slice) else part
|
||||
for part in pool.anchor_index_parts
|
||||
]
|
||||
)
|
||||
assert len(anchor_indices) == len(pool.host_indices)
|
||||
bind = entry.device_indices_from_anchor_fn
|
||||
indices = bind(anchor_indices)
|
||||
if indices is None and entry.device_evict_fn:
|
||||
entry.device_evict_fn(len(anchor_indices))
|
||||
indices = bind(anchor_indices)
|
||||
if indices is None:
|
||||
rollback_allocated()
|
||||
return None
|
||||
pool.device_indices = indices
|
||||
newly_allocated.append((pool, entry.device_free_fn, anchor_indices))
|
||||
|
||||
# Assign indices to deferred pools from their source.
|
||||
for pool in derived_transfers:
|
||||
if pool.indices_from_pool == PoolName.KV:
|
||||
|
||||
@@ -210,7 +210,6 @@ def build_pool_entry(
|
||||
device_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_alloc_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_free_fn: Optional[Callable[[Any], Any]] = None,
|
||||
device_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
|
||||
packed_draft_device_pools: tuple[Any, ...] = (),
|
||||
) -> PoolEntry:
|
||||
return PoolEntry(
|
||||
@@ -223,7 +222,6 @@ def build_pool_entry(
|
||||
device_evict_fn=device_evict_fn,
|
||||
device_alloc_fn=device_alloc_fn,
|
||||
device_free_fn=device_free_fn,
|
||||
device_indices_from_anchor_fn=device_indices_from_anchor_fn,
|
||||
packed_draft_device_pools=packed_draft_device_pools,
|
||||
)
|
||||
|
||||
@@ -271,21 +269,16 @@ def build_kv_only_group(
|
||||
)
|
||||
|
||||
|
||||
def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
|
||||
def _swa_allocation_callbacks(allocator) -> dict:
|
||||
"""Keep allocation and rollback in the same ID space for every SWA stack."""
|
||||
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
|
||||
|
||||
if isinstance(allocator, MultiEndedAllocator):
|
||||
# Unified SWA loads into physical reservations, bound before the H2D.
|
||||
return dict(
|
||||
device_alloc_fn=allocator.alloc_physical,
|
||||
device_free_fn=allocator.cancel_physical_reservation,
|
||||
)
|
||||
if bind is not None:
|
||||
assert free_bound is not None
|
||||
return dict(
|
||||
device_indices_from_anchor_fn=bind,
|
||||
device_free_fn=free_bound,
|
||||
)
|
||||
if allocator is None:
|
||||
return {}
|
||||
return dict(device_alloc_fn=allocator.alloc, device_free_fn=allocator.free)
|
||||
@@ -327,8 +320,6 @@ def build_hybrid_swa_group(
|
||||
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
swa_attn_allocator: Any = None,
|
||||
swa_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
|
||||
swa_free_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
|
||||
mtp_swa_device_pools: tuple[Any, ...] = (),
|
||||
) -> HostPoolGroup:
|
||||
"""Anchor (full) + SWA host pool group for a hybrid-SWA device pool."""
|
||||
@@ -394,11 +385,7 @@ def build_hybrid_swa_group(
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(mtp_swa_device_pools),
|
||||
host_evict_fn=host_swa_evict_fn,
|
||||
device_evict_fn=device_swa_evict_fn,
|
||||
**_swa_allocation_callbacks(
|
||||
swa_attn_allocator,
|
||||
swa_indices_from_anchor_fn,
|
||||
swa_free_from_anchor_fn,
|
||||
),
|
||||
**_swa_allocation_callbacks(swa_attn_allocator),
|
||||
packed_draft_device_pools=mtp_swa_device_pools,
|
||||
),
|
||||
]
|
||||
@@ -499,17 +486,6 @@ def build_hybrid_swa_stack(
|
||||
device_swa_evict_fn=device_swa_evict_fn,
|
||||
# For SWA hybrid, device allocation goes through the inner allocator.
|
||||
swa_attn_allocator=params.token_to_kv_pool_allocator.swa_attn_allocator,
|
||||
# Unified SWA binds pages to the full pool's virtual IDs instead.
|
||||
swa_indices_from_anchor_fn=(
|
||||
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
|
||||
if get_memory().enable_unified_memory
|
||||
else None
|
||||
),
|
||||
swa_free_from_anchor_fn=(
|
||||
params.token_to_kv_pool_allocator.free_swa
|
||||
if get_memory().enable_unified_memory
|
||||
else None
|
||||
),
|
||||
mtp_swa_device_pools=mtp_swa_device_pools,
|
||||
)
|
||||
cache_controller = HybridCacheController(
|
||||
@@ -1329,15 +1305,7 @@ def build_hybrid_mamba_swa_stack(
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
host_evict_fn=host_swa_evict_fn,
|
||||
device_evict_fn=device_swa_evict_fn,
|
||||
**_swa_allocation_callbacks(
|
||||
swa_attn_allocator,
|
||||
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
|
||||
if get_memory().enable_unified_memory
|
||||
else None,
|
||||
params.token_to_kv_pool_allocator.free_swa
|
||||
if get_memory().enable_unified_memory
|
||||
else None,
|
||||
),
|
||||
**_swa_allocation_callbacks(swa_attn_allocator),
|
||||
),
|
||||
build_pool_entry(
|
||||
name=PoolName.MAMBA,
|
||||
|
||||
@@ -25,10 +25,6 @@ class PoolEntry:
|
||||
device_evict_fn: Callable[[int], Any] | None = None
|
||||
device_alloc_fn: Callable[[int], Any] | None = None
|
||||
device_free_fn: Callable[[Any], Any] | None = None
|
||||
# Bind rows to the anchor's virtual IDs when pools share an ID space.
|
||||
# Return buffer indices, or None if allocation fails. Rollback through
|
||||
# device_free_fn takes the anchor's virtual IDs, not the returned indices.
|
||||
device_indices_from_anchor_fn: Callable[[Any], Any] | None = None
|
||||
packed_draft_device_pools: tuple[Any, ...] = ()
|
||||
|
||||
|
||||
|
||||
@@ -62,6 +62,9 @@ def estimate_swa_kv_tokens(
|
||||
class PrefillBudget:
|
||||
"""Fixed token pool. Offsets include pending allocations and decode headroom."""
|
||||
|
||||
# When False, admission skips the FULL spec and passes the host hit length.
|
||||
load_back_needs_full_tokens = False
|
||||
|
||||
def __init__(self, allocator, tree_cache, *, num_mixed_decode_tokens: int = 0):
|
||||
self.allocator = allocator
|
||||
self.tree_cache = tree_cache
|
||||
@@ -315,6 +318,8 @@ class SWAPrefillBudget(PrefillBudget):
|
||||
class SharedSWAPrefillBudget(SWAPrefillBudget):
|
||||
"""FULL and SWA reservations compete for the same physical byte budget."""
|
||||
|
||||
load_back_needs_full_tokens = True
|
||||
|
||||
def __init__(self, *args, num_mixed_decode_tokens=0, **kwargs):
|
||||
super().__init__(
|
||||
*args, num_mixed_decode_tokens=num_mixed_decode_tokens, **kwargs
|
||||
|
||||
@@ -568,11 +568,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result = EvictDeviceNextNodeResult(
|
||||
node_id=binding_result.node_id,
|
||||
made_progress=binding_result.made_progress,
|
||||
backup_kv=(
|
||||
_cache_action_from_tagged(binding_result.backup_kv)
|
||||
if binding_result.backup_kv is not None
|
||||
else None
|
||||
),
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
mamba_backup_node_id=binding_result.mamba_backup_node_id,
|
||||
swa_backup_node_id=binding_result.swa_backup_node_id,
|
||||
@@ -788,9 +783,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def set_hicache_enabled(self) -> None:
|
||||
self._binding.set_hicache_enabled()
|
||||
|
||||
def enable_swa_write_back_eviction_barrier(self) -> None:
|
||||
self._binding.enable_swa_write_back_eviction_barrier()
|
||||
|
||||
def set_host_memory_buffer_only(self) -> None:
|
||||
self._binding.set_host_memory_buffer_only()
|
||||
|
||||
@@ -961,27 +953,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
kv_xfer = _transfer_from_binding(kv_xfer)
|
||||
comp_xfers = _comp_xfers_from_binding(comp_xfers)
|
||||
swa_xfers = comp_xfers.get(ComponentType.SWA, ())
|
||||
if swa_xfers:
|
||||
full_node_ids = kv_xfer.nodes_to_load or []
|
||||
full_load_slices = {}
|
||||
offset = 0
|
||||
for full_node_id, count in zip(
|
||||
full_node_ids, self._binding.get_node_key_lengths(full_node_ids)
|
||||
):
|
||||
full_load_slices[full_node_id] = slice(offset, offset + count)
|
||||
offset += count
|
||||
for transfer in swa_xfers:
|
||||
# SWA may have holes between resident nodes, or reload while
|
||||
# FULL stays resident. Preserve the SWA transfer's node order.
|
||||
transfer.anchor_index_parts = [
|
||||
(
|
||||
full_load_slices[nid]
|
||||
if nid in full_load_slices
|
||||
else self.get_component_device_value(nid, ComponentType.FULL)
|
||||
)
|
||||
for nid in transfer.nodes_to_load or ()
|
||||
]
|
||||
return kv_xfer, comp_xfers
|
||||
|
||||
def prefetch_anchor_info(
|
||||
|
||||
@@ -2494,21 +2494,6 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
nodes_to_load=[],
|
||||
)
|
||||
return empty_kv, {}
|
||||
# SWA can be evicted independently of FULL, including holes between
|
||||
# resident SWA nodes. Describe precisely which full rows back it.
|
||||
full_load_slices = {}
|
||||
offset = 0
|
||||
for nid in kv_xfer.nodes_to_load or ():
|
||||
count = len(self.node_by_id(nid).key)
|
||||
full_load_slices[nid] = slice(offset, offset + count)
|
||||
offset += count
|
||||
for xfer in comp_xfers.get(ComponentType.SWA, ()):
|
||||
xfer.anchor_index_parts = [
|
||||
full_load_slices[nid]
|
||||
if nid in full_load_slices
|
||||
else self.node_by_id(nid).component_data[BASE_COMPONENT_TYPE].value
|
||||
for nid in xfer.nodes_to_load or ()
|
||||
]
|
||||
return kv_xfer, comp_xfers
|
||||
|
||||
def prefetch_anchor_info(
|
||||
|
||||
@@ -52,7 +52,6 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
|
||||
"""
|
||||
|
||||
node_id: Optional[NodeId] = None
|
||||
backup_kv: Optional[BackupKV] = None
|
||||
made_progress: bool = False
|
||||
unbacked_tokens: int = 0
|
||||
mamba_backup_node_id: Optional[NodeId] = None
|
||||
@@ -523,13 +522,6 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""Mark the host tier (HiCache) as wired."""
|
||||
...
|
||||
|
||||
def enable_swa_write_back_eviction_barrier(self) -> None:
|
||||
"""Enable a backend-managed barrier when needed.
|
||||
|
||||
The Python core demotes through SWAComponent directly. Native cores
|
||||
may return a backup action to the cache executor before eviction.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def set_host_memory_buffer_only(self) -> None:
|
||||
"""Mark the host tier as buffer-only: one node staged per backup intent."""
|
||||
|
||||
@@ -532,13 +532,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.cache_controller is not None
|
||||
and self.cache_controller.write_policy == "write_back"
|
||||
)
|
||||
# Preserve the SWA host window before device eviction makes it unrecoverable.
|
||||
if (
|
||||
get_memory().enable_unified_memory
|
||||
and self.host_memory_mode == "cache"
|
||||
and self.tree_core.has_swa_host_pool
|
||||
):
|
||||
self.tree_core.enable_swa_write_back_eviction_barrier()
|
||||
# Pre-seed the logical dropped-tokens series.
|
||||
if self.metrics_collector is not None and self.cache_controller is not None:
|
||||
reasons = ["host_pressure"]
|
||||
@@ -805,74 +798,45 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def _evict_device_next_node(
|
||||
self, component_type: ComponentType, tracker: dict[ComponentType, int]
|
||||
) -> tuple[Optional[NodeId], bool]:
|
||||
"""Advance the walk, completing pending host-backup barriers."""
|
||||
while True:
|
||||
result = self.tree_core.evict_device_next_node(component_type, tracker)
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
)
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
# proceeds to make room on the device.
|
||||
node_id = result.mamba_backup_node_id
|
||||
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
|
||||
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
|
||||
self.evict_host(1, ComponentType.MAMBA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_mamba_state_eviction(node_id)
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
# even if host allocation fails.
|
||||
node_id = result.swa_backup_node_id
|
||||
needed = result.swa_backup_num_tokens
|
||||
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
|
||||
if (
|
||||
swa_host_pool is not None
|
||||
and swa_host_pool.available_size() < needed
|
||||
):
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
reason="write_through_unbacked_eviction",
|
||||
)
|
||||
self._accumulate_tracker(tracker, result.tracker)
|
||||
if result.backup_kv is None:
|
||||
return result.node_id, result.made_progress
|
||||
|
||||
assert result.node_id is None
|
||||
assert self.buffer_pipeline is None, (
|
||||
"SWA write-back eviction barriers are cache-mode only"
|
||||
"""Advance the eviction walk one node, consuming its step result."""
|
||||
result = self.tree_core.evict_device_next_node(component_type, tracker)
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
)
|
||||
written = self._execute_and_commit_kv_backup(
|
||||
result.backup_kv, write_back=True
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
# proceeds to make room on the device.
|
||||
node_id = result.mamba_backup_node_id
|
||||
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
|
||||
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
|
||||
self.evict_host(1, ComponentType.MAMBA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_mamba_state_eviction(node_id)
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
)
|
||||
if written <= 0:
|
||||
node_id = result.backup_kv.node_ids[0]
|
||||
logger.warning(
|
||||
"write_back: auxiliary backup failed under host pressure "
|
||||
"(component=%s, node=%d); dropping only the component",
|
||||
component_type.name,
|
||||
node_id,
|
||||
)
|
||||
# Match the Python SWA demotion: preserve FULL and descendants;
|
||||
# the resumed native walk tombstones this component after one try.
|
||||
continue
|
||||
self.writing_check(write_back=True)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
# even if host allocation fails.
|
||||
node_id = result.swa_backup_node_id
|
||||
needed = result.swa_backup_num_tokens
|
||||
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
|
||||
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
reason="write_through_unbacked_eviction",
|
||||
)
|
||||
self._accumulate_tracker(tracker, result.tracker)
|
||||
return result.node_id, result.made_progress
|
||||
|
||||
def _evict_device_leaf(
|
||||
self, node_id: NodeId, tracker: dict[ComponentType, int]
|
||||
|
||||
@@ -802,19 +802,6 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
break 'step None;
|
||||
}
|
||||
}
|
||||
if tree_core.is_write_back
|
||||
&& tree_core.swa_write_back_eviction_barrier_enabled
|
||||
&& tree_core.component_state(SWA).evict_device_last_backup != Some(x)
|
||||
&& !tree_core.arena.node(x).backuped()
|
||||
{
|
||||
// A later Full backup cannot recover SWA data after this
|
||||
// internal node is tombstoned. Pause on the same cursor so
|
||||
// the Controller can preserve the dirty path first.
|
||||
tree_core.component_state_mut(SWA).evict_device_backup_node = Some(x);
|
||||
tree_core.component_state_mut(SWA).evict_device_last_backup = Some(x);
|
||||
cursor = Some(x);
|
||||
break 'step None;
|
||||
}
|
||||
// Internal nodes are tombstoned inline (no IO).
|
||||
tree_core.evict_component_and_detach_lru_(
|
||||
x,
|
||||
|
||||
@@ -847,7 +847,6 @@ fn tracker_to_py(tracker: HashMap<ComponentType, usize>) -> HashMap<u8, usize> {
|
||||
#[pyclass(get_all)]
|
||||
pub struct EvictDeviceNextNodeResultBinding {
|
||||
node_id: Option<NodeId>,
|
||||
backup_kv: Option<Py<PyAny>>,
|
||||
mamba_backup_node_id: Option<NodeId>,
|
||||
swa_backup_node_id: Option<NodeId>,
|
||||
swa_backup_num_tokens: usize,
|
||||
@@ -1431,16 +1430,11 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
|
||||
let (node_id, result) =
|
||||
py.allow_threads(move || self.core().evict_device_next_node(ct, &baseline));
|
||||
let made_progress = node_id.is_some()
|
||||
|| result.backup_kv.is_some()
|
||||
|| result.mamba_backup_node_id.is_some()
|
||||
|| result.swa_backup_node_id.is_some()
|
||||
|| !result.tracker.is_empty();
|
||||
Ok(EvictDeviceNextNodeResultBinding {
|
||||
node_id,
|
||||
backup_kv: result
|
||||
.backup_kv
|
||||
.map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup)))
|
||||
.transpose()?,
|
||||
mamba_backup_node_id: result.mamba_backup_node_id,
|
||||
swa_backup_node_id: result.swa_backup_node_id,
|
||||
swa_backup_num_tokens: result.swa_backup_num_tokens,
|
||||
@@ -1479,7 +1473,6 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
|
||||
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
||||
Ok(EvictDeviceNextNodeResultBinding {
|
||||
node_id: None,
|
||||
backup_kv: None,
|
||||
mamba_backup_node_id: None,
|
||||
swa_backup_node_id: None,
|
||||
swa_backup_num_tokens: 0,
|
||||
@@ -1649,11 +1642,6 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
|
||||
py.allow_threads(|| self.core().enable_hicache)
|
||||
}
|
||||
|
||||
/// Preserve dirty SWA data before cache-mode write-back eviction.
|
||||
fn enable_swa_write_back_eviction_barrier(&self, py: Python<'_>) {
|
||||
py.allow_threads(|| self.core().enable_swa_write_back_eviction_barrier());
|
||||
}
|
||||
|
||||
/// Mark the SWA host pool as wired (HiCache).
|
||||
fn set_has_swa_host_pool(&self, py: Python<'_>) {
|
||||
py.allow_threads(|| self.core().set_has_swa_host_pool());
|
||||
@@ -3067,11 +3055,6 @@ macro_rules! tree_core_binding {
|
||||
catch_native_panic(|| Ok(self.inner.enable_hicache(py)))
|
||||
}
|
||||
|
||||
/// Preserve dirty SWA data before cache-mode write-back eviction.
|
||||
fn enable_swa_write_back_eviction_barrier(&self, py: Python<'_>) {
|
||||
self.inner.enable_swa_write_back_eviction_barrier(py)
|
||||
}
|
||||
|
||||
/// Mark the SWA host pool as wired (HiCache).
|
||||
fn set_has_swa_host_pool(&self, py: Python<'_>) -> PyResult<()> {
|
||||
catch_native_panic(|| {
|
||||
|
||||
@@ -509,11 +509,6 @@ pub struct ComponentState {
|
||||
/// leaf may be freed: the leaf's parent for Full, the LRU predecessor for
|
||||
/// SWA and Mamba.
|
||||
pub(crate) evict_device_cursor: Option<NodeIdx_>,
|
||||
/// Internal node whose component value must be backed up before the walk
|
||||
/// can tombstone it. The Controller consumes this request between steps.
|
||||
pub(crate) evict_device_backup_node: Option<NodeIdx_>,
|
||||
/// A resumed victim is tombstoned after its best-effort backup attempt.
|
||||
pub(crate) evict_device_last_backup: Option<NodeIdx_>,
|
||||
/// Internal component victim waiting for the controller's host backup attempt.
|
||||
/// A generation-checked handle survives host eviction during that I/O.
|
||||
pub(crate) evict_device_pending_node: Option<NodeId>,
|
||||
@@ -587,7 +582,6 @@ pub struct EvictionStepResult {
|
||||
pub tracker: HashMap<ComponentType, usize>,
|
||||
pub device_frees: HashMap<ComponentType, Vec<Tensor>>,
|
||||
pub host_frees: HashMap<ComponentType, Vec<Tensor>>,
|
||||
pub backup_kv: Option<BackupKV>,
|
||||
/// Full device tokens freed without a host copy during this device step.
|
||||
pub unbacked_tokens: usize,
|
||||
/// Back up this internal Mamba state before resuming its device tombstone.
|
||||
@@ -641,8 +635,6 @@ pub struct UnifiedTreeCore<K: ChildKeyType> {
|
||||
pub(crate) enable_external_cache_linker: bool,
|
||||
/// Whether the cache wired a host SWA pool (HiCache).
|
||||
pub(crate) has_swa_host_pool: bool,
|
||||
/// Whether dirty internal SWA nodes must be backed up before eviction.
|
||||
pub(crate) swa_write_back_eviction_barrier_enabled: bool,
|
||||
/// Whether tree mutations emit BlockStored/BlockRemoved events.
|
||||
pub(crate) enable_kv_cache_events: bool,
|
||||
/// Queued placement events, drained by take_events.
|
||||
@@ -755,8 +747,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
state.is_evict_device_ongoing = true;
|
||||
state.evict_device_request_cnt = request_cnt;
|
||||
state.evict_device_cursor = None;
|
||||
state.evict_device_backup_node = None;
|
||||
state.evict_device_last_backup = None;
|
||||
state.evict_device_pending_node = None;
|
||||
state.evict_device_pending_num_tokens = 0;
|
||||
}
|
||||
@@ -771,8 +761,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
);
|
||||
state.is_evict_device_ongoing = false;
|
||||
state.evict_device_cursor = None;
|
||||
state.evict_device_backup_node = None;
|
||||
state.evict_device_last_backup = None;
|
||||
state.evict_device_pending_node = None;
|
||||
state.evict_device_pending_num_tokens = 0;
|
||||
}
|
||||
@@ -841,7 +829,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
enable_storage: false,
|
||||
enable_external_cache_linker: false,
|
||||
has_swa_host_pool: params.has_swa_host_pool,
|
||||
swa_write_back_eviction_barrier_enabled: false,
|
||||
enable_kv_cache_events: params.enable_kv_cache_events,
|
||||
kv_event_queue: Vec::new(),
|
||||
namespaced_event_hashes: HashMap::new(),
|
||||
@@ -2397,15 +2384,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
&mut result.device_frees,
|
||||
&mut result.host_frees,
|
||||
);
|
||||
let backup_node = self
|
||||
.component_state_mut(component_type)
|
||||
.evict_device_backup_node
|
||||
.take();
|
||||
if let Some(backup_node) = backup_node {
|
||||
assert!(node_id.is_none());
|
||||
result.backup_kv =
|
||||
Some(self.build_backup_kv_action_(self.arena.node(backup_node), true));
|
||||
}
|
||||
result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap();
|
||||
let state = self.component_state(component_type);
|
||||
if component_type == MAMBA {
|
||||
@@ -3161,11 +3139,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
self.enable_hicache = true;
|
||||
}
|
||||
|
||||
/// Preserve dirty internal SWA nodes before cache-mode write-back eviction.
|
||||
pub fn enable_swa_write_back_eviction_barrier(&mut self) {
|
||||
self.swa_write_back_eviction_barrier_enabled = true;
|
||||
}
|
||||
|
||||
/// Mark the host tier as buffer-only; wired after the host pools are built.
|
||||
pub fn set_host_memory_buffer_only(&mut self) {
|
||||
self.is_host_memory_buffer_only = true;
|
||||
|
||||
@@ -29,6 +29,7 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
_require_single_row_dsv4_swa_pages,
|
||||
_split_hicache_size,
|
||||
_SwaStrategy,
|
||||
_uses_unified_page_envelope_host,
|
||||
build_full_draft_pools,
|
||||
build_host_pool_group,
|
||||
build_hybrid_swa_group,
|
||||
@@ -1245,5 +1246,40 @@ class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
|
||||
self.assertIsNotNone(swa_pool.alloc(page_size))
|
||||
|
||||
|
||||
class TestUnifiedPageEnvelopeSelection(CustomTestCase):
|
||||
"""Only no backend and Mori store the complete shared page, so only they
|
||||
keep the page-envelope host arena."""
|
||||
|
||||
def test_only_no_backend_and_mori_select_the_envelope(self):
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
pool = _build_unified_swa_pool().token_to_kv_pool
|
||||
self.addCleanup(reset_context)
|
||||
for backend, expected in (
|
||||
(None, True),
|
||||
("mori", True),
|
||||
("mooncake", False),
|
||||
("dynamic", False),
|
||||
):
|
||||
with self.subTest(backend=backend):
|
||||
reset_context()
|
||||
publish(
|
||||
ServerArgs(
|
||||
model_path="dummy",
|
||||
enable_unified_memory=True,
|
||||
hicache_storage_backend=backend,
|
||||
hicache_mem_layout="layer_first",
|
||||
),
|
||||
role="tokenizer",
|
||||
)
|
||||
self.assertEqual(
|
||||
_uses_unified_page_envelope_host(
|
||||
pool.full_kv_pool, pool.swa_kv_pool, use_mla=False
|
||||
),
|
||||
expected,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -2192,12 +2192,7 @@ def test_swa_host_pressure_retains_device_resident_backups(backend, guard):
|
||||
|
||||
@pytest.mark.parametrize("backend", ["python", "rust"])
|
||||
@pytest.mark.parametrize("resident_full", [(), (0,), (0, 1, 2, 3)])
|
||||
def test_swa_load_back_preserves_full_anchors_across_holes(backend, resident_full):
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.group import PoolEntry
|
||||
|
||||
def test_swa_load_back_spec_skips_resident_swa_holes(backend, resident_full):
|
||||
core, _ = _swa_transfer_core(backend)
|
||||
nodes = []
|
||||
boundaries = (0, 2, 5, 6, 8)
|
||||
@@ -2241,52 +2236,10 @@ def test_swa_load_back_preserves_full_anchors_across_holes(backend, resident_ful
|
||||
)
|
||||
core.finish_load_back(nodes[index])
|
||||
|
||||
kv, auxiliary = core.build_load_back_spec(nodes[-1])
|
||||
_, auxiliary = core.build_load_back_spec(nodes[-1])
|
||||
(swa,) = auxiliary[ComponentType.SWA]
|
||||
assert swa.nodes_to_load == [nodes[0], nodes[2], nodes[3]]
|
||||
assert swa.host_indices.tolist() == [300, 301, 305, 306, 307]
|
||||
expected = {
|
||||
(): [slice(0, 2), slice(5, 6), slice(6, 8)],
|
||||
(0,): [torch.tensor([10, 11]), slice(3, 4), slice(4, 6)],
|
||||
(0, 1, 2, 3): [
|
||||
torch.tensor([10, 11]),
|
||||
torch.tensor([15]),
|
||||
torch.tensor([16, 17]),
|
||||
],
|
||||
}[resident_full]
|
||||
assert swa.anchor_index_parts is not None
|
||||
assert len(swa.anchor_index_parts) == len(expected)
|
||||
for actual, wanted in zip(swa.anchor_index_parts, expected):
|
||||
if isinstance(wanted, slice):
|
||||
assert actual == wanted
|
||||
else:
|
||||
torch.testing.assert_close(actual, wanted)
|
||||
|
||||
# Exercise the consumer too: new Full rows and resident virtual IDs must
|
||||
# bind the same five SWA rows, without consuming the resident-SWA hole.
|
||||
bind = Mock(side_effect=lambda indices: indices + 1000)
|
||||
controller = object.__new__(HybridCacheController)
|
||||
controller.mem_pool_host = SimpleNamespace(
|
||||
entry_map={
|
||||
PoolName.SWA: PoolEntry(
|
||||
name=PoolName.SWA,
|
||||
host_pool=SimpleNamespace(),
|
||||
device_pool=SimpleNamespace(),
|
||||
layer_mapper=lambda i: i,
|
||||
device_indices_from_anchor_fn=bind,
|
||||
device_free_fn=Mock(),
|
||||
)
|
||||
}
|
||||
)
|
||||
loaded = torch.arange(200, 200 + len(kv.host_indices))
|
||||
assert controller._resolve_device_transfers([swa], loaded) is not None
|
||||
expected_bound = {
|
||||
(): [1200, 1201, 1205, 1206, 1207],
|
||||
(0,): [1010, 1011, 1203, 1204, 1205],
|
||||
(0, 1, 2, 3): [1010, 1011, 1015, 1016, 1017],
|
||||
}[resident_full]
|
||||
assert swa.device_indices.tolist() == expected_bound
|
||||
assert bind.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["python", "rust"])
|
||||
|
||||
@@ -86,7 +86,6 @@ def _staged_fixture(full_match=2):
|
||||
entry_map={
|
||||
PoolName.SWA: SimpleNamespace(
|
||||
host_pool=SimpleNamespace(free=Mock()),
|
||||
device_indices_from_anchor_fn=None,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
@@ -7,15 +7,10 @@ from unittest.mock import MagicMock, Mock, patch
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
)
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.pool_host.group import PoolEntry
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small")
|
||||
@@ -74,131 +69,6 @@ class TestHostGateCapacity(unittest.TestCase):
|
||||
self.assertEqual(full.schedulable_available_size(), before)
|
||||
|
||||
|
||||
class TestSwaLoadAllocation(unittest.TestCase):
|
||||
def _controller(self, bind, free=None, evict=None):
|
||||
controller = object.__new__(HybridCacheController)
|
||||
entry = PoolEntry(
|
||||
name=PoolName.SWA,
|
||||
host_pool=SimpleNamespace(),
|
||||
device_pool=SimpleNamespace(),
|
||||
layer_mapper=lambda i: i,
|
||||
device_indices_from_anchor_fn=bind,
|
||||
device_free_fn=free or Mock(),
|
||||
device_evict_fn=evict,
|
||||
)
|
||||
controller.mem_pool_host = SimpleNamespace(entry_map={PoolName.SWA: entry})
|
||||
return controller
|
||||
|
||||
def _transfer(self, parts, count):
|
||||
transfer = PoolTransfer(name=PoolName.SWA, host_indices=torch.arange(count))
|
||||
transfer.anchor_index_parts = parts
|
||||
return transfer
|
||||
|
||||
def test_swa_only_load_uses_resident_full_ids(self):
|
||||
bind = Mock(side_effect=lambda x: x + 100)
|
||||
controller = self._controller(bind)
|
||||
transfer = self._transfer([torch.tensor([13, 14])], 2)
|
||||
result = controller._resolve_device_transfers(
|
||||
[transfer], torch.empty(0, dtype=torch.int64)
|
||||
)
|
||||
self.assertIsNotNone(result)
|
||||
torch.testing.assert_close(transfer.device_indices, torch.tensor([113, 114]))
|
||||
|
||||
def test_mixed_load_skips_resident_swa_nodes(self):
|
||||
bind = Mock(side_effect=lambda x: x + 100)
|
||||
controller = self._controller(bind)
|
||||
transfer = self._transfer([torch.tensor([13, 14]), slice(2, 4)], 4)
|
||||
controller._resolve_device_transfers(
|
||||
[transfer], torch.tensor([20, 21, 22, 23, 24, 25])
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
transfer.device_indices, torch.tensor([113, 114, 122, 123])
|
||||
)
|
||||
|
||||
def test_binding_retries_after_eviction(self):
|
||||
bind = Mock(side_effect=[None, torch.tensor([41, 42])])
|
||||
evict = Mock()
|
||||
controller = self._controller(bind, evict=evict)
|
||||
transfer = self._transfer([slice(0, 2)], 2)
|
||||
self.assertIsNotNone(
|
||||
controller._resolve_device_transfers([transfer], torch.tensor([11, 12]))
|
||||
)
|
||||
evict.assert_called_once_with(2)
|
||||
self.assertEqual(bind.call_count, 2)
|
||||
|
||||
def test_rollback_releases_swa_binding_by_virtual_ids(self):
|
||||
free = Mock()
|
||||
controller = self._controller(lambda x: x + 100, free=free)
|
||||
transfer = self._transfer([torch.tensor([13, 14])], 2)
|
||||
# A missing sidecar source fails after SWA has bound its pages.
|
||||
sidecar = PoolTransfer(name=PoolName.MAMBA, indices_from_pool=PoolName.INDEXER)
|
||||
result = controller._resolve_device_transfers(
|
||||
[transfer, sidecar], torch.empty(0, dtype=torch.int64)
|
||||
)
|
||||
self.assertIsNone(result)
|
||||
torch.testing.assert_close(free.call_args.args[0], torch.tensor([13, 14]))
|
||||
self.assertIsNone(transfer.device_indices)
|
||||
|
||||
def test_independent_allocations_precede_kernel_id_resolution(self):
|
||||
events = []
|
||||
controller = self._controller(lambda x: events.append("bind") or x + 100)
|
||||
controller.mem_pool_host.entry_map[PoolName.MAMBA] = PoolEntry(
|
||||
name=PoolName.MAMBA,
|
||||
host_pool=SimpleNamespace(),
|
||||
device_pool=SimpleNamespace(),
|
||||
layer_mapper=lambda i: i,
|
||||
device_alloc_fn=lambda n: events.append("mamba") or torch.arange(n),
|
||||
device_free_fn=Mock(),
|
||||
)
|
||||
swa = self._transfer([slice(0, 2)], 2)
|
||||
mamba = PoolTransfer(name=PoolName.MAMBA, host_indices=torch.arange(1))
|
||||
self.assertIsNotNone(
|
||||
controller._resolve_device_transfers([swa, mamba], torch.tensor([10, 11]))
|
||||
)
|
||||
self.assertEqual(events, ["mamba", "bind"])
|
||||
|
||||
def test_tree_spec_preserves_node_correspondence(self):
|
||||
kv = PoolTransfer(
|
||||
name=PoolName.KV, host_indices=torch.arange(4), nodes_to_load=[2, 3]
|
||||
)
|
||||
swa = PoolTransfer(
|
||||
name=PoolName.SWA, host_indices=torch.arange(4), nodes_to_load=[1, 3]
|
||||
)
|
||||
nodes = {
|
||||
i: SimpleNamespace(
|
||||
id=i,
|
||||
key=[i, i],
|
||||
load_back_pending_id=None,
|
||||
component_data={
|
||||
ComponentType.FULL: SimpleNamespace(
|
||||
value=torch.tensor([10, 11]) if i == 1 else None
|
||||
)
|
||||
},
|
||||
)
|
||||
for i in (1, 2, 3)
|
||||
}
|
||||
full_component = SimpleNamespace(
|
||||
component_type=ComponentType.FULL,
|
||||
build_hicache_transfers=lambda *a, **k: [kv],
|
||||
)
|
||||
swa_component = SimpleNamespace(
|
||||
component_type=ComponentType.SWA,
|
||||
build_hicache_transfers=lambda *a, **k: [swa],
|
||||
)
|
||||
core = SimpleNamespace(
|
||||
node_by_id=nodes.__getitem__,
|
||||
components=[full_component, swa_component],
|
||||
components_by_type={
|
||||
ComponentType.FULL: full_component,
|
||||
ComponentType.SWA: swa_component,
|
||||
},
|
||||
)
|
||||
UnifiedTreeCore.build_load_back_spec(core, 3)
|
||||
parts = swa.anchor_index_parts
|
||||
torch.testing.assert_close(parts[0], torch.tensor([10, 11]))
|
||||
self.assertEqual(parts[1], slice(2, 4))
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "CUDA required")
|
||||
class TestTransferStreamOrdering(unittest.TestCase):
|
||||
def test_load_translation_follows_supplied_start_event(self):
|
||||
@@ -288,16 +158,20 @@ class TestDirectBackendTranslation(unittest.TestCase):
|
||||
|
||||
class TestTriPoolAssembly(unittest.TestCase):
|
||||
def test_swa_allocation_and_rollback_match_pool_id_ownership(self):
|
||||
from sglang.srt.mem_cache.allocator.unified_sub_pool import (
|
||||
MultiEndedAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler as assembler
|
||||
|
||||
for unified in (False, True):
|
||||
with self.subTest(unified=unified):
|
||||
swa_allocator = SimpleNamespace(alloc=Mock(), free=Mock())
|
||||
composite = SimpleNamespace(
|
||||
swa_attn_allocator=swa_allocator,
|
||||
bind_swa_for_loaded_rows=Mock(),
|
||||
free_swa=Mock(),
|
||||
# Unified memory always builds a MultiEndedAllocator SWA end.
|
||||
swa_allocator = (
|
||||
object.__new__(MultiEndedAllocator)
|
||||
if unified
|
||||
else SimpleNamespace(alloc=Mock(), free=Mock())
|
||||
)
|
||||
composite = SimpleNamespace(swa_attn_allocator=swa_allocator)
|
||||
params = SimpleNamespace(
|
||||
token_to_kv_pool_allocator=composite,
|
||||
req_to_token_pool=SimpleNamespace(
|
||||
@@ -330,16 +204,15 @@ class TestTriPoolAssembly(unittest.TestCase):
|
||||
)
|
||||
entry = group.entry_map[PoolName.SWA]
|
||||
if unified:
|
||||
self.assertIs(
|
||||
entry.device_indices_from_anchor_fn,
|
||||
composite.bind_swa_for_loaded_rows,
|
||||
self.assertEqual(
|
||||
entry.device_alloc_fn, swa_allocator.alloc_physical
|
||||
)
|
||||
self.assertEqual(
|
||||
entry.device_free_fn, swa_allocator.cancel_physical_reservation
|
||||
)
|
||||
self.assertIs(entry.device_free_fn, composite.free_swa)
|
||||
self.assertIsNone(entry.device_alloc_fn)
|
||||
else:
|
||||
self.assertIs(entry.device_alloc_fn, swa_allocator.alloc)
|
||||
self.assertIs(entry.device_free_fn, swa_allocator.free)
|
||||
self.assertIsNone(entry.device_indices_from_anchor_fn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,626 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Unified-memory HiCache on the real SWA allocator stack: physical reservations
|
||||
and the moves they block, the transfer-done event waits, load-back admission,
|
||||
and the write-policy switch check.
|
||||
|
||||
The pools are real CPU tensors. CUDA stream waits are recorded by a stand-in
|
||||
for `torch.cuda.current_stream()` rather than executed.
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py -v
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import contextlib
|
||||
import unittest
|
||||
from array import array
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestHandle,
|
||||
EvictParams,
|
||||
InsertParams,
|
||||
MatchPrefixParams,
|
||||
)
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
_swa_allocation_callbacks,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.pool_host import common as host_memory
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.tree_core_registry import _TREE_CORE_REGISTRY
|
||||
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.unified_allocator_fixtures import build_tri_pool
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
PAGE = 4
|
||||
D2H = ("controller", "write")
|
||||
H2D = ("controller", "load")
|
||||
FULL, SWA = ComponentType.FULL, ComponentType.SWA
|
||||
|
||||
|
||||
def _python_inspector(params, components):
|
||||
from unified_tree_core_inspector import UnifiedTreeCoreInspector
|
||||
|
||||
return UnifiedTreeCoreInspector(params, components)
|
||||
|
||||
|
||||
def _rust_inspector(params, _components):
|
||||
from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector
|
||||
|
||||
return RustUnifiedTreeCoreInspector(params)
|
||||
|
||||
|
||||
def _unified_swa_bundle(*, tokens: int = 64):
|
||||
return init_unified_swa_pools(
|
||||
device="cpu",
|
||||
kv_cache_dtype=torch.float16,
|
||||
head_num=1,
|
||||
head_dim=8,
|
||||
v_head_dim=8,
|
||||
swa_head_num=1,
|
||||
swa_head_dim=8,
|
||||
swa_v_head_dim=8,
|
||||
page_size=PAGE,
|
||||
start_layer=0,
|
||||
end_layer=2,
|
||||
swa_attention_layer_ids=[1],
|
||||
full_attention_layer_ids=[0],
|
||||
full_max_total_num_tokens=tokens,
|
||||
swa_max_total_num_tokens=tokens,
|
||||
enable_memory_saver=False,
|
||||
need_sort=False,
|
||||
lazy_compaction=True,
|
||||
)
|
||||
|
||||
|
||||
def _unified_swa():
|
||||
bundle = _unified_swa_bundle()
|
||||
return bundle.token_to_kv_pool_allocator, _Rows(bundle)
|
||||
|
||||
|
||||
class _Rows:
|
||||
"""One marker per token row, read and written through the allocator's own
|
||||
virtual-to-physical translation, so a relocated row is found where it went."""
|
||||
|
||||
def __init__(self, bundle):
|
||||
self.allocator = bundle.token_to_kv_pool_allocator
|
||||
self.full = bundle.token_to_kv_pool.full_kv_pool.get_key_buffer(0)
|
||||
self.swa = bundle.token_to_kv_pool.swa_kv_pool.get_key_buffer(0)
|
||||
|
||||
def full_rows(self, ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.allocator.full_attn_allocator.translate_kv_loc(ids.to(torch.int64))
|
||||
|
||||
def swa_rows(self, ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.allocator.translate_loc_from_full_to_swa(ids)
|
||||
|
||||
def markers(self, base: int, count: int) -> torch.Tensor:
|
||||
# One value per token row; float16 holds these integers exactly.
|
||||
values = torch.arange(base, base + count, dtype=torch.float16)
|
||||
return values[:, None, None].expand(count, *self.swa.shape[1:]).clone()
|
||||
|
||||
def seed(self, ids: torch.Tensor, base: int):
|
||||
self.full[self.full_rows(ids)] = self.markers(base, len(ids))
|
||||
self.swa[self.swa_rows(ids)] = self.markers(base + 1000, len(ids))
|
||||
return self.read(ids)
|
||||
|
||||
def read(self, ids: torch.Tensor):
|
||||
return (
|
||||
self.full[self.full_rows(ids)].clone(),
|
||||
self.swa[self.swa_rows(ids)].clone(),
|
||||
)
|
||||
|
||||
|
||||
class _Stream:
|
||||
"""Stands in for `torch.cuda.current_stream()` and records what it waits on."""
|
||||
|
||||
def __init__(self, log: list):
|
||||
self.log = log
|
||||
|
||||
def wait_event(self, event) -> None:
|
||||
self.log.append(("wait", event))
|
||||
|
||||
def wait_stream(self, stream) -> None:
|
||||
self.log.append(("wait_stream", stream))
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _recorded_stream():
|
||||
log = []
|
||||
with mock.patch.object(
|
||||
torch.cuda, "current_stream", lambda *args, **kwargs: _Stream(log)
|
||||
):
|
||||
yield log
|
||||
|
||||
|
||||
def _record_moves(allocator, log: list) -> None:
|
||||
"""Log every compaction copy on either side into `log`, then perform it."""
|
||||
for side in (allocator.full_attn_allocator, allocator.swa_attn_allocator):
|
||||
kvcache = side._kvcache
|
||||
move = kvcache.move_kv_cache
|
||||
|
||||
def recorded(dst, src, *, move=move, name=side.sub_pool_name):
|
||||
log.append(("move", name))
|
||||
return move(dst, src)
|
||||
|
||||
kvcache.move_kv_cache = recorded
|
||||
|
||||
|
||||
class _ConfigCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
|
||||
class TestPhysicalReservation(_ConfigCase):
|
||||
"""A pending reservation blocks moves on the SWA end until its load is queued."""
|
||||
|
||||
def test_pending_reservation_blocks_moves_until_its_load_is_queued(self):
|
||||
allocator, rows = _unified_swa()
|
||||
swa = allocator.swa_attn_allocator
|
||||
head = allocator.alloc(2 * PAGE)
|
||||
gap = allocator.alloc(2 * PAGE)
|
||||
tail = allocator.alloc(2 * PAGE)
|
||||
expected_head = rows.seed(head, 100)
|
||||
expected_tail = rows.seed(tail, 200)
|
||||
# The node being loaded: its FULL rows are resident, SWA is tombstoned.
|
||||
target = allocator.alloc(2 * PAGE)
|
||||
allocator.free_swa(target)
|
||||
allocator.free(gap)
|
||||
|
||||
reserved = swa.alloc_physical(2 * PAGE)
|
||||
# Stand in for the H2D copy into the reserved rows.
|
||||
rows.swa[reserved] = rows.markers(300, 2 * PAGE)
|
||||
allocator.set_full_to_swa_mapping(target, reserved)
|
||||
|
||||
moves = []
|
||||
_record_moves(allocator, moves)
|
||||
with _recorded_stream() as waits:
|
||||
# Not queued yet: nothing on the SWA end may move.
|
||||
self.assertEqual(swa._flush(urgent=True), 0)
|
||||
self.assertEqual(swa.flush_opportunistic(), 0)
|
||||
self.assertEqual(moves, [])
|
||||
|
||||
load_done = object()
|
||||
allocator.set_hicache_transfer_done_event(H2D, load_done)
|
||||
# Queued: compaction waits on the copy, then relocates rows.
|
||||
self.assertGreater(swa._flush(urgent=True), 0)
|
||||
self.assertEqual(waits[0], ("wait", load_done))
|
||||
self.assertIn(("move", "swa"), moves)
|
||||
_, target_swa = rows.read(target)
|
||||
self.assertTrue(torch.equal(target_swa, rows.markers(300, 2 * PAGE)))
|
||||
self.assertTrue(torch.equal(rows.read(head)[1], expected_head[1]))
|
||||
self.assertTrue(torch.equal(rows.read(tail)[1], expected_tail[1]))
|
||||
self.assertEqual(allocator.verify_byte_accounting(), [])
|
||||
|
||||
|
||||
class TestFloatSwaReservation(_ConfigCase):
|
||||
"""The tri-pool's SWA middle floats: it relocates its pages to make room for
|
||||
a neighbour, so a load's SWA reservation must pin it as on an end pool."""
|
||||
|
||||
def test_state_allocation_cannot_move_a_reservation_before_its_load_is_queued(
|
||||
self,
|
||||
):
|
||||
bundle, allocator = build_tri_pool(page_size=PAGE)
|
||||
full, swa = allocator.full_attn_allocator, allocator.swa_attn_allocator
|
||||
states = bundle.req_to_token_pool.mamba_allocator
|
||||
mamba = allocator.mamba_allocator
|
||||
resident = allocator.alloc(4 * PAGE)
|
||||
# The Mamba end is full: its next slot needs the float to move.
|
||||
mamba.alloc(mamba.available_size())
|
||||
# A window slides past its oldest page: a hole at the float's low edge.
|
||||
allocator.free_swa(resident[:PAGE])
|
||||
moves = []
|
||||
_record_moves(allocator, moves)
|
||||
|
||||
# Only what `HybridCacheController.load` reads, with production callbacks.
|
||||
controller = object.__new__(HybridCacheController)
|
||||
controller.mem_pool_device_allocator = allocator
|
||||
controller.device = "cpu"
|
||||
controller.load_queue, controller.ack_load_queue = [], []
|
||||
controller.mem_pool_host = SimpleNamespace(
|
||||
entry_map={
|
||||
PoolName.SWA: SimpleNamespace(
|
||||
device_evict_fn=None, **_swa_allocation_callbacks(swa)
|
||||
),
|
||||
PoolName.MAMBA: SimpleNamespace(
|
||||
device_alloc_fn=states.alloc,
|
||||
device_free_fn=states.free,
|
||||
device_evict_fn=None,
|
||||
),
|
||||
}
|
||||
)
|
||||
allocated = (full.allocated_count(), swa.allocated_count())
|
||||
# The production order: FULL, then SWA, then the Mamba checkpoint.
|
||||
transfers = [
|
||||
PoolTransfer(name=PoolName.SWA, host_indices=torch.arange(PAGE)),
|
||||
PoolTransfer(name=PoolName.MAMBA, host_indices=torch.tensor([1])),
|
||||
]
|
||||
|
||||
# The state slot needs the float to move, which the SWA reservation
|
||||
# taken just before forbids: the load rolls back instead.
|
||||
self.assertIsNone(
|
||||
controller.load(host_indices=torch.arange(PAGE), extra_pools=transfers)
|
||||
)
|
||||
self.assertEqual(moves, [])
|
||||
self.assertEqual(controller.load_queue, [])
|
||||
self.assertIsNone(transfers[0].device_indices)
|
||||
self.assertEqual((full.allocated_count(), swa.allocated_count()), allocated)
|
||||
self.assertEqual(allocator.verify_byte_accounting(), [])
|
||||
|
||||
# Cancelled, so the float makes room again.
|
||||
self.assertIsNotNone(states.alloc(1))
|
||||
self.assertIn(("move", "swa"), moves)
|
||||
|
||||
|
||||
class TestAllocationAndTransferOrder(_ConfigCase):
|
||||
"""Allocation reuses free pages without waiting on HiCache copies."""
|
||||
|
||||
def test_hole_reuse_does_not_wait_for_unrelated_copies(self):
|
||||
allocator, rows = _unified_swa()
|
||||
swa = allocator.swa_attn_allocator
|
||||
source = allocator.alloc(2 * PAGE) # a write-through D2H source (locked)
|
||||
gap = allocator.alloc(3 * PAGE)
|
||||
target = allocator.alloc(2 * PAGE) # an H2D target owned by its load
|
||||
expected_source = rows.seed(source, 100)
|
||||
expected_target = rows.seed(target, 200)
|
||||
allocator.free(gap)
|
||||
allocator.set_hicache_transfer_done_event(D2H, object())
|
||||
allocator.set_hicache_transfer_done_event(H2D, object())
|
||||
|
||||
with _recorded_stream() as waits:
|
||||
fresh = allocator.alloc(2 * PAGE)
|
||||
staged = swa.alloc_physical(PAGE)
|
||||
|
||||
self.assertEqual(waits, [])
|
||||
# Writing the new rows leaves the copies' rows intact.
|
||||
rows.seed(fresh, 600)
|
||||
rows.swa[staged] = rows.markers(700, PAGE)
|
||||
for ids, expected in ((source, expected_source), (target, expected_target)):
|
||||
got = rows.read(ids)
|
||||
self.assertTrue(torch.equal(got[0], expected[0]))
|
||||
self.assertTrue(torch.equal(got[1], expected[1]))
|
||||
|
||||
|
||||
class _UnifiedHiCacheCase(_ConfigCase):
|
||||
"""A FULL+SWA `UnifiedRadixCache` with HiCache over the unified pool and no
|
||||
storage backend, so the host side is the shared page-envelope pool. The
|
||||
TreeCore is whichever backend loads (Rust, else Python)."""
|
||||
|
||||
window = 2 * PAGE
|
||||
|
||||
def _cache(
|
||||
self,
|
||||
*,
|
||||
write_policy: str = "write_through",
|
||||
tokens: int = 128,
|
||||
):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
page_size=PAGE,
|
||||
enable_unified_memory=True,
|
||||
hicache_write_policy=write_policy,
|
||||
hicache_mem_layout="layer_first",
|
||||
hicache_host_memory_mode="cache",
|
||||
)
|
||||
set_global_server_args_for_scheduler(server_args)
|
||||
bundle = _unified_swa_bundle(tokens=tokens)
|
||||
self.allocator = bundle.token_to_kv_pool_allocator
|
||||
self.rows = _Rows(bundle)
|
||||
self.req_to_token_pool = ReqToTokenPool(
|
||||
size=4, max_context_len=tokens, device="cpu", enable_memory_saver=False
|
||||
)
|
||||
params = CacheInitParams(
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
page_size=PAGE,
|
||||
disable=False,
|
||||
sliding_window_size=self.window,
|
||||
tree_components=(ComponentType.FULL, ComponentType.SWA),
|
||||
)
|
||||
with mock.patch.dict(
|
||||
_TREE_CORE_REGISTRY, {"python": _python_inspector, "rust": _rust_inspector}
|
||||
):
|
||||
cache = UnifiedRadixCache(params=params)
|
||||
# CPU host tensors need no page-locking.
|
||||
with mock.patch.object(host_memory, "_cuda_host_register"):
|
||||
cache.init_hicache(server_args, params)
|
||||
atexit.unregister(cache.shutdown)
|
||||
self.addCleanup(cache.release_host_resources)
|
||||
controller = cache.cache_controller
|
||||
# The scheduler's gate: rows stay put until host copies are acknowledged.
|
||||
self.allocator.set_host_transfer_move_gate(
|
||||
lambda: not controller.has_inflight_device_transfers()
|
||||
)
|
||||
self.cache = cache
|
||||
self.controller = controller
|
||||
# Compaction and frees wait through this stand-in stream.
|
||||
stream = _recorded_stream()
|
||||
stream.__enter__()
|
||||
self.addCleanup(stream.__exit__, None, None, None)
|
||||
return cache
|
||||
|
||||
def _insert(self, tokens, base):
|
||||
ids = self.allocator.alloc(len(tokens))
|
||||
expected = self.rows.seed(ids, base)
|
||||
self.cache.insert(InsertParams(key=RadixKey(array("q", tokens)), value=ids))
|
||||
return ids, expected
|
||||
|
||||
def _match(self, tokens):
|
||||
return self.cache.match_prefix(
|
||||
MatchPrefixParams(key=RadixKey(array("q", tokens)))
|
||||
)
|
||||
|
||||
def _assert_rows(self, ids, expected, *, swa_tail: int):
|
||||
full, swa = self.rows.read(ids)
|
||||
self.assertTrue(torch.equal(full, expected[0]))
|
||||
self.assertTrue(torch.equal(swa[-swa_tail:], expected[1][-swa_tail:]))
|
||||
|
||||
|
||||
class TestSwaLoadBackReservation(_UnifiedHiCacheCase):
|
||||
"""Cache-mode SWA load-back loads into physical reservations."""
|
||||
|
||||
def test_reservation_retries_once_after_device_eviction(self):
|
||||
self._cache(write_policy="write_through", tokens=64)
|
||||
swa = self.allocator.swa_attn_allocator
|
||||
# Tree-owned rows fill the SWA end; once their backups are acked,
|
||||
# device eviction may reclaim them.
|
||||
fillers = 0
|
||||
while self.allocator.swa_available_size() >= 2 * PAGE:
|
||||
fillers += 1
|
||||
self._insert(
|
||||
list(range(100 * fillers, 100 * fillers + PAGE)), 100 * fillers
|
||||
)
|
||||
self.cache.flush_pending_backups()
|
||||
self.cache.writing_check()
|
||||
entry = self.controller.mem_pool_host.entry_map[PoolName.SWA]
|
||||
evict = entry.device_evict_fn
|
||||
evictions = []
|
||||
|
||||
def recorded_evict(need_size):
|
||||
evictions.append(need_size)
|
||||
return evict(need_size)
|
||||
|
||||
transfer = PoolTransfer(name=PoolName.SWA, host_indices=torch.arange(2 * PAGE))
|
||||
with mock.patch.object(entry, "device_evict_fn", side_effect=recorded_evict):
|
||||
result = self.controller._resolve_device_transfers(
|
||||
[transfer], kv_device_indices=torch.empty(0, dtype=torch.int64)
|
||||
)
|
||||
|
||||
# No room on the first try; one device eviction makes it.
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(evictions, [2 * PAGE])
|
||||
swa.cancel_physical_reservation(transfer.device_indices)
|
||||
self.assertEqual(self.allocator.verify_byte_accounting(), [])
|
||||
self.cache.sanity_check()
|
||||
|
||||
|
||||
class TestLoadBackAdmission(_UnifiedHiCacheCase):
|
||||
"""PrefillAdder load-back admission over the real cache and shared budget."""
|
||||
|
||||
def _adder(self, num_mixed_decode_tokens: int = 0):
|
||||
running = mock.MagicMock()
|
||||
running.reqs = []
|
||||
return PrefillAdder(
|
||||
page_size=PAGE,
|
||||
tree_cache=self.cache,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
running_batch=running,
|
||||
new_token_ratio=1.0,
|
||||
rem_input_tokens=10_000,
|
||||
rem_chunk_tokens=None,
|
||||
num_mixed_decode_tokens=num_mixed_decode_tokens,
|
||||
priority_scheduling_preemption_threshold=0,
|
||||
)
|
||||
|
||||
def _request(self, tokens):
|
||||
match = self._match(tokens)
|
||||
req = mock.MagicMock(spec=Req)
|
||||
req.rid = "load-back"
|
||||
req.cache_request_handle = CacheRequestHandle(req.rid, 0)
|
||||
req.priority = 0
|
||||
req.prefix_len = match.device_prefix_len
|
||||
req.last_node = match.last_device_node
|
||||
req.best_match_node = match.best_match_node
|
||||
req.full_untruncated_fill_ids = list(tokens)
|
||||
req.output_ids = []
|
||||
req.sampling_params = SimpleNamespace(max_new_tokens=PAGE, ignore_eos=False)
|
||||
req.host_hit_length = match.host_hit_length
|
||||
req.swa_host_hit_length = match.swa_host_hit_length
|
||||
req.mamba_host_hit_length = 0
|
||||
req.storage_hit_length = 0
|
||||
req.storage_hit_start = None
|
||||
req.host_hit_is_storage = False
|
||||
req.host_loaded_length = 0
|
||||
req.retracted_stain = False
|
||||
req.needs_host_load_back.return_value = True
|
||||
req.finished.return_value = False
|
||||
req.materialized_host_hit_len.return_value = match.host_hit_length
|
||||
req.fulfilled_storage_hit_len.return_value = 0
|
||||
req.kv = SimpleNamespace(
|
||||
cache_protected_len=match.device_prefix_len, mamba_pool_idx=None
|
||||
)
|
||||
type(req).extend_len = mock.PropertyMock(
|
||||
side_effect=lambda: req.extend_end - req.prefix_len
|
||||
)
|
||||
return req
|
||||
|
||||
def _locks(self, node):
|
||||
core = self.cache.tree_core
|
||||
return (
|
||||
core.get_component_device_lock_ref(node, FULL),
|
||||
core.get_component_host_lock_ref(node, FULL),
|
||||
core.get_component_host_lock_ref(node, SWA),
|
||||
)
|
||||
|
||||
def _host_pressure(self):
|
||||
"""A host-only match, a host pool with almost no room, and an unbacked
|
||||
device victim whose write-back needs host room."""
|
||||
self._cache(write_policy="write_back", tokens=64)
|
||||
match_tokens = list(range(1, 17))
|
||||
_, expected = self._insert(match_tokens, 100)
|
||||
self.cache.evict(EvictParams(num_tokens=len(match_tokens)))
|
||||
host = self.cache.host_pool_group
|
||||
filler = 1000
|
||||
while host.available_size() >= 2 * len(match_tokens):
|
||||
tokens = list(range(filler, filler + 16))
|
||||
self._insert(tokens, 300)
|
||||
self.cache.evict(EvictParams(num_tokens=len(tokens)))
|
||||
filler += 100
|
||||
self._insert(list(range(5000, 5040)), 500)
|
||||
return match_tokens + list(range(17, 25)), expected
|
||||
|
||||
def test_host_match_stays_pinned_while_reclaim_evicts_host_leaves(self):
|
||||
tokens, expected = self._host_pressure()
|
||||
req = self._request(tokens)
|
||||
node = req.best_match_node
|
||||
evict_host = self.cache.evict_host
|
||||
pins_during_host_eviction = []
|
||||
|
||||
def recorded(*args, **kwargs):
|
||||
pins_during_host_eviction.append(self._locks(node)[1:])
|
||||
return evict_host(*args, **kwargs)
|
||||
|
||||
with mock.patch.object(self.cache, "evict_host", side_effect=recorded):
|
||||
verdict = self._adder().add_one_req(
|
||||
req, has_chunked_req=False, truncation_align_size=None
|
||||
)
|
||||
|
||||
self.assertIs(verdict, AddReqResult.CONTINUE)
|
||||
# Reclaim had to make host room, and the match was pinned meanwhile.
|
||||
self.assertTrue(pins_during_host_eviction)
|
||||
for full_pin, swa_pin in pins_during_host_eviction:
|
||||
self.assertGreater(full_pin, 0)
|
||||
self.assertGreater(swa_pin, 0)
|
||||
self.cache.ready_to_load_host_cache()
|
||||
self.cache.loading_check()
|
||||
# The CPU copy survived until the load, so the request sees its data.
|
||||
full, swa = self.rows.read(self.cache.prefix_device_indices(req))
|
||||
self.assertTrue(torch.equal(full, expected[0]))
|
||||
self.assertTrue(torch.equal(swa[-self.window :], expected[1][-self.window :]))
|
||||
# Only the request's own device lock is left once the load is acked.
|
||||
self.assertEqual(self._locks(node), (1, 0, 0))
|
||||
self.cache.sanity_check()
|
||||
|
||||
def test_shared_budget_reclaims_for_the_load_and_pending_demand(self):
|
||||
# FULL stays resident under a host-only SWA window.
|
||||
self._cache(write_policy="write_through", tokens=80)
|
||||
tokens = list(range(1, 17))
|
||||
_, expected = self._insert(tokens, 100)
|
||||
self.cache.flush_pending_backups()
|
||||
self.cache.writing_check()
|
||||
self.cache.evict(EvictParams(swa_num_tokens=len(tokens)))
|
||||
req = self._request(tokens + list(range(17, 21)))
|
||||
# A running request locks its whole path; only its SWA past the window
|
||||
# can be reclaimed.
|
||||
running = list(range(5000, 5040))
|
||||
running_ids, running_expected = self._insert(running, 500)
|
||||
self.cache.flush_pending_backups()
|
||||
self.cache.writing_check()
|
||||
running_node = self._match(running).last_device_node
|
||||
pin = self.cache.inc_lock_ref(running_node).to_dec_params()
|
||||
reclaimable_swa = self.cache.swa_evictable_size()
|
||||
# The batch's decode tokens are pending demand on both bands. With
|
||||
# them, the load fits the shared gap only after that SWA is reclaimed.
|
||||
decode = 6 * PAGE
|
||||
moves = []
|
||||
_record_moves(self.allocator, moves)
|
||||
|
||||
verdict = self._adder(num_mixed_decode_tokens=decode).add_one_req(
|
||||
req, has_chunked_req=False, truncation_align_size=None
|
||||
)
|
||||
|
||||
self.assertIs(verdict, AddReqResult.CONTINUE)
|
||||
self.assertEqual(len(self.controller.load_queue), 1)
|
||||
# The queued load blocks every page move. The batch's own extend and
|
||||
# the pending decode tokens still fit.
|
||||
moved = len(moves)
|
||||
own = self.allocator.alloc(PAGE)
|
||||
pending = self.allocator.alloc(decode)
|
||||
self.assertIsNotNone(own)
|
||||
self.assertIsNotNone(pending)
|
||||
self.assertEqual(len(moves), moved)
|
||||
# That room came from reclaiming the running request's out-of-window
|
||||
# SWA; its FULL rows and its window stay.
|
||||
self.assertLess(self.cache.swa_evictable_size(), reclaimable_swa)
|
||||
self._assert_rows(running_ids, running_expected, swa_tail=self.window)
|
||||
|
||||
self.cache.ready_to_load_host_cache()
|
||||
self.cache.loading_check()
|
||||
full, swa = self.rows.read(self.cache.prefix_device_indices(req))
|
||||
self.assertTrue(torch.equal(full, expected[0]))
|
||||
self.assertTrue(torch.equal(swa[-self.window :], expected[1][-self.window :]))
|
||||
self.allocator.free(torch.cat([own, pending]))
|
||||
self.cache.dec_lock_ref(running_node, pin)
|
||||
self.assertEqual(self.allocator.verify_byte_accounting(), [])
|
||||
self.cache.sanity_check()
|
||||
|
||||
|
||||
class TestWritePolicySwitch(_UnifiedHiCacheCase):
|
||||
"""Attaching storage with write_through over a write_back host tree."""
|
||||
|
||||
def _policy_state(self):
|
||||
cache, controller = self.cache, self.controller
|
||||
return (
|
||||
controller.write_policy,
|
||||
cache.write_through_threshold,
|
||||
cache.is_write_back,
|
||||
cache.prefetch_stop_policy,
|
||||
cache.enable_storage,
|
||||
controller.storage_backend,
|
||||
controller.storage_backend_type,
|
||||
)
|
||||
|
||||
def test_incompatible_host_tree_is_rejected_without_side_effects(self):
|
||||
self._cache(write_policy="write_back", tokens=64)
|
||||
self._insert(list(range(1, 9)), 100)
|
||||
self._insert(list(range(1, 17)), 200)
|
||||
# Write-back backs up only the evicted leaf: a host suffix below an
|
||||
# unbacked FULL parent.
|
||||
self.cache.evict(EvictParams(num_tokens=8))
|
||||
before = self._policy_state()
|
||||
|
||||
with mock.patch.object(
|
||||
self.controller, "attach_storage_backend"
|
||||
) as attach_backend:
|
||||
ok, message = self.cache._storage_attachment.attach(
|
||||
"mori", hicache_write_policy="write_through"
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("from write_back to write_through", message)
|
||||
attach_backend.assert_not_called()
|
||||
self.assertEqual(self._policy_state(), before)
|
||||
self.cache.sanity_check()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -911,21 +911,19 @@ class TestUnifiedRadixAllocationEvictionRealComponents(CustomTestCase):
|
||||
for session in _session_radix_cache_test_values():
|
||||
for pinned in (False, True):
|
||||
with self.subTest(session=session, pinned=pinned):
|
||||
# This exercises the Rust backup barrier, independently of
|
||||
# the shared suite's default backend.
|
||||
# Rust independently of the shared suite's default backend;
|
||||
# session caches fall back to the Python TreeCore.
|
||||
with mock.patch(f"{__name__}._TREE_CORE_TEST_BACKEND", "rust"):
|
||||
cache, first, second, leaf = self._build_internal_chain(
|
||||
ct, session
|
||||
)
|
||||
cache.tree_core.is_write_back = True
|
||||
cache.tree_core.has_swa_host_pool = True
|
||||
cache.tree_core.enable_swa_write_back_eviction_barrier()
|
||||
if session:
|
||||
# Session caches use main's Python fallback, whose
|
||||
# component backup is gated by the HiCache attachment.
|
||||
cache.tree_core.enable_hicache = True
|
||||
cache.host_pool_group = mock.Mock()
|
||||
cache.host_pool_group.get_pool.return_value = None
|
||||
# Both cores gate the internal SWA component backup on the
|
||||
# HiCache attachment.
|
||||
cache.tree_core.set_hicache_enabled()
|
||||
cache.host_pool_group = mock.Mock()
|
||||
cache.host_pool_group.get_pool.return_value = None
|
||||
if pinned:
|
||||
receipt = cache.inc_host_lock_ref(first).to_dec_params()
|
||||
tracker = {ComponentType.FULL: 0, ct: 0}
|
||||
|
||||
Reference in New Issue
Block a user