[unified memory pool] Allocate without transfer waits, pin floating SWA reservations, drop the HiCache startup asserts (#43206)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-09 20:19:30 -07:00
committed by GitHub
co-authored by Hanming Lu ZYHowell
parent a65f4ffba5
commit f2e9a2bed1
23 changed files with 749 additions and 565 deletions
@@ -659,15 +659,6 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"preserve unified page-envelope indices and compaction lifetimes. "
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
)
if cfg.enable_hierarchical_cache:
assert cfg.pp_size == 1, (
"--enable-unified-memory with hierarchical cache does not support "
"pipeline parallelism (--pp-size > 1)."
)
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
"--enable-unified-memory with hierarchical cache requires lazy "
"compaction so pending H2D physical reservations remain stable."
)
assert not (
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
and cfg.disaggregation_decode_retraction_backup == "host_pool"
@@ -1357,7 +1357,8 @@ class PrefillAdder:
):
full_load_tokens = req.host_hit_length
if (
isinstance(self.tree_cache, UnifiedRadixCache)
self.memory_budget.load_back_needs_full_tokens
and isinstance(self.tree_cache, UnifiedRadixCache)
and self.tree_cache.buffer_pipeline is None
and not (
self.tree_cache.linker is not None
@@ -335,33 +335,6 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
lazy_compaction=self.lazy_compaction,
)
def bind_swa_for_loaded_rows(
self, full_token_ids: torch.Tensor
) -> Optional[torch.Tensor]:
"""Bind SWA pages to resident or newly loaded full-attention virtual IDs.
Bind before translating: unbound pages translate to the padding sink.
Return physical IDs, or None if capacity cannot be reclaimed.
"""
ids = full_token_ids.to(torch.int64)
if ids.numel() == 0:
return ids
ps = self.page_size
pages = torch.unique(ids // ps)
# Tombstones (-1) and the padding sink (0) both need a physical binding.
unbound = pages[self.swa_attn_allocator.virtual_to_physical[pages] <= 0]
need = int(unbound.numel()) * ps
if need:
if need > self.swa_available_size():
return None
if (
need > self.swa_attn_allocator.available_size()
and not _relieve_for_alloc(self.swa_attn_allocator, need)
):
return None
self.swa_attn_allocator.alloc_with_virtual(unbound)
return self.translate_loc_from_full_to_swa(ids)
def set_host_transfer_move_gate(self, gate: Callable[[], bool]) -> None:
"""Block page relocation while host transfers use resolved device indices."""
install_move_gate(
@@ -953,8 +953,6 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
with record_function("MultiEndedAlloc._alloc_bind_fast_or_slow"):
if N == 0:
return torch.empty(0, dtype=torch.int64, device=self.device)
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
self._wait_hicache_transfers()
# FAST PATH: eager, or lazy with no current holes.
if not self.lazy_compaction or self._free_phys_pages.numel() == 0:
@@ -2014,6 +2012,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self._pending_hicache_load_pages = 0
def _wait_hicache_transfers(self) -> None:
"""Order the current stream after outstanding HiCache copies. Only moves
and frees need this: a copy's source and target are not free pages until
its finish event is observed, so allocation never reuses them."""
if not self._hicache_transfer_done_events:
return
current_stream = torch.cuda.current_stream()
@@ -2041,26 +2042,23 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
if need_size > self.available_size():
if not _relieve_for_alloc(self, need_size):
return None
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
self._wait_hicache_transfers()
physical_pages = self.take_physical_pages(need_size // self.page_size)
if physical_pages is None:
return None
if self.lazy_compaction:
self._pending_hicache_load_pages += int(physical_pages.shape[0])
# Blocks page moves (`moves_blocked`) until the H2D is queued or cancelled.
self._pending_hicache_load_pages += int(physical_pages.shape[0])
return self._expand_pages_to_tokens(physical_pages)
def cancel_physical_reservation(self, free_index: torch.Tensor) -> None:
"""Roll back a HiCache physical allocation before its H2D is submitted."""
if free_index is None or free_index.numel() == 0:
return
if self.lazy_compaction:
num_pages = free_index.numel() // self.page_size
assert num_pages <= self._pending_hicache_load_pages, (
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
)
self._pending_hicache_load_pages -= num_pages
num_pages = free_index.numel() // self.page_size
assert num_pages <= self._pending_hicache_load_pages, (
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
)
self._pending_hicache_load_pages -= num_pages
self.free_physical(free_index)
def free_physical(self, free_index: torch.Tensor) -> None:
@@ -28,7 +28,6 @@ from __future__ import annotations
import logging
from array import array
from collections import deque
from dataclasses import replace
from typing import TYPE_CHECKING, Optional
import msgspec
@@ -1532,51 +1531,6 @@ class BufferModePipeline:
),
0,
)
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
binds_swa_to_full = (
swa_entry is not None
and swa_entry.device_indices_from_anchor_fn is not None
)
repair_ranges = []
if binds_swa_to_full and staged_swa:
window_start = span_end - staged_swa
repair_end = min(splice_base, span_end)
if window_start < repair_end:
repair_ranges = cache.tree_core.swa_tombstone_ranges(
key, window_start, repair_end
)
# Only missing SWA rows belong to this load. Existing bindings may
# be in use by another request and must survive allocation rollback.
anchor_parts = []
host_parts = []
for repair_start, repair_end_ in repair_ranges:
anchor_parts.append(prefix_indices[repair_start:repair_end_])
host_parts.append(
slice(repair_start - window_start, repair_end_ - window_start)
)
tail_start = max(splice_base, window_start)
if tail_start < span_end:
anchor_parts.append(
slice(tail_start - splice_base, span_end - splice_base)
)
host_parts.append(
slice(tail_start - window_start, span_end - window_start)
)
for i, transfer in enumerate(load_xfers):
if transfer.name != PoolName.SWA:
continue
# Keep the original complete host bounce for ack/drop release.
load_xfers[i] = replace(
transfer,
host_indices=(
torch.cat([transfer.host_indices[part] for part in host_parts])
if host_parts
else transfer.host_indices[:0]
),
anchor_index_parts=anchor_parts,
)
if not anchor_parts:
load_xfers = [t for t in load_xfers if t.name != PoolName.SWA]
device_indices = cc.load(
host_indices=f.host_indices[trim_tokens:],
@@ -1605,21 +1559,10 @@ class BufferModePipeline:
None,
)
aux_device_releases: list[tuple[PoolName, torch.Tensor]] = []
if swa_dev is not None and binds_swa_to_full:
# Binding already updated the allocator's virtual-to-physical table.
# The tree owns virtual FULL rows, not the H2D kernel-facing IDs.
for repair_start, repair_end_ in repair_ranges:
for action in cache.tree_core.attach_swa_window(
key,
repair_start,
repair_end_,
prefix_indices[repair_start:repair_end_],
):
cache._apply_cache_action(action)
elif swa_dev is not None:
# Register the window's FULL->SWA translation now (attention reads
# through it). Keep SWA slots another request may still hold; their
# redundant H2D destinations are reclaimed at the transfer ack.
if swa_dev is not None:
# SWA loads into fresh reservations. Register the window's FULL->SWA
# translation now (attention reads through it); slots another request
# still holds are kept, and their redundant destinations freed at ack.
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
allocator = cache.token_to_kv_pool_allocator
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
@@ -115,9 +115,6 @@ class PoolTransfer:
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
nodes_to_load: Optional[List[Any]] = None
indices_from_pool: Optional[PoolName] = None
# Full IDs backing a dependent device allocation: resident tensors or
# slices of the full rows allocated by this load, in transfer order.
anchor_index_parts: Optional[List[torch.Tensor | slice]] = None
@dataclass(frozen=True)
@@ -1719,7 +1719,6 @@ class HybridCacheController(BaseHiCacheController):
return None
newly_allocated: list[tuple[PoolTransfer, Callable, torch.Tensor]] = []
derived_transfers: list[PoolTransfer] = []
anchor_transfers = []
def rollback_allocated() -> None:
for prev_pool, prev_free_fn, prev_indices in reversed(newly_allocated):
@@ -1735,11 +1734,6 @@ class HybridCacheController(BaseHiCacheController):
continue
if pool.device_indices is not None or pool.host_indices is None:
continue
if entry.device_indices_from_anchor_fn is not None:
# Allocate independent pools first: their allocation/eviction
# can compact SWA before its physical IDs are captured.
anchor_transfers.append((pool, entry))
continue
# device_alloc_fn / device_free_fn override entry.device_pool's
# methods for pools whose device_pool is a raw KV pool (layout)
# rather than an allocator (e.g. SWA).
@@ -1758,28 +1752,6 @@ class HybridCacheController(BaseHiCacheController):
pool.device_indices = indices
newly_allocated.append((pool, free_fn, indices))
for pool, entry in anchor_transfers:
if kv_device_indices is None or not pool.anchor_index_parts:
rollback_allocated()
return None
anchor_indices = torch.cat(
[
kv_device_indices[part] if isinstance(part, slice) else part
for part in pool.anchor_index_parts
]
)
assert len(anchor_indices) == len(pool.host_indices)
bind = entry.device_indices_from_anchor_fn
indices = bind(anchor_indices)
if indices is None and entry.device_evict_fn:
entry.device_evict_fn(len(anchor_indices))
indices = bind(anchor_indices)
if indices is None:
rollback_allocated()
return None
pool.device_indices = indices
newly_allocated.append((pool, entry.device_free_fn, anchor_indices))
# Assign indices to deferred pools from their source.
for pool in derived_transfers:
if pool.indices_from_pool == PoolName.KV:
@@ -210,7 +210,6 @@ def build_pool_entry(
device_evict_fn: Optional[Callable[[int], Any]] = None,
device_alloc_fn: Optional[Callable[[int], Any]] = None,
device_free_fn: Optional[Callable[[Any], Any]] = None,
device_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
packed_draft_device_pools: tuple[Any, ...] = (),
) -> PoolEntry:
return PoolEntry(
@@ -223,7 +222,6 @@ def build_pool_entry(
device_evict_fn=device_evict_fn,
device_alloc_fn=device_alloc_fn,
device_free_fn=device_free_fn,
device_indices_from_anchor_fn=device_indices_from_anchor_fn,
packed_draft_device_pools=packed_draft_device_pools,
)
@@ -271,21 +269,16 @@ def build_kv_only_group(
)
def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
def _swa_allocation_callbacks(allocator) -> dict:
"""Keep allocation and rollback in the same ID space for every SWA stack."""
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
if isinstance(allocator, MultiEndedAllocator):
# Unified SWA loads into physical reservations, bound before the H2D.
return dict(
device_alloc_fn=allocator.alloc_physical,
device_free_fn=allocator.cancel_physical_reservation,
)
if bind is not None:
assert free_bound is not None
return dict(
device_indices_from_anchor_fn=bind,
device_free_fn=free_bound,
)
if allocator is None:
return {}
return dict(device_alloc_fn=allocator.alloc, device_free_fn=allocator.free)
@@ -327,8 +320,6 @@ def build_hybrid_swa_group(
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
swa_attn_allocator: Any = None,
swa_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
swa_free_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
mtp_swa_device_pools: tuple[Any, ...] = (),
) -> HostPoolGroup:
"""Anchor (full) + SWA host pool group for a hybrid-SWA device pool."""
@@ -394,11 +385,7 @@ def build_hybrid_swa_group(
transfer_layer_id_max=transfer_layer_id_max + len(mtp_swa_device_pools),
host_evict_fn=host_swa_evict_fn,
device_evict_fn=device_swa_evict_fn,
**_swa_allocation_callbacks(
swa_attn_allocator,
swa_indices_from_anchor_fn,
swa_free_from_anchor_fn,
),
**_swa_allocation_callbacks(swa_attn_allocator),
packed_draft_device_pools=mtp_swa_device_pools,
),
]
@@ -499,17 +486,6 @@ def build_hybrid_swa_stack(
device_swa_evict_fn=device_swa_evict_fn,
# For SWA hybrid, device allocation goes through the inner allocator.
swa_attn_allocator=params.token_to_kv_pool_allocator.swa_attn_allocator,
# Unified SWA binds pages to the full pool's virtual IDs instead.
swa_indices_from_anchor_fn=(
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
if get_memory().enable_unified_memory
else None
),
swa_free_from_anchor_fn=(
params.token_to_kv_pool_allocator.free_swa
if get_memory().enable_unified_memory
else None
),
mtp_swa_device_pools=mtp_swa_device_pools,
)
cache_controller = HybridCacheController(
@@ -1329,15 +1305,7 @@ def build_hybrid_mamba_swa_stack(
transfer_layer_id_max=transfer_layer_id_max,
host_evict_fn=host_swa_evict_fn,
device_evict_fn=device_swa_evict_fn,
**_swa_allocation_callbacks(
swa_attn_allocator,
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
if get_memory().enable_unified_memory
else None,
params.token_to_kv_pool_allocator.free_swa
if get_memory().enable_unified_memory
else None,
),
**_swa_allocation_callbacks(swa_attn_allocator),
),
build_pool_entry(
name=PoolName.MAMBA,
@@ -25,10 +25,6 @@ class PoolEntry:
device_evict_fn: Callable[[int], Any] | None = None
device_alloc_fn: Callable[[int], Any] | None = None
device_free_fn: Callable[[Any], Any] | None = None
# Bind rows to the anchor's virtual IDs when pools share an ID space.
# Return buffer indices, or None if allocation fails. Rollback through
# device_free_fn takes the anchor's virtual IDs, not the returned indices.
device_indices_from_anchor_fn: Callable[[Any], Any] | None = None
packed_draft_device_pools: tuple[Any, ...] = ()
@@ -62,6 +62,9 @@ def estimate_swa_kv_tokens(
class PrefillBudget:
"""Fixed token pool. Offsets include pending allocations and decode headroom."""
# When False, admission skips the FULL spec and passes the host hit length.
load_back_needs_full_tokens = False
def __init__(self, allocator, tree_cache, *, num_mixed_decode_tokens: int = 0):
self.allocator = allocator
self.tree_cache = tree_cache
@@ -315,6 +318,8 @@ class SWAPrefillBudget(PrefillBudget):
class SharedSWAPrefillBudget(SWAPrefillBudget):
"""FULL and SWA reservations compete for the same physical byte budget."""
load_back_needs_full_tokens = True
def __init__(self, *args, num_mixed_decode_tokens=0, **kwargs):
super().__init__(
*args, num_mixed_decode_tokens=num_mixed_decode_tokens, **kwargs
@@ -568,11 +568,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
result = EvictDeviceNextNodeResult(
node_id=binding_result.node_id,
made_progress=binding_result.made_progress,
backup_kv=(
_cache_action_from_tagged(binding_result.backup_kv)
if binding_result.backup_kv is not None
else None
),
unbacked_tokens=binding_result.unbacked_tokens,
mamba_backup_node_id=binding_result.mamba_backup_node_id,
swa_backup_node_id=binding_result.swa_backup_node_id,
@@ -788,9 +783,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def set_hicache_enabled(self) -> None:
self._binding.set_hicache_enabled()
def enable_swa_write_back_eviction_barrier(self) -> None:
self._binding.enable_swa_write_back_eviction_barrier()
def set_host_memory_buffer_only(self) -> None:
self._binding.set_host_memory_buffer_only()
@@ -961,27 +953,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
kv_xfer = _transfer_from_binding(kv_xfer)
comp_xfers = _comp_xfers_from_binding(comp_xfers)
swa_xfers = comp_xfers.get(ComponentType.SWA, ())
if swa_xfers:
full_node_ids = kv_xfer.nodes_to_load or []
full_load_slices = {}
offset = 0
for full_node_id, count in zip(
full_node_ids, self._binding.get_node_key_lengths(full_node_ids)
):
full_load_slices[full_node_id] = slice(offset, offset + count)
offset += count
for transfer in swa_xfers:
# SWA may have holes between resident nodes, or reload while
# FULL stays resident. Preserve the SWA transfer's node order.
transfer.anchor_index_parts = [
(
full_load_slices[nid]
if nid in full_load_slices
else self.get_component_device_value(nid, ComponentType.FULL)
)
for nid in transfer.nodes_to_load or ()
]
return kv_xfer, comp_xfers
def prefetch_anchor_info(
@@ -2494,21 +2494,6 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
nodes_to_load=[],
)
return empty_kv, {}
# SWA can be evicted independently of FULL, including holes between
# resident SWA nodes. Describe precisely which full rows back it.
full_load_slices = {}
offset = 0
for nid in kv_xfer.nodes_to_load or ():
count = len(self.node_by_id(nid).key)
full_load_slices[nid] = slice(offset, offset + count)
offset += count
for xfer in comp_xfers.get(ComponentType.SWA, ()):
xfer.anchor_index_parts = [
full_load_slices[nid]
if nid in full_load_slices
else self.node_by_id(nid).component_data[BASE_COMPONENT_TYPE].value
for nid in xfer.nodes_to_load or ()
]
return kv_xfer, comp_xfers
def prefetch_anchor_info(
@@ -52,7 +52,6 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
"""
node_id: Optional[NodeId] = None
backup_kv: Optional[BackupKV] = None
made_progress: bool = False
unbacked_tokens: int = 0
mamba_backup_node_id: Optional[NodeId] = None
@@ -523,13 +522,6 @@ class UnifiedTreeCoreInterface(ABC):
"""Mark the host tier (HiCache) as wired."""
...
def enable_swa_write_back_eviction_barrier(self) -> None:
"""Enable a backend-managed barrier when needed.
The Python core demotes through SWAComponent directly. Native cores
may return a backup action to the cache executor before eviction.
"""
@abstractmethod
def set_host_memory_buffer_only(self) -> None:
"""Mark the host tier as buffer-only: one node staged per backup intent."""
@@ -532,13 +532,6 @@ class UnifiedRadixCache(BasePrefixCache):
self.cache_controller is not None
and self.cache_controller.write_policy == "write_back"
)
# Preserve the SWA host window before device eviction makes it unrecoverable.
if (
get_memory().enable_unified_memory
and self.host_memory_mode == "cache"
and self.tree_core.has_swa_host_pool
):
self.tree_core.enable_swa_write_back_eviction_barrier()
# Pre-seed the logical dropped-tokens series.
if self.metrics_collector is not None and self.cache_controller is not None:
reasons = ["host_pressure"]
@@ -805,74 +798,45 @@ class UnifiedRadixCache(BasePrefixCache):
def _evict_device_next_node(
self, component_type: ComponentType, tracker: dict[ComponentType, int]
) -> tuple[Optional[NodeId], bool]:
"""Advance the walk, completing pending host-backup barriers."""
while True:
result = self.tree_core.evict_device_next_node(component_type, tracker)
if result.mamba_backup_node_id is not None:
assert component_type == ComponentType.MAMBA and result.node_id is None
assert (
not result.device_frees
and not result.host_frees
and not result.tracker
)
# Reserve a host state slot and wait for the backup acknowledgment
# before freeing device state. If allocation fails, eviction still
# proceeds to make room on the device.
node_id = result.mamba_backup_node_id
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
self.evict_host(1, ComponentType.MAMBA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_mamba_state_eviction(node_id)
elif result.swa_backup_node_id is not None:
assert component_type == ComponentType.SWA and result.node_id is None
assert (
not result.device_frees
and not result.host_frees
and not result.tracker
)
# The backup can cover several unbacked SWA segments. Reserve host
# space for the whole window before copying it, then resume eviction
# even if host allocation fails.
node_id = result.swa_backup_node_id
needed = result.swa_backup_num_tokens
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
if (
swa_host_pool is not None
and swa_host_pool.available_size() < needed
):
self.evict_host(needed, ComponentType.SWA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
result.unbacked_tokens,
reason="write_through_unbacked_eviction",
)
self._accumulate_tracker(tracker, result.tracker)
if result.backup_kv is None:
return result.node_id, result.made_progress
assert result.node_id is None
assert self.buffer_pipeline is None, (
"SWA write-back eviction barriers are cache-mode only"
"""Advance the eviction walk one node, consuming its step result."""
result = self.tree_core.evict_device_next_node(component_type, tracker)
if result.mamba_backup_node_id is not None:
assert component_type == ComponentType.MAMBA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
)
written = self._execute_and_commit_kv_backup(
result.backup_kv, write_back=True
# Reserve a host state slot and wait for the backup acknowledgment
# before freeing device state. If allocation fails, eviction still
# proceeds to make room on the device.
node_id = result.mamba_backup_node_id
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
self.evict_host(1, ComponentType.MAMBA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_mamba_state_eviction(node_id)
elif result.swa_backup_node_id is not None:
assert component_type == ComponentType.SWA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
)
if written <= 0:
node_id = result.backup_kv.node_ids[0]
logger.warning(
"write_back: auxiliary backup failed under host pressure "
"(component=%s, node=%d); dropping only the component",
component_type.name,
node_id,
)
# Match the Python SWA demotion: preserve FULL and descendants;
# the resumed native walk tombstones this component after one try.
continue
self.writing_check(write_back=True)
# The backup can cover several unbacked SWA segments. Reserve host
# space for the whole window before copying it, then resume eviction
# even if host allocation fails.
node_id = result.swa_backup_node_id
needed = result.swa_backup_num_tokens
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
self.evict_host(needed, ComponentType.SWA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
result.unbacked_tokens,
reason="write_through_unbacked_eviction",
)
self._accumulate_tracker(tracker, result.tracker)
return result.node_id, result.made_progress
def _evict_device_leaf(
self, node_id: NodeId, tracker: dict[ComponentType, int]
@@ -802,19 +802,6 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
break 'step None;
}
}
if tree_core.is_write_back
&& tree_core.swa_write_back_eviction_barrier_enabled
&& tree_core.component_state(SWA).evict_device_last_backup != Some(x)
&& !tree_core.arena.node(x).backuped()
{
// A later Full backup cannot recover SWA data after this
// internal node is tombstoned. Pause on the same cursor so
// the Controller can preserve the dirty path first.
tree_core.component_state_mut(SWA).evict_device_backup_node = Some(x);
tree_core.component_state_mut(SWA).evict_device_last_backup = Some(x);
cursor = Some(x);
break 'step None;
}
// Internal nodes are tombstoned inline (no IO).
tree_core.evict_component_and_detach_lru_(
x,
@@ -847,7 +847,6 @@ fn tracker_to_py(tracker: HashMap<ComponentType, usize>) -> HashMap<u8, usize> {
#[pyclass(get_all)]
pub struct EvictDeviceNextNodeResultBinding {
node_id: Option<NodeId>,
backup_kv: Option<Py<PyAny>>,
mamba_backup_node_id: Option<NodeId>,
swa_backup_node_id: Option<NodeId>,
swa_backup_num_tokens: usize,
@@ -1431,16 +1430,11 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
let (node_id, result) =
py.allow_threads(move || self.core().evict_device_next_node(ct, &baseline));
let made_progress = node_id.is_some()
|| result.backup_kv.is_some()
|| result.mamba_backup_node_id.is_some()
|| result.swa_backup_node_id.is_some()
|| !result.tracker.is_empty();
Ok(EvictDeviceNextNodeResultBinding {
node_id,
backup_kv: result
.backup_kv
.map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup)))
.transpose()?,
mamba_backup_node_id: result.mamba_backup_node_id,
swa_backup_node_id: result.swa_backup_node_id,
swa_backup_num_tokens: result.swa_backup_num_tokens,
@@ -1479,7 +1473,6 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
) -> PyResult<EvictDeviceNextNodeResultBinding> {
Ok(EvictDeviceNextNodeResultBinding {
node_id: None,
backup_kv: None,
mamba_backup_node_id: None,
swa_backup_node_id: None,
swa_backup_num_tokens: 0,
@@ -1649,11 +1642,6 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
py.allow_threads(|| self.core().enable_hicache)
}
/// Preserve dirty SWA data before cache-mode write-back eviction.
fn enable_swa_write_back_eviction_barrier(&self, py: Python<'_>) {
py.allow_threads(|| self.core().enable_swa_write_back_eviction_barrier());
}
/// Mark the SWA host pool as wired (HiCache).
fn set_has_swa_host_pool(&self, py: Python<'_>) {
py.allow_threads(|| self.core().set_has_swa_host_pool());
@@ -3067,11 +3055,6 @@ macro_rules! tree_core_binding {
catch_native_panic(|| Ok(self.inner.enable_hicache(py)))
}
/// Preserve dirty SWA data before cache-mode write-back eviction.
fn enable_swa_write_back_eviction_barrier(&self, py: Python<'_>) {
self.inner.enable_swa_write_back_eviction_barrier(py)
}
/// Mark the SWA host pool as wired (HiCache).
fn set_has_swa_host_pool(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| {
@@ -509,11 +509,6 @@ pub struct ComponentState {
/// leaf may be freed: the leaf's parent for Full, the LRU predecessor for
/// SWA and Mamba.
pub(crate) evict_device_cursor: Option<NodeIdx_>,
/// Internal node whose component value must be backed up before the walk
/// can tombstone it. The Controller consumes this request between steps.
pub(crate) evict_device_backup_node: Option<NodeIdx_>,
/// A resumed victim is tombstoned after its best-effort backup attempt.
pub(crate) evict_device_last_backup: Option<NodeIdx_>,
/// Internal component victim waiting for the controller's host backup attempt.
/// A generation-checked handle survives host eviction during that I/O.
pub(crate) evict_device_pending_node: Option<NodeId>,
@@ -587,7 +582,6 @@ pub struct EvictionStepResult {
pub tracker: HashMap<ComponentType, usize>,
pub device_frees: HashMap<ComponentType, Vec<Tensor>>,
pub host_frees: HashMap<ComponentType, Vec<Tensor>>,
pub backup_kv: Option<BackupKV>,
/// Full device tokens freed without a host copy during this device step.
pub unbacked_tokens: usize,
/// Back up this internal Mamba state before resuming its device tombstone.
@@ -641,8 +635,6 @@ pub struct UnifiedTreeCore<K: ChildKeyType> {
pub(crate) enable_external_cache_linker: bool,
/// Whether the cache wired a host SWA pool (HiCache).
pub(crate) has_swa_host_pool: bool,
/// Whether dirty internal SWA nodes must be backed up before eviction.
pub(crate) swa_write_back_eviction_barrier_enabled: bool,
/// Whether tree mutations emit BlockStored/BlockRemoved events.
pub(crate) enable_kv_cache_events: bool,
/// Queued placement events, drained by take_events.
@@ -755,8 +747,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
state.is_evict_device_ongoing = true;
state.evict_device_request_cnt = request_cnt;
state.evict_device_cursor = None;
state.evict_device_backup_node = None;
state.evict_device_last_backup = None;
state.evict_device_pending_node = None;
state.evict_device_pending_num_tokens = 0;
}
@@ -771,8 +761,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
);
state.is_evict_device_ongoing = false;
state.evict_device_cursor = None;
state.evict_device_backup_node = None;
state.evict_device_last_backup = None;
state.evict_device_pending_node = None;
state.evict_device_pending_num_tokens = 0;
}
@@ -841,7 +829,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
enable_storage: false,
enable_external_cache_linker: false,
has_swa_host_pool: params.has_swa_host_pool,
swa_write_back_eviction_barrier_enabled: false,
enable_kv_cache_events: params.enable_kv_cache_events,
kv_event_queue: Vec::new(),
namespaced_event_hashes: HashMap::new(),
@@ -2397,15 +2384,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
&mut result.device_frees,
&mut result.host_frees,
);
let backup_node = self
.component_state_mut(component_type)
.evict_device_backup_node
.take();
if let Some(backup_node) = backup_node {
assert!(node_id.is_none());
result.backup_kv =
Some(self.build_backup_kv_action_(self.arena.node(backup_node), true));
}
result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap();
let state = self.component_state(component_type);
if component_type == MAMBA {
@@ -3161,11 +3139,6 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
self.enable_hicache = true;
}
/// Preserve dirty internal SWA nodes before cache-mode write-back eviction.
pub fn enable_swa_write_back_eviction_barrier(&mut self) {
self.swa_write_back_eviction_barrier_enabled = true;
}
/// Mark the host tier as buffer-only; wired after the host pools are built.
pub fn set_host_memory_buffer_only(&mut self) {
self.is_host_memory_buffer_only = true;
@@ -29,6 +29,7 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_require_single_row_dsv4_swa_pages,
_split_hicache_size,
_SwaStrategy,
_uses_unified_page_envelope_host,
build_full_draft_pools,
build_host_pool_group,
build_hybrid_swa_group,
@@ -1245,5 +1246,40 @@ class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
self.assertIsNotNone(swa_pool.alloc(page_size))
class TestUnifiedPageEnvelopeSelection(CustomTestCase):
"""Only no backend and Mori store the complete shared page, so only they
keep the page-envelope host arena."""
def test_only_no_backend_and_mori_select_the_envelope(self):
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
pool = _build_unified_swa_pool().token_to_kv_pool
self.addCleanup(reset_context)
for backend, expected in (
(None, True),
("mori", True),
("mooncake", False),
("dynamic", False),
):
with self.subTest(backend=backend):
reset_context()
publish(
ServerArgs(
model_path="dummy",
enable_unified_memory=True,
hicache_storage_backend=backend,
hicache_mem_layout="layer_first",
),
role="tokenizer",
)
self.assertEqual(
_uses_unified_page_envelope_host(
pool.full_kv_pool, pool.swa_kv_pool, use_mla=False
),
expected,
)
if __name__ == "__main__":
unittest.main()
@@ -2192,12 +2192,7 @@ def test_swa_host_pressure_retains_device_resident_backups(backend, guard):
@pytest.mark.parametrize("backend", ["python", "rust"])
@pytest.mark.parametrize("resident_full", [(), (0,), (0, 1, 2, 3)])
def test_swa_load_back_preserves_full_anchors_across_holes(backend, resident_full):
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
)
from sglang.srt.mem_cache.pool_host.group import PoolEntry
def test_swa_load_back_spec_skips_resident_swa_holes(backend, resident_full):
core, _ = _swa_transfer_core(backend)
nodes = []
boundaries = (0, 2, 5, 6, 8)
@@ -2241,52 +2236,10 @@ def test_swa_load_back_preserves_full_anchors_across_holes(backend, resident_ful
)
core.finish_load_back(nodes[index])
kv, auxiliary = core.build_load_back_spec(nodes[-1])
_, auxiliary = core.build_load_back_spec(nodes[-1])
(swa,) = auxiliary[ComponentType.SWA]
assert swa.nodes_to_load == [nodes[0], nodes[2], nodes[3]]
assert swa.host_indices.tolist() == [300, 301, 305, 306, 307]
expected = {
(): [slice(0, 2), slice(5, 6), slice(6, 8)],
(0,): [torch.tensor([10, 11]), slice(3, 4), slice(4, 6)],
(0, 1, 2, 3): [
torch.tensor([10, 11]),
torch.tensor([15]),
torch.tensor([16, 17]),
],
}[resident_full]
assert swa.anchor_index_parts is not None
assert len(swa.anchor_index_parts) == len(expected)
for actual, wanted in zip(swa.anchor_index_parts, expected):
if isinstance(wanted, slice):
assert actual == wanted
else:
torch.testing.assert_close(actual, wanted)
# Exercise the consumer too: new Full rows and resident virtual IDs must
# bind the same five SWA rows, without consuming the resident-SWA hole.
bind = Mock(side_effect=lambda indices: indices + 1000)
controller = object.__new__(HybridCacheController)
controller.mem_pool_host = SimpleNamespace(
entry_map={
PoolName.SWA: PoolEntry(
name=PoolName.SWA,
host_pool=SimpleNamespace(),
device_pool=SimpleNamespace(),
layer_mapper=lambda i: i,
device_indices_from_anchor_fn=bind,
device_free_fn=Mock(),
)
}
)
loaded = torch.arange(200, 200 + len(kv.host_indices))
assert controller._resolve_device_transfers([swa], loaded) is not None
expected_bound = {
(): [1200, 1201, 1205, 1206, 1207],
(0,): [1010, 1011, 1203, 1204, 1205],
(0, 1, 2, 3): [1010, 1011, 1015, 1016, 1017],
}[resident_full]
assert swa.device_indices.tolist() == expected_bound
assert bind.call_count == 1
@pytest.mark.parametrize("backend", ["python", "rust"])
@@ -86,7 +86,6 @@ def _staged_fixture(full_match=2):
entry_map={
PoolName.SWA: SimpleNamespace(
host_pool=SimpleNamespace(free=Mock()),
device_indices_from_anchor_fn=None,
)
},
)
@@ -7,15 +7,10 @@ from unittest.mock import MagicMock, Mock, patch
import torch
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
)
from sglang.srt.mem_cache.hicache_storage import PoolName
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.pool_host.group import PoolEntry
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small")
@@ -74,131 +69,6 @@ class TestHostGateCapacity(unittest.TestCase):
self.assertEqual(full.schedulable_available_size(), before)
class TestSwaLoadAllocation(unittest.TestCase):
def _controller(self, bind, free=None, evict=None):
controller = object.__new__(HybridCacheController)
entry = PoolEntry(
name=PoolName.SWA,
host_pool=SimpleNamespace(),
device_pool=SimpleNamespace(),
layer_mapper=lambda i: i,
device_indices_from_anchor_fn=bind,
device_free_fn=free or Mock(),
device_evict_fn=evict,
)
controller.mem_pool_host = SimpleNamespace(entry_map={PoolName.SWA: entry})
return controller
def _transfer(self, parts, count):
transfer = PoolTransfer(name=PoolName.SWA, host_indices=torch.arange(count))
transfer.anchor_index_parts = parts
return transfer
def test_swa_only_load_uses_resident_full_ids(self):
bind = Mock(side_effect=lambda x: x + 100)
controller = self._controller(bind)
transfer = self._transfer([torch.tensor([13, 14])], 2)
result = controller._resolve_device_transfers(
[transfer], torch.empty(0, dtype=torch.int64)
)
self.assertIsNotNone(result)
torch.testing.assert_close(transfer.device_indices, torch.tensor([113, 114]))
def test_mixed_load_skips_resident_swa_nodes(self):
bind = Mock(side_effect=lambda x: x + 100)
controller = self._controller(bind)
transfer = self._transfer([torch.tensor([13, 14]), slice(2, 4)], 4)
controller._resolve_device_transfers(
[transfer], torch.tensor([20, 21, 22, 23, 24, 25])
)
torch.testing.assert_close(
transfer.device_indices, torch.tensor([113, 114, 122, 123])
)
def test_binding_retries_after_eviction(self):
bind = Mock(side_effect=[None, torch.tensor([41, 42])])
evict = Mock()
controller = self._controller(bind, evict=evict)
transfer = self._transfer([slice(0, 2)], 2)
self.assertIsNotNone(
controller._resolve_device_transfers([transfer], torch.tensor([11, 12]))
)
evict.assert_called_once_with(2)
self.assertEqual(bind.call_count, 2)
def test_rollback_releases_swa_binding_by_virtual_ids(self):
free = Mock()
controller = self._controller(lambda x: x + 100, free=free)
transfer = self._transfer([torch.tensor([13, 14])], 2)
# A missing sidecar source fails after SWA has bound its pages.
sidecar = PoolTransfer(name=PoolName.MAMBA, indices_from_pool=PoolName.INDEXER)
result = controller._resolve_device_transfers(
[transfer, sidecar], torch.empty(0, dtype=torch.int64)
)
self.assertIsNone(result)
torch.testing.assert_close(free.call_args.args[0], torch.tensor([13, 14]))
self.assertIsNone(transfer.device_indices)
def test_independent_allocations_precede_kernel_id_resolution(self):
events = []
controller = self._controller(lambda x: events.append("bind") or x + 100)
controller.mem_pool_host.entry_map[PoolName.MAMBA] = PoolEntry(
name=PoolName.MAMBA,
host_pool=SimpleNamespace(),
device_pool=SimpleNamespace(),
layer_mapper=lambda i: i,
device_alloc_fn=lambda n: events.append("mamba") or torch.arange(n),
device_free_fn=Mock(),
)
swa = self._transfer([slice(0, 2)], 2)
mamba = PoolTransfer(name=PoolName.MAMBA, host_indices=torch.arange(1))
self.assertIsNotNone(
controller._resolve_device_transfers([swa, mamba], torch.tensor([10, 11]))
)
self.assertEqual(events, ["mamba", "bind"])
def test_tree_spec_preserves_node_correspondence(self):
kv = PoolTransfer(
name=PoolName.KV, host_indices=torch.arange(4), nodes_to_load=[2, 3]
)
swa = PoolTransfer(
name=PoolName.SWA, host_indices=torch.arange(4), nodes_to_load=[1, 3]
)
nodes = {
i: SimpleNamespace(
id=i,
key=[i, i],
load_back_pending_id=None,
component_data={
ComponentType.FULL: SimpleNamespace(
value=torch.tensor([10, 11]) if i == 1 else None
)
},
)
for i in (1, 2, 3)
}
full_component = SimpleNamespace(
component_type=ComponentType.FULL,
build_hicache_transfers=lambda *a, **k: [kv],
)
swa_component = SimpleNamespace(
component_type=ComponentType.SWA,
build_hicache_transfers=lambda *a, **k: [swa],
)
core = SimpleNamespace(
node_by_id=nodes.__getitem__,
components=[full_component, swa_component],
components_by_type={
ComponentType.FULL: full_component,
ComponentType.SWA: swa_component,
},
)
UnifiedTreeCore.build_load_back_spec(core, 3)
parts = swa.anchor_index_parts
torch.testing.assert_close(parts[0], torch.tensor([10, 11]))
self.assertEqual(parts[1], slice(2, 4))
@unittest.skipUnless(torch.cuda.is_available(), "CUDA required")
class TestTransferStreamOrdering(unittest.TestCase):
def test_load_translation_follows_supplied_start_event(self):
@@ -288,16 +158,20 @@ class TestDirectBackendTranslation(unittest.TestCase):
class TestTriPoolAssembly(unittest.TestCase):
def test_swa_allocation_and_rollback_match_pool_id_ownership(self):
from sglang.srt.mem_cache.allocator.unified_sub_pool import (
MultiEndedAllocator,
)
from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler as assembler
for unified in (False, True):
with self.subTest(unified=unified):
swa_allocator = SimpleNamespace(alloc=Mock(), free=Mock())
composite = SimpleNamespace(
swa_attn_allocator=swa_allocator,
bind_swa_for_loaded_rows=Mock(),
free_swa=Mock(),
# Unified memory always builds a MultiEndedAllocator SWA end.
swa_allocator = (
object.__new__(MultiEndedAllocator)
if unified
else SimpleNamespace(alloc=Mock(), free=Mock())
)
composite = SimpleNamespace(swa_attn_allocator=swa_allocator)
params = SimpleNamespace(
token_to_kv_pool_allocator=composite,
req_to_token_pool=SimpleNamespace(
@@ -330,16 +204,15 @@ class TestTriPoolAssembly(unittest.TestCase):
)
entry = group.entry_map[PoolName.SWA]
if unified:
self.assertIs(
entry.device_indices_from_anchor_fn,
composite.bind_swa_for_loaded_rows,
self.assertEqual(
entry.device_alloc_fn, swa_allocator.alloc_physical
)
self.assertEqual(
entry.device_free_fn, swa_allocator.cancel_physical_reservation
)
self.assertIs(entry.device_free_fn, composite.free_swa)
self.assertIsNone(entry.device_alloc_fn)
else:
self.assertIs(entry.device_alloc_fn, swa_allocator.alloc)
self.assertIs(entry.device_free_fn, swa_allocator.free)
self.assertIsNone(entry.device_indices_from_anchor_fn)
if __name__ == "__main__":
@@ -0,0 +1,626 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unified-memory HiCache on the real SWA allocator stack: physical reservations
and the moves they block, the transfer-done event waits, load-back admission,
and the write-policy switch check.
The pools are real CPU tensors. CUDA stream waits are recorded by a stand-in
for `torch.cuda.current_stream()` rather than executed.
python -m pytest test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py -v
"""
import atexit
import contextlib
import unittest
from array import array
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestHandle,
EvictParams,
InsertParams,
MatchPrefixParams,
)
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_swa_allocation_callbacks,
)
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.pool_host import common as host_memory
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.components.base import ComponentType
from sglang.srt.mem_cache.unified_cache.tree_core_registry import _TREE_CORE_REGISTRY
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.unified_allocator_fixtures import build_tri_pool
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
PAGE = 4
D2H = ("controller", "write")
H2D = ("controller", "load")
FULL, SWA = ComponentType.FULL, ComponentType.SWA
def _python_inspector(params, components):
from unified_tree_core_inspector import UnifiedTreeCoreInspector
return UnifiedTreeCoreInspector(params, components)
def _rust_inspector(params, _components):
from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector
return RustUnifiedTreeCoreInspector(params)
def _unified_swa_bundle(*, tokens: int = 64):
return init_unified_swa_pools(
device="cpu",
kv_cache_dtype=torch.float16,
head_num=1,
head_dim=8,
v_head_dim=8,
swa_head_num=1,
swa_head_dim=8,
swa_v_head_dim=8,
page_size=PAGE,
start_layer=0,
end_layer=2,
swa_attention_layer_ids=[1],
full_attention_layer_ids=[0],
full_max_total_num_tokens=tokens,
swa_max_total_num_tokens=tokens,
enable_memory_saver=False,
need_sort=False,
lazy_compaction=True,
)
def _unified_swa():
bundle = _unified_swa_bundle()
return bundle.token_to_kv_pool_allocator, _Rows(bundle)
class _Rows:
"""One marker per token row, read and written through the allocator's own
virtual-to-physical translation, so a relocated row is found where it went."""
def __init__(self, bundle):
self.allocator = bundle.token_to_kv_pool_allocator
self.full = bundle.token_to_kv_pool.full_kv_pool.get_key_buffer(0)
self.swa = bundle.token_to_kv_pool.swa_kv_pool.get_key_buffer(0)
def full_rows(self, ids: torch.Tensor) -> torch.Tensor:
return self.allocator.full_attn_allocator.translate_kv_loc(ids.to(torch.int64))
def swa_rows(self, ids: torch.Tensor) -> torch.Tensor:
return self.allocator.translate_loc_from_full_to_swa(ids)
def markers(self, base: int, count: int) -> torch.Tensor:
# One value per token row; float16 holds these integers exactly.
values = torch.arange(base, base + count, dtype=torch.float16)
return values[:, None, None].expand(count, *self.swa.shape[1:]).clone()
def seed(self, ids: torch.Tensor, base: int):
self.full[self.full_rows(ids)] = self.markers(base, len(ids))
self.swa[self.swa_rows(ids)] = self.markers(base + 1000, len(ids))
return self.read(ids)
def read(self, ids: torch.Tensor):
return (
self.full[self.full_rows(ids)].clone(),
self.swa[self.swa_rows(ids)].clone(),
)
class _Stream:
"""Stands in for `torch.cuda.current_stream()` and records what it waits on."""
def __init__(self, log: list):
self.log = log
def wait_event(self, event) -> None:
self.log.append(("wait", event))
def wait_stream(self, stream) -> None:
self.log.append(("wait_stream", stream))
@contextlib.contextmanager
def _recorded_stream():
log = []
with mock.patch.object(
torch.cuda, "current_stream", lambda *args, **kwargs: _Stream(log)
):
yield log
def _record_moves(allocator, log: list) -> None:
"""Log every compaction copy on either side into `log`, then perform it."""
for side in (allocator.full_attn_allocator, allocator.swa_attn_allocator):
kvcache = side._kvcache
move = kvcache.move_kv_cache
def recorded(dst, src, *, move=move, name=side.sub_pool_name):
log.append(("move", name))
return move(dst, src)
kvcache.move_kv_cache = recorded
class _ConfigCase(unittest.TestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
class TestPhysicalReservation(_ConfigCase):
"""A pending reservation blocks moves on the SWA end until its load is queued."""
def test_pending_reservation_blocks_moves_until_its_load_is_queued(self):
allocator, rows = _unified_swa()
swa = allocator.swa_attn_allocator
head = allocator.alloc(2 * PAGE)
gap = allocator.alloc(2 * PAGE)
tail = allocator.alloc(2 * PAGE)
expected_head = rows.seed(head, 100)
expected_tail = rows.seed(tail, 200)
# The node being loaded: its FULL rows are resident, SWA is tombstoned.
target = allocator.alloc(2 * PAGE)
allocator.free_swa(target)
allocator.free(gap)
reserved = swa.alloc_physical(2 * PAGE)
# Stand in for the H2D copy into the reserved rows.
rows.swa[reserved] = rows.markers(300, 2 * PAGE)
allocator.set_full_to_swa_mapping(target, reserved)
moves = []
_record_moves(allocator, moves)
with _recorded_stream() as waits:
# Not queued yet: nothing on the SWA end may move.
self.assertEqual(swa._flush(urgent=True), 0)
self.assertEqual(swa.flush_opportunistic(), 0)
self.assertEqual(moves, [])
load_done = object()
allocator.set_hicache_transfer_done_event(H2D, load_done)
# Queued: compaction waits on the copy, then relocates rows.
self.assertGreater(swa._flush(urgent=True), 0)
self.assertEqual(waits[0], ("wait", load_done))
self.assertIn(("move", "swa"), moves)
_, target_swa = rows.read(target)
self.assertTrue(torch.equal(target_swa, rows.markers(300, 2 * PAGE)))
self.assertTrue(torch.equal(rows.read(head)[1], expected_head[1]))
self.assertTrue(torch.equal(rows.read(tail)[1], expected_tail[1]))
self.assertEqual(allocator.verify_byte_accounting(), [])
class TestFloatSwaReservation(_ConfigCase):
"""The tri-pool's SWA middle floats: it relocates its pages to make room for
a neighbour, so a load's SWA reservation must pin it as on an end pool."""
def test_state_allocation_cannot_move_a_reservation_before_its_load_is_queued(
self,
):
bundle, allocator = build_tri_pool(page_size=PAGE)
full, swa = allocator.full_attn_allocator, allocator.swa_attn_allocator
states = bundle.req_to_token_pool.mamba_allocator
mamba = allocator.mamba_allocator
resident = allocator.alloc(4 * PAGE)
# The Mamba end is full: its next slot needs the float to move.
mamba.alloc(mamba.available_size())
# A window slides past its oldest page: a hole at the float's low edge.
allocator.free_swa(resident[:PAGE])
moves = []
_record_moves(allocator, moves)
# Only what `HybridCacheController.load` reads, with production callbacks.
controller = object.__new__(HybridCacheController)
controller.mem_pool_device_allocator = allocator
controller.device = "cpu"
controller.load_queue, controller.ack_load_queue = [], []
controller.mem_pool_host = SimpleNamespace(
entry_map={
PoolName.SWA: SimpleNamespace(
device_evict_fn=None, **_swa_allocation_callbacks(swa)
),
PoolName.MAMBA: SimpleNamespace(
device_alloc_fn=states.alloc,
device_free_fn=states.free,
device_evict_fn=None,
),
}
)
allocated = (full.allocated_count(), swa.allocated_count())
# The production order: FULL, then SWA, then the Mamba checkpoint.
transfers = [
PoolTransfer(name=PoolName.SWA, host_indices=torch.arange(PAGE)),
PoolTransfer(name=PoolName.MAMBA, host_indices=torch.tensor([1])),
]
# The state slot needs the float to move, which the SWA reservation
# taken just before forbids: the load rolls back instead.
self.assertIsNone(
controller.load(host_indices=torch.arange(PAGE), extra_pools=transfers)
)
self.assertEqual(moves, [])
self.assertEqual(controller.load_queue, [])
self.assertIsNone(transfers[0].device_indices)
self.assertEqual((full.allocated_count(), swa.allocated_count()), allocated)
self.assertEqual(allocator.verify_byte_accounting(), [])
# Cancelled, so the float makes room again.
self.assertIsNotNone(states.alloc(1))
self.assertIn(("move", "swa"), moves)
class TestAllocationAndTransferOrder(_ConfigCase):
"""Allocation reuses free pages without waiting on HiCache copies."""
def test_hole_reuse_does_not_wait_for_unrelated_copies(self):
allocator, rows = _unified_swa()
swa = allocator.swa_attn_allocator
source = allocator.alloc(2 * PAGE) # a write-through D2H source (locked)
gap = allocator.alloc(3 * PAGE)
target = allocator.alloc(2 * PAGE) # an H2D target owned by its load
expected_source = rows.seed(source, 100)
expected_target = rows.seed(target, 200)
allocator.free(gap)
allocator.set_hicache_transfer_done_event(D2H, object())
allocator.set_hicache_transfer_done_event(H2D, object())
with _recorded_stream() as waits:
fresh = allocator.alloc(2 * PAGE)
staged = swa.alloc_physical(PAGE)
self.assertEqual(waits, [])
# Writing the new rows leaves the copies' rows intact.
rows.seed(fresh, 600)
rows.swa[staged] = rows.markers(700, PAGE)
for ids, expected in ((source, expected_source), (target, expected_target)):
got = rows.read(ids)
self.assertTrue(torch.equal(got[0], expected[0]))
self.assertTrue(torch.equal(got[1], expected[1]))
class _UnifiedHiCacheCase(_ConfigCase):
"""A FULL+SWA `UnifiedRadixCache` with HiCache over the unified pool and no
storage backend, so the host side is the shared page-envelope pool. The
TreeCore is whichever backend loads (Rust, else Python)."""
window = 2 * PAGE
def _cache(
self,
*,
write_policy: str = "write_through",
tokens: int = 128,
):
server_args = ServerArgs(
model_path="dummy",
page_size=PAGE,
enable_unified_memory=True,
hicache_write_policy=write_policy,
hicache_mem_layout="layer_first",
hicache_host_memory_mode="cache",
)
set_global_server_args_for_scheduler(server_args)
bundle = _unified_swa_bundle(tokens=tokens)
self.allocator = bundle.token_to_kv_pool_allocator
self.rows = _Rows(bundle)
self.req_to_token_pool = ReqToTokenPool(
size=4, max_context_len=tokens, device="cpu", enable_memory_saver=False
)
params = CacheInitParams(
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.allocator,
page_size=PAGE,
disable=False,
sliding_window_size=self.window,
tree_components=(ComponentType.FULL, ComponentType.SWA),
)
with mock.patch.dict(
_TREE_CORE_REGISTRY, {"python": _python_inspector, "rust": _rust_inspector}
):
cache = UnifiedRadixCache(params=params)
# CPU host tensors need no page-locking.
with mock.patch.object(host_memory, "_cuda_host_register"):
cache.init_hicache(server_args, params)
atexit.unregister(cache.shutdown)
self.addCleanup(cache.release_host_resources)
controller = cache.cache_controller
# The scheduler's gate: rows stay put until host copies are acknowledged.
self.allocator.set_host_transfer_move_gate(
lambda: not controller.has_inflight_device_transfers()
)
self.cache = cache
self.controller = controller
# Compaction and frees wait through this stand-in stream.
stream = _recorded_stream()
stream.__enter__()
self.addCleanup(stream.__exit__, None, None, None)
return cache
def _insert(self, tokens, base):
ids = self.allocator.alloc(len(tokens))
expected = self.rows.seed(ids, base)
self.cache.insert(InsertParams(key=RadixKey(array("q", tokens)), value=ids))
return ids, expected
def _match(self, tokens):
return self.cache.match_prefix(
MatchPrefixParams(key=RadixKey(array("q", tokens)))
)
def _assert_rows(self, ids, expected, *, swa_tail: int):
full, swa = self.rows.read(ids)
self.assertTrue(torch.equal(full, expected[0]))
self.assertTrue(torch.equal(swa[-swa_tail:], expected[1][-swa_tail:]))
class TestSwaLoadBackReservation(_UnifiedHiCacheCase):
"""Cache-mode SWA load-back loads into physical reservations."""
def test_reservation_retries_once_after_device_eviction(self):
self._cache(write_policy="write_through", tokens=64)
swa = self.allocator.swa_attn_allocator
# Tree-owned rows fill the SWA end; once their backups are acked,
# device eviction may reclaim them.
fillers = 0
while self.allocator.swa_available_size() >= 2 * PAGE:
fillers += 1
self._insert(
list(range(100 * fillers, 100 * fillers + PAGE)), 100 * fillers
)
self.cache.flush_pending_backups()
self.cache.writing_check()
entry = self.controller.mem_pool_host.entry_map[PoolName.SWA]
evict = entry.device_evict_fn
evictions = []
def recorded_evict(need_size):
evictions.append(need_size)
return evict(need_size)
transfer = PoolTransfer(name=PoolName.SWA, host_indices=torch.arange(2 * PAGE))
with mock.patch.object(entry, "device_evict_fn", side_effect=recorded_evict):
result = self.controller._resolve_device_transfers(
[transfer], kv_device_indices=torch.empty(0, dtype=torch.int64)
)
# No room on the first try; one device eviction makes it.
self.assertIsNotNone(result)
self.assertEqual(evictions, [2 * PAGE])
swa.cancel_physical_reservation(transfer.device_indices)
self.assertEqual(self.allocator.verify_byte_accounting(), [])
self.cache.sanity_check()
class TestLoadBackAdmission(_UnifiedHiCacheCase):
"""PrefillAdder load-back admission over the real cache and shared budget."""
def _adder(self, num_mixed_decode_tokens: int = 0):
running = mock.MagicMock()
running.reqs = []
return PrefillAdder(
page_size=PAGE,
tree_cache=self.cache,
token_to_kv_pool_allocator=self.allocator,
running_batch=running,
new_token_ratio=1.0,
rem_input_tokens=10_000,
rem_chunk_tokens=None,
num_mixed_decode_tokens=num_mixed_decode_tokens,
priority_scheduling_preemption_threshold=0,
)
def _request(self, tokens):
match = self._match(tokens)
req = mock.MagicMock(spec=Req)
req.rid = "load-back"
req.cache_request_handle = CacheRequestHandle(req.rid, 0)
req.priority = 0
req.prefix_len = match.device_prefix_len
req.last_node = match.last_device_node
req.best_match_node = match.best_match_node
req.full_untruncated_fill_ids = list(tokens)
req.output_ids = []
req.sampling_params = SimpleNamespace(max_new_tokens=PAGE, ignore_eos=False)
req.host_hit_length = match.host_hit_length
req.swa_host_hit_length = match.swa_host_hit_length
req.mamba_host_hit_length = 0
req.storage_hit_length = 0
req.storage_hit_start = None
req.host_hit_is_storage = False
req.host_loaded_length = 0
req.retracted_stain = False
req.needs_host_load_back.return_value = True
req.finished.return_value = False
req.materialized_host_hit_len.return_value = match.host_hit_length
req.fulfilled_storage_hit_len.return_value = 0
req.kv = SimpleNamespace(
cache_protected_len=match.device_prefix_len, mamba_pool_idx=None
)
type(req).extend_len = mock.PropertyMock(
side_effect=lambda: req.extend_end - req.prefix_len
)
return req
def _locks(self, node):
core = self.cache.tree_core
return (
core.get_component_device_lock_ref(node, FULL),
core.get_component_host_lock_ref(node, FULL),
core.get_component_host_lock_ref(node, SWA),
)
def _host_pressure(self):
"""A host-only match, a host pool with almost no room, and an unbacked
device victim whose write-back needs host room."""
self._cache(write_policy="write_back", tokens=64)
match_tokens = list(range(1, 17))
_, expected = self._insert(match_tokens, 100)
self.cache.evict(EvictParams(num_tokens=len(match_tokens)))
host = self.cache.host_pool_group
filler = 1000
while host.available_size() >= 2 * len(match_tokens):
tokens = list(range(filler, filler + 16))
self._insert(tokens, 300)
self.cache.evict(EvictParams(num_tokens=len(tokens)))
filler += 100
self._insert(list(range(5000, 5040)), 500)
return match_tokens + list(range(17, 25)), expected
def test_host_match_stays_pinned_while_reclaim_evicts_host_leaves(self):
tokens, expected = self._host_pressure()
req = self._request(tokens)
node = req.best_match_node
evict_host = self.cache.evict_host
pins_during_host_eviction = []
def recorded(*args, **kwargs):
pins_during_host_eviction.append(self._locks(node)[1:])
return evict_host(*args, **kwargs)
with mock.patch.object(self.cache, "evict_host", side_effect=recorded):
verdict = self._adder().add_one_req(
req, has_chunked_req=False, truncation_align_size=None
)
self.assertIs(verdict, AddReqResult.CONTINUE)
# Reclaim had to make host room, and the match was pinned meanwhile.
self.assertTrue(pins_during_host_eviction)
for full_pin, swa_pin in pins_during_host_eviction:
self.assertGreater(full_pin, 0)
self.assertGreater(swa_pin, 0)
self.cache.ready_to_load_host_cache()
self.cache.loading_check()
# The CPU copy survived until the load, so the request sees its data.
full, swa = self.rows.read(self.cache.prefix_device_indices(req))
self.assertTrue(torch.equal(full, expected[0]))
self.assertTrue(torch.equal(swa[-self.window :], expected[1][-self.window :]))
# Only the request's own device lock is left once the load is acked.
self.assertEqual(self._locks(node), (1, 0, 0))
self.cache.sanity_check()
def test_shared_budget_reclaims_for_the_load_and_pending_demand(self):
# FULL stays resident under a host-only SWA window.
self._cache(write_policy="write_through", tokens=80)
tokens = list(range(1, 17))
_, expected = self._insert(tokens, 100)
self.cache.flush_pending_backups()
self.cache.writing_check()
self.cache.evict(EvictParams(swa_num_tokens=len(tokens)))
req = self._request(tokens + list(range(17, 21)))
# A running request locks its whole path; only its SWA past the window
# can be reclaimed.
running = list(range(5000, 5040))
running_ids, running_expected = self._insert(running, 500)
self.cache.flush_pending_backups()
self.cache.writing_check()
running_node = self._match(running).last_device_node
pin = self.cache.inc_lock_ref(running_node).to_dec_params()
reclaimable_swa = self.cache.swa_evictable_size()
# The batch's decode tokens are pending demand on both bands. With
# them, the load fits the shared gap only after that SWA is reclaimed.
decode = 6 * PAGE
moves = []
_record_moves(self.allocator, moves)
verdict = self._adder(num_mixed_decode_tokens=decode).add_one_req(
req, has_chunked_req=False, truncation_align_size=None
)
self.assertIs(verdict, AddReqResult.CONTINUE)
self.assertEqual(len(self.controller.load_queue), 1)
# The queued load blocks every page move. The batch's own extend and
# the pending decode tokens still fit.
moved = len(moves)
own = self.allocator.alloc(PAGE)
pending = self.allocator.alloc(decode)
self.assertIsNotNone(own)
self.assertIsNotNone(pending)
self.assertEqual(len(moves), moved)
# That room came from reclaiming the running request's out-of-window
# SWA; its FULL rows and its window stay.
self.assertLess(self.cache.swa_evictable_size(), reclaimable_swa)
self._assert_rows(running_ids, running_expected, swa_tail=self.window)
self.cache.ready_to_load_host_cache()
self.cache.loading_check()
full, swa = self.rows.read(self.cache.prefix_device_indices(req))
self.assertTrue(torch.equal(full, expected[0]))
self.assertTrue(torch.equal(swa[-self.window :], expected[1][-self.window :]))
self.allocator.free(torch.cat([own, pending]))
self.cache.dec_lock_ref(running_node, pin)
self.assertEqual(self.allocator.verify_byte_accounting(), [])
self.cache.sanity_check()
class TestWritePolicySwitch(_UnifiedHiCacheCase):
"""Attaching storage with write_through over a write_back host tree."""
def _policy_state(self):
cache, controller = self.cache, self.controller
return (
controller.write_policy,
cache.write_through_threshold,
cache.is_write_back,
cache.prefetch_stop_policy,
cache.enable_storage,
controller.storage_backend,
controller.storage_backend_type,
)
def test_incompatible_host_tree_is_rejected_without_side_effects(self):
self._cache(write_policy="write_back", tokens=64)
self._insert(list(range(1, 9)), 100)
self._insert(list(range(1, 17)), 200)
# Write-back backs up only the evicted leaf: a host suffix below an
# unbacked FULL parent.
self.cache.evict(EvictParams(num_tokens=8))
before = self._policy_state()
with mock.patch.object(
self.controller, "attach_storage_backend"
) as attach_backend:
ok, message = self.cache._storage_attachment.attach(
"mori", hicache_write_policy="write_through"
)
self.assertFalse(ok)
self.assertIn("from write_back to write_through", message)
attach_backend.assert_not_called()
self.assertEqual(self._policy_state(), before)
self.cache.sanity_check()
if __name__ == "__main__":
unittest.main()
@@ -911,21 +911,19 @@ class TestUnifiedRadixAllocationEvictionRealComponents(CustomTestCase):
for session in _session_radix_cache_test_values():
for pinned in (False, True):
with self.subTest(session=session, pinned=pinned):
# This exercises the Rust backup barrier, independently of
# the shared suite's default backend.
# Rust independently of the shared suite's default backend;
# session caches fall back to the Python TreeCore.
with mock.patch(f"{__name__}._TREE_CORE_TEST_BACKEND", "rust"):
cache, first, second, leaf = self._build_internal_chain(
ct, session
)
cache.tree_core.is_write_back = True
cache.tree_core.has_swa_host_pool = True
cache.tree_core.enable_swa_write_back_eviction_barrier()
if session:
# Session caches use main's Python fallback, whose
# component backup is gated by the HiCache attachment.
cache.tree_core.enable_hicache = True
cache.host_pool_group = mock.Mock()
cache.host_pool_group.get_pool.return_value = None
# Both cores gate the internal SWA component backup on the
# HiCache attachment.
cache.tree_core.set_hicache_enabled()
cache.host_pool_group = mock.Mock()
cache.host_pool_group.get_pool.return_value = None
if pinned:
receipt = cache.inc_host_lock_ref(first).to_dec_params()
tracker = {ComponentType.FULL: 0, ct: 0}