[unified memory pool] Allocate without transfer waits, pin floating SWA reservations, drop the HiCache startup asserts (#43206)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
ZYHowell
parent
a65f4ffba5
commit
f2e9a2bed1
@@ -659,15 +659,6 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
"preserve unified page-envelope indices and compaction lifetimes. "
|
||||
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
|
||||
)
|
||||
if cfg.enable_hierarchical_cache:
|
||||
assert cfg.pp_size == 1, (
|
||||
"--enable-unified-memory with hierarchical cache does not support "
|
||||
"pipeline parallelism (--pp-size > 1)."
|
||||
)
|
||||
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
|
||||
"--enable-unified-memory with hierarchical cache requires lazy "
|
||||
"compaction so pending H2D physical reservations remain stable."
|
||||
)
|
||||
assert not (
|
||||
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
|
||||
and cfg.disaggregation_decode_retraction_backup == "host_pool"
|
||||
|
||||
@@ -1357,7 +1357,8 @@ class PrefillAdder:
|
||||
):
|
||||
full_load_tokens = req.host_hit_length
|
||||
if (
|
||||
isinstance(self.tree_cache, UnifiedRadixCache)
|
||||
self.memory_budget.load_back_needs_full_tokens
|
||||
and isinstance(self.tree_cache, UnifiedRadixCache)
|
||||
and self.tree_cache.buffer_pipeline is None
|
||||
and not (
|
||||
self.tree_cache.linker is not None
|
||||
|
||||
@@ -335,33 +335,6 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
lazy_compaction=self.lazy_compaction,
|
||||
)
|
||||
|
||||
def bind_swa_for_loaded_rows(
|
||||
self, full_token_ids: torch.Tensor
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Bind SWA pages to resident or newly loaded full-attention virtual IDs.
|
||||
|
||||
Bind before translating: unbound pages translate to the padding sink.
|
||||
Return physical IDs, or None if capacity cannot be reclaimed.
|
||||
"""
|
||||
ids = full_token_ids.to(torch.int64)
|
||||
if ids.numel() == 0:
|
||||
return ids
|
||||
ps = self.page_size
|
||||
pages = torch.unique(ids // ps)
|
||||
# Tombstones (-1) and the padding sink (0) both need a physical binding.
|
||||
unbound = pages[self.swa_attn_allocator.virtual_to_physical[pages] <= 0]
|
||||
need = int(unbound.numel()) * ps
|
||||
if need:
|
||||
if need > self.swa_available_size():
|
||||
return None
|
||||
if (
|
||||
need > self.swa_attn_allocator.available_size()
|
||||
and not _relieve_for_alloc(self.swa_attn_allocator, need)
|
||||
):
|
||||
return None
|
||||
self.swa_attn_allocator.alloc_with_virtual(unbound)
|
||||
return self.translate_loc_from_full_to_swa(ids)
|
||||
|
||||
def set_host_transfer_move_gate(self, gate: Callable[[], bool]) -> None:
|
||||
"""Block page relocation while host transfers use resolved device indices."""
|
||||
install_move_gate(
|
||||
|
||||
@@ -953,8 +953,6 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
with record_function("MultiEndedAlloc._alloc_bind_fast_or_slow"):
|
||||
if N == 0:
|
||||
return torch.empty(0, dtype=torch.int64, device=self.device)
|
||||
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
|
||||
self._wait_hicache_transfers()
|
||||
|
||||
# FAST PATH: eager, or lazy with no current holes.
|
||||
if not self.lazy_compaction or self._free_phys_pages.numel() == 0:
|
||||
@@ -2014,6 +2012,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self._pending_hicache_load_pages = 0
|
||||
|
||||
def _wait_hicache_transfers(self) -> None:
|
||||
"""Order the current stream after outstanding HiCache copies. Only moves
|
||||
and frees need this: a copy's source and target are not free pages until
|
||||
its finish event is observed, so allocation never reuses them."""
|
||||
if not self._hicache_transfer_done_events:
|
||||
return
|
||||
current_stream = torch.cuda.current_stream()
|
||||
@@ -2041,26 +2042,23 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
if need_size > self.available_size():
|
||||
if not _relieve_for_alloc(self, need_size):
|
||||
return None
|
||||
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
|
||||
self._wait_hicache_transfers()
|
||||
physical_pages = self.take_physical_pages(need_size // self.page_size)
|
||||
if physical_pages is None:
|
||||
return None
|
||||
if self.lazy_compaction:
|
||||
self._pending_hicache_load_pages += int(physical_pages.shape[0])
|
||||
# Blocks page moves (`moves_blocked`) until the H2D is queued or cancelled.
|
||||
self._pending_hicache_load_pages += int(physical_pages.shape[0])
|
||||
return self._expand_pages_to_tokens(physical_pages)
|
||||
|
||||
def cancel_physical_reservation(self, free_index: torch.Tensor) -> None:
|
||||
"""Roll back a HiCache physical allocation before its H2D is submitted."""
|
||||
if free_index is None or free_index.numel() == 0:
|
||||
return
|
||||
if self.lazy_compaction:
|
||||
num_pages = free_index.numel() // self.page_size
|
||||
assert num_pages <= self._pending_hicache_load_pages, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
|
||||
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
|
||||
)
|
||||
self._pending_hicache_load_pages -= num_pages
|
||||
num_pages = free_index.numel() // self.page_size
|
||||
assert num_pages <= self._pending_hicache_load_pages, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
|
||||
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
|
||||
)
|
||||
self._pending_hicache_load_pages -= num_pages
|
||||
self.free_physical(free_index)
|
||||
|
||||
def free_physical(self, free_index: torch.Tensor) -> None:
|
||||
|
||||
@@ -28,7 +28,6 @@ from __future__ import annotations
|
||||
import logging
|
||||
from array import array
|
||||
from collections import deque
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
@@ -1532,51 +1531,6 @@ class BufferModePipeline:
|
||||
),
|
||||
0,
|
||||
)
|
||||
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
|
||||
binds_swa_to_full = (
|
||||
swa_entry is not None
|
||||
and swa_entry.device_indices_from_anchor_fn is not None
|
||||
)
|
||||
repair_ranges = []
|
||||
if binds_swa_to_full and staged_swa:
|
||||
window_start = span_end - staged_swa
|
||||
repair_end = min(splice_base, span_end)
|
||||
if window_start < repair_end:
|
||||
repair_ranges = cache.tree_core.swa_tombstone_ranges(
|
||||
key, window_start, repair_end
|
||||
)
|
||||
# Only missing SWA rows belong to this load. Existing bindings may
|
||||
# be in use by another request and must survive allocation rollback.
|
||||
anchor_parts = []
|
||||
host_parts = []
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
anchor_parts.append(prefix_indices[repair_start:repair_end_])
|
||||
host_parts.append(
|
||||
slice(repair_start - window_start, repair_end_ - window_start)
|
||||
)
|
||||
tail_start = max(splice_base, window_start)
|
||||
if tail_start < span_end:
|
||||
anchor_parts.append(
|
||||
slice(tail_start - splice_base, span_end - splice_base)
|
||||
)
|
||||
host_parts.append(
|
||||
slice(tail_start - window_start, span_end - window_start)
|
||||
)
|
||||
for i, transfer in enumerate(load_xfers):
|
||||
if transfer.name != PoolName.SWA:
|
||||
continue
|
||||
# Keep the original complete host bounce for ack/drop release.
|
||||
load_xfers[i] = replace(
|
||||
transfer,
|
||||
host_indices=(
|
||||
torch.cat([transfer.host_indices[part] for part in host_parts])
|
||||
if host_parts
|
||||
else transfer.host_indices[:0]
|
||||
),
|
||||
anchor_index_parts=anchor_parts,
|
||||
)
|
||||
if not anchor_parts:
|
||||
load_xfers = [t for t in load_xfers if t.name != PoolName.SWA]
|
||||
|
||||
device_indices = cc.load(
|
||||
host_indices=f.host_indices[trim_tokens:],
|
||||
@@ -1605,21 +1559,10 @@ class BufferModePipeline:
|
||||
None,
|
||||
)
|
||||
aux_device_releases: list[tuple[PoolName, torch.Tensor]] = []
|
||||
if swa_dev is not None and binds_swa_to_full:
|
||||
# Binding already updated the allocator's virtual-to-physical table.
|
||||
# The tree owns virtual FULL rows, not the H2D kernel-facing IDs.
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
for action in cache.tree_core.attach_swa_window(
|
||||
key,
|
||||
repair_start,
|
||||
repair_end_,
|
||||
prefix_indices[repair_start:repair_end_],
|
||||
):
|
||||
cache._apply_cache_action(action)
|
||||
elif swa_dev is not None:
|
||||
# Register the window's FULL->SWA translation now (attention reads
|
||||
# through it). Keep SWA slots another request may still hold; their
|
||||
# redundant H2D destinations are reclaimed at the transfer ack.
|
||||
if swa_dev is not None:
|
||||
# SWA loads into fresh reservations. Register the window's FULL->SWA
|
||||
# translation now (attention reads through it); slots another request
|
||||
# still holds are kept, and their redundant destinations freed at ack.
|
||||
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
|
||||
|
||||
@@ -115,9 +115,6 @@ class PoolTransfer:
|
||||
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
|
||||
nodes_to_load: Optional[List[Any]] = None
|
||||
indices_from_pool: Optional[PoolName] = None
|
||||
# Full IDs backing a dependent device allocation: resident tensors or
|
||||
# slices of the full rows allocated by this load, in transfer order.
|
||||
anchor_index_parts: Optional[List[torch.Tensor | slice]] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -1719,7 +1719,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
return None
|
||||
newly_allocated: list[tuple[PoolTransfer, Callable, torch.Tensor]] = []
|
||||
derived_transfers: list[PoolTransfer] = []
|
||||
anchor_transfers = []
|
||||
|
||||
def rollback_allocated() -> None:
|
||||
for prev_pool, prev_free_fn, prev_indices in reversed(newly_allocated):
|
||||
@@ -1735,11 +1734,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
continue
|
||||
if pool.device_indices is not None or pool.host_indices is None:
|
||||
continue
|
||||
if entry.device_indices_from_anchor_fn is not None:
|
||||
# Allocate independent pools first: their allocation/eviction
|
||||
# can compact SWA before its physical IDs are captured.
|
||||
anchor_transfers.append((pool, entry))
|
||||
continue
|
||||
# device_alloc_fn / device_free_fn override entry.device_pool's
|
||||
# methods for pools whose device_pool is a raw KV pool (layout)
|
||||
# rather than an allocator (e.g. SWA).
|
||||
@@ -1758,28 +1752,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool.device_indices = indices
|
||||
newly_allocated.append((pool, free_fn, indices))
|
||||
|
||||
for pool, entry in anchor_transfers:
|
||||
if kv_device_indices is None or not pool.anchor_index_parts:
|
||||
rollback_allocated()
|
||||
return None
|
||||
anchor_indices = torch.cat(
|
||||
[
|
||||
kv_device_indices[part] if isinstance(part, slice) else part
|
||||
for part in pool.anchor_index_parts
|
||||
]
|
||||
)
|
||||
assert len(anchor_indices) == len(pool.host_indices)
|
||||
bind = entry.device_indices_from_anchor_fn
|
||||
indices = bind(anchor_indices)
|
||||
if indices is None and entry.device_evict_fn:
|
||||
entry.device_evict_fn(len(anchor_indices))
|
||||
indices = bind(anchor_indices)
|
||||
if indices is None:
|
||||
rollback_allocated()
|
||||
return None
|
||||
pool.device_indices = indices
|
||||
newly_allocated.append((pool, entry.device_free_fn, anchor_indices))
|
||||
|
||||
# Assign indices to deferred pools from their source.
|
||||
for pool in derived_transfers:
|
||||
if pool.indices_from_pool == PoolName.KV:
|
||||
|
||||
@@ -210,7 +210,6 @@ def build_pool_entry(
|
||||
device_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_alloc_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_free_fn: Optional[Callable[[Any], Any]] = None,
|
||||
device_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
|
||||
packed_draft_device_pools: tuple[Any, ...] = (),
|
||||
) -> PoolEntry:
|
||||
return PoolEntry(
|
||||
@@ -223,7 +222,6 @@ def build_pool_entry(
|
||||
device_evict_fn=device_evict_fn,
|
||||
device_alloc_fn=device_alloc_fn,
|
||||
device_free_fn=device_free_fn,
|
||||
device_indices_from_anchor_fn=device_indices_from_anchor_fn,
|
||||
packed_draft_device_pools=packed_draft_device_pools,
|
||||
)
|
||||
|
||||
@@ -271,21 +269,16 @@ def build_kv_only_group(
|
||||
)
|
||||
|
||||
|
||||
def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
|
||||
def _swa_allocation_callbacks(allocator) -> dict:
|
||||
"""Keep allocation and rollback in the same ID space for every SWA stack."""
|
||||
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
|
||||
|
||||
if isinstance(allocator, MultiEndedAllocator):
|
||||
# Unified SWA loads into physical reservations, bound before the H2D.
|
||||
return dict(
|
||||
device_alloc_fn=allocator.alloc_physical,
|
||||
device_free_fn=allocator.cancel_physical_reservation,
|
||||
)
|
||||
if bind is not None:
|
||||
assert free_bound is not None
|
||||
return dict(
|
||||
device_indices_from_anchor_fn=bind,
|
||||
device_free_fn=free_bound,
|
||||
)
|
||||
if allocator is None:
|
||||
return {}
|
||||
return dict(device_alloc_fn=allocator.alloc, device_free_fn=allocator.free)
|
||||
@@ -327,8 +320,6 @@ def build_hybrid_swa_group(
|
||||
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
swa_attn_allocator: Any = None,
|
||||
swa_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
|
||||
swa_free_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
|
||||
mtp_swa_device_pools: tuple[Any, ...] = (),
|
||||
) -> HostPoolGroup:
|
||||
"""Anchor (full) + SWA host pool group for a hybrid-SWA device pool."""
|
||||
@@ -394,11 +385,7 @@ def build_hybrid_swa_group(
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(mtp_swa_device_pools),
|
||||
host_evict_fn=host_swa_evict_fn,
|
||||
device_evict_fn=device_swa_evict_fn,
|
||||
**_swa_allocation_callbacks(
|
||||
swa_attn_allocator,
|
||||
swa_indices_from_anchor_fn,
|
||||
swa_free_from_anchor_fn,
|
||||
),
|
||||
**_swa_allocation_callbacks(swa_attn_allocator),
|
||||
packed_draft_device_pools=mtp_swa_device_pools,
|
||||
),
|
||||
]
|
||||
@@ -499,17 +486,6 @@ def build_hybrid_swa_stack(
|
||||
device_swa_evict_fn=device_swa_evict_fn,
|
||||
# For SWA hybrid, device allocation goes through the inner allocator.
|
||||
swa_attn_allocator=params.token_to_kv_pool_allocator.swa_attn_allocator,
|
||||
# Unified SWA binds pages to the full pool's virtual IDs instead.
|
||||
swa_indices_from_anchor_fn=(
|
||||
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
|
||||
if get_memory().enable_unified_memory
|
||||
else None
|
||||
),
|
||||
swa_free_from_anchor_fn=(
|
||||
params.token_to_kv_pool_allocator.free_swa
|
||||
if get_memory().enable_unified_memory
|
||||
else None
|
||||
),
|
||||
mtp_swa_device_pools=mtp_swa_device_pools,
|
||||
)
|
||||
cache_controller = HybridCacheController(
|
||||
@@ -1329,15 +1305,7 @@ def build_hybrid_mamba_swa_stack(
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
host_evict_fn=host_swa_evict_fn,
|
||||
device_evict_fn=device_swa_evict_fn,
|
||||
**_swa_allocation_callbacks(
|
||||
swa_attn_allocator,
|
||||
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
|
||||
if get_memory().enable_unified_memory
|
||||
else None,
|
||||
params.token_to_kv_pool_allocator.free_swa
|
||||
if get_memory().enable_unified_memory
|
||||
else None,
|
||||
),
|
||||
**_swa_allocation_callbacks(swa_attn_allocator),
|
||||
),
|
||||
build_pool_entry(
|
||||
name=PoolName.MAMBA,
|
||||
|
||||
@@ -25,10 +25,6 @@ class PoolEntry:
|
||||
device_evict_fn: Callable[[int], Any] | None = None
|
||||
device_alloc_fn: Callable[[int], Any] | None = None
|
||||
device_free_fn: Callable[[Any], Any] | None = None
|
||||
# Bind rows to the anchor's virtual IDs when pools share an ID space.
|
||||
# Return buffer indices, or None if allocation fails. Rollback through
|
||||
# device_free_fn takes the anchor's virtual IDs, not the returned indices.
|
||||
device_indices_from_anchor_fn: Callable[[Any], Any] | None = None
|
||||
packed_draft_device_pools: tuple[Any, ...] = ()
|
||||
|
||||
|
||||
|
||||
@@ -62,6 +62,9 @@ def estimate_swa_kv_tokens(
|
||||
class PrefillBudget:
|
||||
"""Fixed token pool. Offsets include pending allocations and decode headroom."""
|
||||
|
||||
# When False, admission skips the FULL spec and passes the host hit length.
|
||||
load_back_needs_full_tokens = False
|
||||
|
||||
def __init__(self, allocator, tree_cache, *, num_mixed_decode_tokens: int = 0):
|
||||
self.allocator = allocator
|
||||
self.tree_cache = tree_cache
|
||||
@@ -315,6 +318,8 @@ class SWAPrefillBudget(PrefillBudget):
|
||||
class SharedSWAPrefillBudget(SWAPrefillBudget):
|
||||
"""FULL and SWA reservations compete for the same physical byte budget."""
|
||||
|
||||
load_back_needs_full_tokens = True
|
||||
|
||||
def __init__(self, *args, num_mixed_decode_tokens=0, **kwargs):
|
||||
super().__init__(
|
||||
*args, num_mixed_decode_tokens=num_mixed_decode_tokens, **kwargs
|
||||
|
||||
@@ -568,11 +568,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result = EvictDeviceNextNodeResult(
|
||||
node_id=binding_result.node_id,
|
||||
made_progress=binding_result.made_progress,
|
||||
backup_kv=(
|
||||
_cache_action_from_tagged(binding_result.backup_kv)
|
||||
if binding_result.backup_kv is not None
|
||||
else None
|
||||
),
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
mamba_backup_node_id=binding_result.mamba_backup_node_id,
|
||||
swa_backup_node_id=binding_result.swa_backup_node_id,
|
||||
@@ -788,9 +783,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def set_hicache_enabled(self) -> None:
|
||||
self._binding.set_hicache_enabled()
|
||||
|
||||
def enable_swa_write_back_eviction_barrier(self) -> None:
|
||||
self._binding.enable_swa_write_back_eviction_barrier()
|
||||
|
||||
def set_host_memory_buffer_only(self) -> None:
|
||||
self._binding.set_host_memory_buffer_only()
|
||||
|
||||
@@ -961,27 +953,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
kv_xfer = _transfer_from_binding(kv_xfer)
|
||||
comp_xfers = _comp_xfers_from_binding(comp_xfers)
|
||||
swa_xfers = comp_xfers.get(ComponentType.SWA, ())
|
||||
if swa_xfers:
|
||||
full_node_ids = kv_xfer.nodes_to_load or []
|
||||
full_load_slices = {}
|
||||
offset = 0
|
||||
for full_node_id, count in zip(
|
||||
full_node_ids, self._binding.get_node_key_lengths(full_node_ids)
|
||||
):
|
||||
full_load_slices[full_node_id] = slice(offset, offset + count)
|
||||
offset += count
|
||||
for transfer in swa_xfers:
|
||||
# SWA may have holes between resident nodes, or reload while
|
||||
# FULL stays resident. Preserve the SWA transfer's node order.
|
||||
transfer.anchor_index_parts = [
|
||||
(
|
||||
full_load_slices[nid]
|
||||
if nid in full_load_slices
|
||||
else self.get_component_device_value(nid, ComponentType.FULL)
|
||||
)
|
||||
for nid in transfer.nodes_to_load or ()
|
||||
]
|
||||
return kv_xfer, comp_xfers
|
||||
|
||||
def prefetch_anchor_info(
|
||||
|
||||
@@ -2494,21 +2494,6 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
nodes_to_load=[],
|
||||
)
|
||||
return empty_kv, {}
|
||||
# SWA can be evicted independently of FULL, including holes between
|
||||
# resident SWA nodes. Describe precisely which full rows back it.
|
||||
full_load_slices = {}
|
||||
offset = 0
|
||||
for nid in kv_xfer.nodes_to_load or ():
|
||||
count = len(self.node_by_id(nid).key)
|
||||
full_load_slices[nid] = slice(offset, offset + count)
|
||||
offset += count
|
||||
for xfer in comp_xfers.get(ComponentType.SWA, ()):
|
||||
xfer.anchor_index_parts = [
|
||||
full_load_slices[nid]
|
||||
if nid in full_load_slices
|
||||
else self.node_by_id(nid).component_data[BASE_COMPONENT_TYPE].value
|
||||
for nid in xfer.nodes_to_load or ()
|
||||
]
|
||||
return kv_xfer, comp_xfers
|
||||
|
||||
def prefetch_anchor_info(
|
||||
|
||||
@@ -52,7 +52,6 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
|
||||
"""
|
||||
|
||||
node_id: Optional[NodeId] = None
|
||||
backup_kv: Optional[BackupKV] = None
|
||||
made_progress: bool = False
|
||||
unbacked_tokens: int = 0
|
||||
mamba_backup_node_id: Optional[NodeId] = None
|
||||
@@ -523,13 +522,6 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""Mark the host tier (HiCache) as wired."""
|
||||
...
|
||||
|
||||
def enable_swa_write_back_eviction_barrier(self) -> None:
|
||||
"""Enable a backend-managed barrier when needed.
|
||||
|
||||
The Python core demotes through SWAComponent directly. Native cores
|
||||
may return a backup action to the cache executor before eviction.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def set_host_memory_buffer_only(self) -> None:
|
||||
"""Mark the host tier as buffer-only: one node staged per backup intent."""
|
||||
|
||||
@@ -532,13 +532,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.cache_controller is not None
|
||||
and self.cache_controller.write_policy == "write_back"
|
||||
)
|
||||
# Preserve the SWA host window before device eviction makes it unrecoverable.
|
||||
if (
|
||||
get_memory().enable_unified_memory
|
||||
and self.host_memory_mode == "cache"
|
||||
and self.tree_core.has_swa_host_pool
|
||||
):
|
||||
self.tree_core.enable_swa_write_back_eviction_barrier()
|
||||
# Pre-seed the logical dropped-tokens series.
|
||||
if self.metrics_collector is not None and self.cache_controller is not None:
|
||||
reasons = ["host_pressure"]
|
||||
@@ -805,74 +798,45 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def _evict_device_next_node(
|
||||
self, component_type: ComponentType, tracker: dict[ComponentType, int]
|
||||
) -> tuple[Optional[NodeId], bool]:
|
||||
"""Advance the walk, completing pending host-backup barriers."""
|
||||
while True:
|
||||
result = self.tree_core.evict_device_next_node(component_type, tracker)
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
)
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
# proceeds to make room on the device.
|
||||
node_id = result.mamba_backup_node_id
|
||||
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
|
||||
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
|
||||
self.evict_host(1, ComponentType.MAMBA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_mamba_state_eviction(node_id)
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
# even if host allocation fails.
|
||||
node_id = result.swa_backup_node_id
|
||||
needed = result.swa_backup_num_tokens
|
||||
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
|
||||
if (
|
||||
swa_host_pool is not None
|
||||
and swa_host_pool.available_size() < needed
|
||||
):
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
reason="write_through_unbacked_eviction",
|
||||
)
|
||||
self._accumulate_tracker(tracker, result.tracker)
|
||||
if result.backup_kv is None:
|
||||
return result.node_id, result.made_progress
|
||||
|
||||
assert result.node_id is None
|
||||
assert self.buffer_pipeline is None, (
|
||||
"SWA write-back eviction barriers are cache-mode only"
|
||||
"""Advance the eviction walk one node, consuming its step result."""
|
||||
result = self.tree_core.evict_device_next_node(component_type, tracker)
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
)
|
||||
written = self._execute_and_commit_kv_backup(
|
||||
result.backup_kv, write_back=True
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
# proceeds to make room on the device.
|
||||
node_id = result.mamba_backup_node_id
|
||||
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
|
||||
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
|
||||
self.evict_host(1, ComponentType.MAMBA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_mamba_state_eviction(node_id)
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
)
|
||||
if written <= 0:
|
||||
node_id = result.backup_kv.node_ids[0]
|
||||
logger.warning(
|
||||
"write_back: auxiliary backup failed under host pressure "
|
||||
"(component=%s, node=%d); dropping only the component",
|
||||
component_type.name,
|
||||
node_id,
|
||||
)
|
||||
# Match the Python SWA demotion: preserve FULL and descendants;
|
||||
# the resumed native walk tombstones this component after one try.
|
||||
continue
|
||||
self.writing_check(write_back=True)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
# even if host allocation fails.
|
||||
node_id = result.swa_backup_node_id
|
||||
needed = result.swa_backup_num_tokens
|
||||
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
|
||||
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
reason="write_through_unbacked_eviction",
|
||||
)
|
||||
self._accumulate_tracker(tracker, result.tracker)
|
||||
return result.node_id, result.made_progress
|
||||
|
||||
def _evict_device_leaf(
|
||||
self, node_id: NodeId, tracker: dict[ComponentType, int]
|
||||
|
||||
Reference in New Issue
Block a user