[unified memory pool] Allocate without transfer waits, pin floating SWA reservations, drop the HiCache startup asserts (#43206)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-09 20:19:30 -07:00
committed by GitHub
co-authored by Hanming Lu ZYHowell
parent a65f4ffba5
commit f2e9a2bed1
23 changed files with 749 additions and 565 deletions
@@ -659,15 +659,6 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"preserve unified page-envelope indices and compaction lifetimes. "
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
)
if cfg.enable_hierarchical_cache:
assert cfg.pp_size == 1, (
"--enable-unified-memory with hierarchical cache does not support "
"pipeline parallelism (--pp-size > 1)."
)
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
"--enable-unified-memory with hierarchical cache requires lazy "
"compaction so pending H2D physical reservations remain stable."
)
assert not (
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
and cfg.disaggregation_decode_retraction_backup == "host_pool"
@@ -1357,7 +1357,8 @@ class PrefillAdder:
):
full_load_tokens = req.host_hit_length
if (
isinstance(self.tree_cache, UnifiedRadixCache)
self.memory_budget.load_back_needs_full_tokens
and isinstance(self.tree_cache, UnifiedRadixCache)
and self.tree_cache.buffer_pipeline is None
and not (
self.tree_cache.linker is not None
@@ -335,33 +335,6 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
lazy_compaction=self.lazy_compaction,
)
def bind_swa_for_loaded_rows(
self, full_token_ids: torch.Tensor
) -> Optional[torch.Tensor]:
"""Bind SWA pages to resident or newly loaded full-attention virtual IDs.
Bind before translating: unbound pages translate to the padding sink.
Return physical IDs, or None if capacity cannot be reclaimed.
"""
ids = full_token_ids.to(torch.int64)
if ids.numel() == 0:
return ids
ps = self.page_size
pages = torch.unique(ids // ps)
# Tombstones (-1) and the padding sink (0) both need a physical binding.
unbound = pages[self.swa_attn_allocator.virtual_to_physical[pages] <= 0]
need = int(unbound.numel()) * ps
if need:
if need > self.swa_available_size():
return None
if (
need > self.swa_attn_allocator.available_size()
and not _relieve_for_alloc(self.swa_attn_allocator, need)
):
return None
self.swa_attn_allocator.alloc_with_virtual(unbound)
return self.translate_loc_from_full_to_swa(ids)
def set_host_transfer_move_gate(self, gate: Callable[[], bool]) -> None:
"""Block page relocation while host transfers use resolved device indices."""
install_move_gate(
@@ -953,8 +953,6 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
with record_function("MultiEndedAlloc._alloc_bind_fast_or_slow"):
if N == 0:
return torch.empty(0, dtype=torch.int64, device=self.device)
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
self._wait_hicache_transfers()
# FAST PATH: eager, or lazy with no current holes.
if not self.lazy_compaction or self._free_phys_pages.numel() == 0:
@@ -2014,6 +2012,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self._pending_hicache_load_pages = 0
def _wait_hicache_transfers(self) -> None:
"""Order the current stream after outstanding HiCache copies. Only moves
and frees need this: a copy's source and target are not free pages until
its finish event is observed, so allocation never reuses them."""
if not self._hicache_transfer_done_events:
return
current_stream = torch.cuda.current_stream()
@@ -2041,26 +2042,23 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
if need_size > self.available_size():
if not _relieve_for_alloc(self, need_size):
return None
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
self._wait_hicache_transfers()
physical_pages = self.take_physical_pages(need_size // self.page_size)
if physical_pages is None:
return None
if self.lazy_compaction:
self._pending_hicache_load_pages += int(physical_pages.shape[0])
# Blocks page moves (`moves_blocked`) until the H2D is queued or cancelled.
self._pending_hicache_load_pages += int(physical_pages.shape[0])
return self._expand_pages_to_tokens(physical_pages)
def cancel_physical_reservation(self, free_index: torch.Tensor) -> None:
"""Roll back a HiCache physical allocation before its H2D is submitted."""
if free_index is None or free_index.numel() == 0:
return
if self.lazy_compaction:
num_pages = free_index.numel() // self.page_size
assert num_pages <= self._pending_hicache_load_pages, (
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
)
self._pending_hicache_load_pages -= num_pages
num_pages = free_index.numel() // self.page_size
assert num_pages <= self._pending_hicache_load_pages, (
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
)
self._pending_hicache_load_pages -= num_pages
self.free_physical(free_index)
def free_physical(self, free_index: torch.Tensor) -> None:
@@ -28,7 +28,6 @@ from __future__ import annotations
import logging
from array import array
from collections import deque
from dataclasses import replace
from typing import TYPE_CHECKING, Optional
import msgspec
@@ -1532,51 +1531,6 @@ class BufferModePipeline:
),
0,
)
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
binds_swa_to_full = (
swa_entry is not None
and swa_entry.device_indices_from_anchor_fn is not None
)
repair_ranges = []
if binds_swa_to_full and staged_swa:
window_start = span_end - staged_swa
repair_end = min(splice_base, span_end)
if window_start < repair_end:
repair_ranges = cache.tree_core.swa_tombstone_ranges(
key, window_start, repair_end
)
# Only missing SWA rows belong to this load. Existing bindings may
# be in use by another request and must survive allocation rollback.
anchor_parts = []
host_parts = []
for repair_start, repair_end_ in repair_ranges:
anchor_parts.append(prefix_indices[repair_start:repair_end_])
host_parts.append(
slice(repair_start - window_start, repair_end_ - window_start)
)
tail_start = max(splice_base, window_start)
if tail_start < span_end:
anchor_parts.append(
slice(tail_start - splice_base, span_end - splice_base)
)
host_parts.append(
slice(tail_start - window_start, span_end - window_start)
)
for i, transfer in enumerate(load_xfers):
if transfer.name != PoolName.SWA:
continue
# Keep the original complete host bounce for ack/drop release.
load_xfers[i] = replace(
transfer,
host_indices=(
torch.cat([transfer.host_indices[part] for part in host_parts])
if host_parts
else transfer.host_indices[:0]
),
anchor_index_parts=anchor_parts,
)
if not anchor_parts:
load_xfers = [t for t in load_xfers if t.name != PoolName.SWA]
device_indices = cc.load(
host_indices=f.host_indices[trim_tokens:],
@@ -1605,21 +1559,10 @@ class BufferModePipeline:
None,
)
aux_device_releases: list[tuple[PoolName, torch.Tensor]] = []
if swa_dev is not None and binds_swa_to_full:
# Binding already updated the allocator's virtual-to-physical table.
# The tree owns virtual FULL rows, not the H2D kernel-facing IDs.
for repair_start, repair_end_ in repair_ranges:
for action in cache.tree_core.attach_swa_window(
key,
repair_start,
repair_end_,
prefix_indices[repair_start:repair_end_],
):
cache._apply_cache_action(action)
elif swa_dev is not None:
# Register the window's FULL->SWA translation now (attention reads
# through it). Keep SWA slots another request may still hold; their
# redundant H2D destinations are reclaimed at the transfer ack.
if swa_dev is not None:
# SWA loads into fresh reservations. Register the window's FULL->SWA
# translation now (attention reads through it); slots another request
# still holds are kept, and their redundant destinations freed at ack.
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
allocator = cache.token_to_kv_pool_allocator
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
@@ -115,9 +115,6 @@ class PoolTransfer:
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
nodes_to_load: Optional[List[Any]] = None
indices_from_pool: Optional[PoolName] = None
# Full IDs backing a dependent device allocation: resident tensors or
# slices of the full rows allocated by this load, in transfer order.
anchor_index_parts: Optional[List[torch.Tensor | slice]] = None
@dataclass(frozen=True)
@@ -1719,7 +1719,6 @@ class HybridCacheController(BaseHiCacheController):
return None
newly_allocated: list[tuple[PoolTransfer, Callable, torch.Tensor]] = []
derived_transfers: list[PoolTransfer] = []
anchor_transfers = []
def rollback_allocated() -> None:
for prev_pool, prev_free_fn, prev_indices in reversed(newly_allocated):
@@ -1735,11 +1734,6 @@ class HybridCacheController(BaseHiCacheController):
continue
if pool.device_indices is not None or pool.host_indices is None:
continue
if entry.device_indices_from_anchor_fn is not None:
# Allocate independent pools first: their allocation/eviction
# can compact SWA before its physical IDs are captured.
anchor_transfers.append((pool, entry))
continue
# device_alloc_fn / device_free_fn override entry.device_pool's
# methods for pools whose device_pool is a raw KV pool (layout)
# rather than an allocator (e.g. SWA).
@@ -1758,28 +1752,6 @@ class HybridCacheController(BaseHiCacheController):
pool.device_indices = indices
newly_allocated.append((pool, free_fn, indices))
for pool, entry in anchor_transfers:
if kv_device_indices is None or not pool.anchor_index_parts:
rollback_allocated()
return None
anchor_indices = torch.cat(
[
kv_device_indices[part] if isinstance(part, slice) else part
for part in pool.anchor_index_parts
]
)
assert len(anchor_indices) == len(pool.host_indices)
bind = entry.device_indices_from_anchor_fn
indices = bind(anchor_indices)
if indices is None and entry.device_evict_fn:
entry.device_evict_fn(len(anchor_indices))
indices = bind(anchor_indices)
if indices is None:
rollback_allocated()
return None
pool.device_indices = indices
newly_allocated.append((pool, entry.device_free_fn, anchor_indices))
# Assign indices to deferred pools from their source.
for pool in derived_transfers:
if pool.indices_from_pool == PoolName.KV:
@@ -210,7 +210,6 @@ def build_pool_entry(
device_evict_fn: Optional[Callable[[int], Any]] = None,
device_alloc_fn: Optional[Callable[[int], Any]] = None,
device_free_fn: Optional[Callable[[Any], Any]] = None,
device_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
packed_draft_device_pools: tuple[Any, ...] = (),
) -> PoolEntry:
return PoolEntry(
@@ -223,7 +222,6 @@ def build_pool_entry(
device_evict_fn=device_evict_fn,
device_alloc_fn=device_alloc_fn,
device_free_fn=device_free_fn,
device_indices_from_anchor_fn=device_indices_from_anchor_fn,
packed_draft_device_pools=packed_draft_device_pools,
)
@@ -271,21 +269,16 @@ def build_kv_only_group(
)
def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
def _swa_allocation_callbacks(allocator) -> dict:
"""Keep allocation and rollback in the same ID space for every SWA stack."""
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
if isinstance(allocator, MultiEndedAllocator):
# Unified SWA loads into physical reservations, bound before the H2D.
return dict(
device_alloc_fn=allocator.alloc_physical,
device_free_fn=allocator.cancel_physical_reservation,
)
if bind is not None:
assert free_bound is not None
return dict(
device_indices_from_anchor_fn=bind,
device_free_fn=free_bound,
)
if allocator is None:
return {}
return dict(device_alloc_fn=allocator.alloc, device_free_fn=allocator.free)
@@ -327,8 +320,6 @@ def build_hybrid_swa_group(
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
swa_attn_allocator: Any = None,
swa_indices_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
swa_free_from_anchor_fn: Optional[Callable[[Any], Any]] = None,
mtp_swa_device_pools: tuple[Any, ...] = (),
) -> HostPoolGroup:
"""Anchor (full) + SWA host pool group for a hybrid-SWA device pool."""
@@ -394,11 +385,7 @@ def build_hybrid_swa_group(
transfer_layer_id_max=transfer_layer_id_max + len(mtp_swa_device_pools),
host_evict_fn=host_swa_evict_fn,
device_evict_fn=device_swa_evict_fn,
**_swa_allocation_callbacks(
swa_attn_allocator,
swa_indices_from_anchor_fn,
swa_free_from_anchor_fn,
),
**_swa_allocation_callbacks(swa_attn_allocator),
packed_draft_device_pools=mtp_swa_device_pools,
),
]
@@ -499,17 +486,6 @@ def build_hybrid_swa_stack(
device_swa_evict_fn=device_swa_evict_fn,
# For SWA hybrid, device allocation goes through the inner allocator.
swa_attn_allocator=params.token_to_kv_pool_allocator.swa_attn_allocator,
# Unified SWA binds pages to the full pool's virtual IDs instead.
swa_indices_from_anchor_fn=(
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
if get_memory().enable_unified_memory
else None
),
swa_free_from_anchor_fn=(
params.token_to_kv_pool_allocator.free_swa
if get_memory().enable_unified_memory
else None
),
mtp_swa_device_pools=mtp_swa_device_pools,
)
cache_controller = HybridCacheController(
@@ -1329,15 +1305,7 @@ def build_hybrid_mamba_swa_stack(
transfer_layer_id_max=transfer_layer_id_max,
host_evict_fn=host_swa_evict_fn,
device_evict_fn=device_swa_evict_fn,
**_swa_allocation_callbacks(
swa_attn_allocator,
params.token_to_kv_pool_allocator.bind_swa_for_loaded_rows
if get_memory().enable_unified_memory
else None,
params.token_to_kv_pool_allocator.free_swa
if get_memory().enable_unified_memory
else None,
),
**_swa_allocation_callbacks(swa_attn_allocator),
),
build_pool_entry(
name=PoolName.MAMBA,
@@ -25,10 +25,6 @@ class PoolEntry:
device_evict_fn: Callable[[int], Any] | None = None
device_alloc_fn: Callable[[int], Any] | None = None
device_free_fn: Callable[[Any], Any] | None = None
# Bind rows to the anchor's virtual IDs when pools share an ID space.
# Return buffer indices, or None if allocation fails. Rollback through
# device_free_fn takes the anchor's virtual IDs, not the returned indices.
device_indices_from_anchor_fn: Callable[[Any], Any] | None = None
packed_draft_device_pools: tuple[Any, ...] = ()
@@ -62,6 +62,9 @@ def estimate_swa_kv_tokens(
class PrefillBudget:
"""Fixed token pool. Offsets include pending allocations and decode headroom."""
# When False, admission skips the FULL spec and passes the host hit length.
load_back_needs_full_tokens = False
def __init__(self, allocator, tree_cache, *, num_mixed_decode_tokens: int = 0):
self.allocator = allocator
self.tree_cache = tree_cache
@@ -315,6 +318,8 @@ class SWAPrefillBudget(PrefillBudget):
class SharedSWAPrefillBudget(SWAPrefillBudget):
"""FULL and SWA reservations compete for the same physical byte budget."""
load_back_needs_full_tokens = True
def __init__(self, *args, num_mixed_decode_tokens=0, **kwargs):
super().__init__(
*args, num_mixed_decode_tokens=num_mixed_decode_tokens, **kwargs
@@ -568,11 +568,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
result = EvictDeviceNextNodeResult(
node_id=binding_result.node_id,
made_progress=binding_result.made_progress,
backup_kv=(
_cache_action_from_tagged(binding_result.backup_kv)
if binding_result.backup_kv is not None
else None
),
unbacked_tokens=binding_result.unbacked_tokens,
mamba_backup_node_id=binding_result.mamba_backup_node_id,
swa_backup_node_id=binding_result.swa_backup_node_id,
@@ -788,9 +783,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def set_hicache_enabled(self) -> None:
self._binding.set_hicache_enabled()
def enable_swa_write_back_eviction_barrier(self) -> None:
self._binding.enable_swa_write_back_eviction_barrier()
def set_host_memory_buffer_only(self) -> None:
self._binding.set_host_memory_buffer_only()
@@ -961,27 +953,6 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
kv_xfer = _transfer_from_binding(kv_xfer)
comp_xfers = _comp_xfers_from_binding(comp_xfers)
swa_xfers = comp_xfers.get(ComponentType.SWA, ())
if swa_xfers:
full_node_ids = kv_xfer.nodes_to_load or []
full_load_slices = {}
offset = 0
for full_node_id, count in zip(
full_node_ids, self._binding.get_node_key_lengths(full_node_ids)
):
full_load_slices[full_node_id] = slice(offset, offset + count)
offset += count
for transfer in swa_xfers:
# SWA may have holes between resident nodes, or reload while
# FULL stays resident. Preserve the SWA transfer's node order.
transfer.anchor_index_parts = [
(
full_load_slices[nid]
if nid in full_load_slices
else self.get_component_device_value(nid, ComponentType.FULL)
)
for nid in transfer.nodes_to_load or ()
]
return kv_xfer, comp_xfers
def prefetch_anchor_info(
@@ -2494,21 +2494,6 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
nodes_to_load=[],
)
return empty_kv, {}
# SWA can be evicted independently of FULL, including holes between
# resident SWA nodes. Describe precisely which full rows back it.
full_load_slices = {}
offset = 0
for nid in kv_xfer.nodes_to_load or ():
count = len(self.node_by_id(nid).key)
full_load_slices[nid] = slice(offset, offset + count)
offset += count
for xfer in comp_xfers.get(ComponentType.SWA, ()):
xfer.anchor_index_parts = [
full_load_slices[nid]
if nid in full_load_slices
else self.node_by_id(nid).component_data[BASE_COMPONENT_TYPE].value
for nid in xfer.nodes_to_load or ()
]
return kv_xfer, comp_xfers
def prefetch_anchor_info(
@@ -52,7 +52,6 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
"""
node_id: Optional[NodeId] = None
backup_kv: Optional[BackupKV] = None
made_progress: bool = False
unbacked_tokens: int = 0
mamba_backup_node_id: Optional[NodeId] = None
@@ -523,13 +522,6 @@ class UnifiedTreeCoreInterface(ABC):
"""Mark the host tier (HiCache) as wired."""
...
def enable_swa_write_back_eviction_barrier(self) -> None:
"""Enable a backend-managed barrier when needed.
The Python core demotes through SWAComponent directly. Native cores
may return a backup action to the cache executor before eviction.
"""
@abstractmethod
def set_host_memory_buffer_only(self) -> None:
"""Mark the host tier as buffer-only: one node staged per backup intent."""
@@ -532,13 +532,6 @@ class UnifiedRadixCache(BasePrefixCache):
self.cache_controller is not None
and self.cache_controller.write_policy == "write_back"
)
# Preserve the SWA host window before device eviction makes it unrecoverable.
if (
get_memory().enable_unified_memory
and self.host_memory_mode == "cache"
and self.tree_core.has_swa_host_pool
):
self.tree_core.enable_swa_write_back_eviction_barrier()
# Pre-seed the logical dropped-tokens series.
if self.metrics_collector is not None and self.cache_controller is not None:
reasons = ["host_pressure"]
@@ -805,74 +798,45 @@ class UnifiedRadixCache(BasePrefixCache):
def _evict_device_next_node(
self, component_type: ComponentType, tracker: dict[ComponentType, int]
) -> tuple[Optional[NodeId], bool]:
"""Advance the walk, completing pending host-backup barriers."""
while True:
result = self.tree_core.evict_device_next_node(component_type, tracker)
if result.mamba_backup_node_id is not None:
assert component_type == ComponentType.MAMBA and result.node_id is None
assert (
not result.device_frees
and not result.host_frees
and not result.tracker
)
# Reserve a host state slot and wait for the backup acknowledgment
# before freeing device state. If allocation fails, eviction still
# proceeds to make room on the device.
node_id = result.mamba_backup_node_id
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
self.evict_host(1, ComponentType.MAMBA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_mamba_state_eviction(node_id)
elif result.swa_backup_node_id is not None:
assert component_type == ComponentType.SWA and result.node_id is None
assert (
not result.device_frees
and not result.host_frees
and not result.tracker
)
# The backup can cover several unbacked SWA segments. Reserve host
# space for the whole window before copying it, then resume eviction
# even if host allocation fails.
node_id = result.swa_backup_node_id
needed = result.swa_backup_num_tokens
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
if (
swa_host_pool is not None
and swa_host_pool.available_size() < needed
):
self.evict_host(needed, ComponentType.SWA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
result.unbacked_tokens,
reason="write_through_unbacked_eviction",
)
self._accumulate_tracker(tracker, result.tracker)
if result.backup_kv is None:
return result.node_id, result.made_progress
assert result.node_id is None
assert self.buffer_pipeline is None, (
"SWA write-back eviction barriers are cache-mode only"
"""Advance the eviction walk one node, consuming its step result."""
result = self.tree_core.evict_device_next_node(component_type, tracker)
if result.mamba_backup_node_id is not None:
assert component_type == ComponentType.MAMBA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
)
written = self._execute_and_commit_kv_backup(
result.backup_kv, write_back=True
# Reserve a host state slot and wait for the backup acknowledgment
# before freeing device state. If allocation fails, eviction still
# proceeds to make room on the device.
node_id = result.mamba_backup_node_id
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
self.evict_host(1, ComponentType.MAMBA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_mamba_state_eviction(node_id)
elif result.swa_backup_node_id is not None:
assert component_type == ComponentType.SWA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
)
if written <= 0:
node_id = result.backup_kv.node_ids[0]
logger.warning(
"write_back: auxiliary backup failed under host pressure "
"(component=%s, node=%d); dropping only the component",
component_type.name,
node_id,
)
# Match the Python SWA demotion: preserve FULL and descendants;
# the resumed native walk tombstones this component after one try.
continue
self.writing_check(write_back=True)
# The backup can cover several unbacked SWA segments. Reserve host
# space for the whole window before copying it, then resume eviction
# even if host allocation fails.
node_id = result.swa_backup_node_id
needed = result.swa_backup_num_tokens
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
self.evict_host(needed, ComponentType.SWA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
result.unbacked_tokens,
reason="write_through_unbacked_eviction",
)
self._accumulate_tracker(tracker, result.tracker)
return result.node_id, result.made_progress
def _evict_device_leaf(
self, node_id: NodeId, tracker: dict[ComponentType, int]