Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: houseroad <houseroad@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
1971 lines
83 KiB
Python
1971 lines
83 KiB
Python
"""Buffer-only mode transfer pipelines for the unified radix cache.
|
|
|
|
``BufferModePipeline`` owns all buffer-mode state and the two pipelines that
|
|
move KV through the transient host staging buffer:
|
|
|
|
- backup (write path): admission-gated FIFO intents, head-of-line D2H
|
|
staging launches, storage writes at the D2H ack, staging freed at the
|
|
storage ack;
|
|
- load back (read path): completed storage fetches parked as op-owned host
|
|
bounces, consumed at prefill admission via a device alloc + layer-gated
|
|
H2D + plain tree insert, bounce freed at the H2D ack.
|
|
|
|
The pipeline is an intimate collaborator of ``UnifiedRadixCache``: it is
|
|
constructed by ``init_hicache`` only when ``--hicache-host-memory-mode
|
|
buffer_only`` is active, and it drives tree/controller operations (insert,
|
|
match, evict, lock refs, cache actions) through the owning cache. All
|
|
buffer-mode-only state lives here; the cache dispatches to this object at
|
|
its mode branches.
|
|
|
|
TP-lockstep contract: every mutation runs on the scheduler thread at
|
|
rank-synchronized points (insert walks, rank-MIN-reduced drains, ack
|
|
drains), so per-rank state never diverges. There is no runtime
|
|
verification; a violation surfaces as an unexplained collective hang.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from array import array
|
|
from collections import deque
|
|
from dataclasses import replace
|
|
from typing import TYPE_CHECKING, Optional
|
|
|
|
import msgspec
|
|
import torch
|
|
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.managers.cache_controller import HICACHE_WRITE_STAGING_POOL_FRACTION
|
|
from sglang.srt.mem_cache.base_prefix_cache import (
|
|
CacheRequestHandle,
|
|
DecLockRefParams,
|
|
EvictParams,
|
|
InitLoadBackParams,
|
|
InsertParams,
|
|
InsertResult,
|
|
MatchPrefixParams,
|
|
)
|
|
from sglang.srt.mem_cache.hicache_storage import (
|
|
PoolHitPolicy,
|
|
PoolName,
|
|
PoolTransfer,
|
|
SidecarPoolSpec,
|
|
)
|
|
from sglang.srt.mem_cache.pool_host.base import HostKVCache
|
|
from sglang.srt.mem_cache.radix_cache import RadixKey
|
|
from sglang.srt.mem_cache.storage_prefetch import (
|
|
StagedPrefetchPlan,
|
|
WindowLoadBackPlan,
|
|
)
|
|
from sglang.srt.mem_cache.unified_cache.components import (
|
|
CacheTransferPhase,
|
|
ComponentType,
|
|
)
|
|
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
|
BufferBackupSnapshot,
|
|
BufferBackupState,
|
|
NodeId,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from sglang.srt.managers.schedule_batch import Req
|
|
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
|
PrefetchOperation,
|
|
)
|
|
from sglang.srt.mem_cache.pool_host import HostPoolGroup
|
|
from sglang.srt.mem_cache.unified_cache.components import SWAComponent
|
|
from sglang.srt.mem_cache.unified_cache.components.base import (
|
|
BufferLoadBackContext,
|
|
)
|
|
from sglang.srt.mem_cache.unified_radix_cache import (
|
|
UnifiedRadixCache,
|
|
_OngoingPrefetch,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
# Prefetch declines log under the cache's category, with its other outcomes.
|
|
prefetch_logger = logging.getLogger("sglang.srt.mem_cache.unified_radix_cache")
|
|
|
|
|
|
def _minimum_full_transfer_tokens(
|
|
host_pool: HostKVCache, prefetch_threshold: int
|
|
) -> int:
|
|
page_size = host_pool.page_size
|
|
return max(page_size, -(-prefetch_threshold // page_size) * page_size)
|
|
|
|
|
|
class _UnifiedBackupIntent(msgspec.Struct):
|
|
"""One pool of one node queued for a buffer-mode write (unpinned while
|
|
queued); a derived sidecar rides its source pool's intent.
|
|
|
|
Snapshots node identity at enqueue time: a split rewrites the node's
|
|
key/hash in place while these copies stay intact, so a key-length change
|
|
detects a split (repaired during queue refresh) and a missing FULL device
|
|
value detects eviction (``_validate_backup_intent``). ``keys`` is the
|
|
admission view (it sizes drop metrics); the D2H launch re-reads rows and
|
|
keys off the node.
|
|
"""
|
|
|
|
snapshot: BufferBackupSnapshot
|
|
pool: PoolName
|
|
keys: list[str]
|
|
|
|
|
|
class _UnifiedBufferBackupEntry(msgspec.Struct):
|
|
"""A buffer-mode backup after its D2H launch: intent + staging slots.
|
|
|
|
``host_indices`` are the KV staging slots (empty for another pool's
|
|
intent); ``aux_xfers`` the staged transfers and riding sidecars; all are
|
|
freed at the storage-write ack (host memory is never a cache tier).
|
|
``keys_by_pool`` are the keys the write stores, learned at the ack.
|
|
"""
|
|
|
|
intent: _UnifiedBackupIntent
|
|
host_indices: torch.Tensor
|
|
aux_xfers: list[PoolTransfer]
|
|
lock_params: DecLockRefParams
|
|
occupied_units: int
|
|
keys_by_pool: dict[PoolName, list[str]]
|
|
|
|
|
|
class _StagedPrefetch(msgspec.Struct):
|
|
"""A completed buffer-mode fetch parked until prefill admission: only
|
|
the op-owned host bounce exists (no device state, nothing in the tree).
|
|
"""
|
|
|
|
request: CacheRequestHandle
|
|
key_tokens: array
|
|
extra_key: Optional[str]
|
|
cache_salt: Optional[str]
|
|
matched_len: int
|
|
num_tokens: int
|
|
occupied_tokens: int
|
|
host_indices: torch.Tensor
|
|
aux_xfers: list[PoolTransfer]
|
|
hash_values: list[str]
|
|
operation_id: int
|
|
|
|
|
|
class _OngoingBufferLoadBack(msgspec.Struct):
|
|
"""A queued H2D load, registered after its prefix is inserted.
|
|
|
|
The full host bounce, and any redundant device slots a component's load
|
|
context reports, remain owned here until the copy completes.
|
|
``load_transfers`` is the transfer list submitted to H2D; ``aux_xfers``
|
|
retains the original staging owners and ``load_contexts`` each
|
|
component's part in the load.
|
|
"""
|
|
|
|
request: CacheRequestHandle
|
|
num_tokens: int
|
|
occupied_tokens: int
|
|
aux_xfers: list[PoolTransfer]
|
|
host_indices: torch.Tensor
|
|
hash_values: list[str]
|
|
aux_device_releases: list[tuple[PoolName, torch.Tensor]]
|
|
device_indices: torch.Tensor
|
|
load_transfers: list[PoolTransfer]
|
|
load_contexts: list[BufferLoadBackContext]
|
|
|
|
|
|
class _AnchorLock(msgspec.Struct):
|
|
"""Pins a staged prefetch's FULL device anchor until consumption."""
|
|
|
|
node_id: NodeId
|
|
tokens: int
|
|
|
|
|
|
def validate_buffer_only_stack(
|
|
sidecar_pool_specs: list[SidecarPoolSpec],
|
|
host_pool_group: HostPoolGroup,
|
|
swa_component: Optional[SWAComponent],
|
|
storage_prefetch_threshold: int = 256,
|
|
*,
|
|
auxiliary_swa_component: Optional[SWAComponent] = None,
|
|
) -> None:
|
|
"""Post-assembly buffer-mode fences.
|
|
|
|
Sidecars reuse their source pool's transient slot ids, so every sidecar
|
|
host pool must expose the full source slot namespace. unified_kv SWA
|
|
(device-only ring, never offloaded) still has no staging path.
|
|
"""
|
|
entry_map = host_pool_group.entry_map
|
|
for spec in sidecar_pool_specs:
|
|
source = entry_map.get(spec.indices_from_pool)
|
|
sidecar = entry_map.get(spec.pool_name)
|
|
if source is None or sidecar is None:
|
|
raise ValueError(
|
|
"--hicache-host-memory-mode buffer_only sidecar pool mapping "
|
|
f"is incomplete: pool={spec.pool_name}, "
|
|
f"indices_from_pool={spec.indices_from_pool}."
|
|
)
|
|
source_size = source.host_pool.logical_size
|
|
sidecar_size = sidecar.host_pool.logical_size
|
|
if sidecar_size < source_size:
|
|
raise ValueError(
|
|
"--hicache-host-memory-mode buffer_only sidecar host pool is "
|
|
"smaller than its index source: "
|
|
f"pool={spec.pool_name}, host_slots={sidecar_size}, "
|
|
f"source={spec.indices_from_pool}, source_slots={source_size}."
|
|
)
|
|
window_pools = []
|
|
for component in (swa_component, auxiliary_swa_component):
|
|
if component is None:
|
|
continue
|
|
pool = component._swa_kv_pool_host
|
|
if pool is None:
|
|
raise ValueError(
|
|
"--hicache-host-memory-mode buffer_only on SWA models "
|
|
"requires an SWA host staging pool; the unified_kv layout "
|
|
"keeps SWA as a device-only ring."
|
|
)
|
|
window_pools.append((pool, component.full_window_pages * pool.page_size))
|
|
|
|
for pool, window_tokens in window_pools:
|
|
# Every window needs a write bounce plus a loads-priority reserve.
|
|
# For a shared arena the reserve must hold all windows together.
|
|
shared_domain = (
|
|
pool.shared_allocation_domain if isinstance(pool, HostKVCache) else None
|
|
)
|
|
if shared_domain is not None:
|
|
full_host_pool = host_pool_group.anchor_entry.host_pool
|
|
min_full_tokens = _minimum_full_transfer_tokens(
|
|
full_host_pool, storage_prefetch_threshold
|
|
)
|
|
shared_pools = [
|
|
(host, tokens)
|
|
for host, tokens in [(full_host_pool, min_full_tokens), *window_pools]
|
|
if isinstance(host, HostKVCache)
|
|
and host.shared_allocation_domain is shared_domain
|
|
]
|
|
one_transfer_bytes = sum(
|
|
tokens * host.size_per_token for host, tokens in shared_pools
|
|
)
|
|
one_transfer = [(host.pool_label, tokens) for host, tokens in shared_pools]
|
|
enough_capacity = shared_domain.can_fit_many_then(
|
|
one_transfer, one_transfer, empty=True
|
|
)
|
|
capacity = f"{shared_domain.capacity_bytes} shared bytes"
|
|
requirement = f"{2 * one_transfer_bytes} shared bytes"
|
|
else:
|
|
enough_capacity = pool.size >= 2 * window_tokens
|
|
capacity = f"{pool.size} SWA tokens"
|
|
requirement = f"{2 * window_tokens} SWA tokens"
|
|
if not enough_capacity:
|
|
raise ValueError(
|
|
"--hicache-host-memory-mode buffer_only requires a host arena "
|
|
"large enough for two minimum Full/SWA transfers "
|
|
f"({requirement}; got {capacity}): one staging a write "
|
|
"while one stays reserved for prefetch window allocs."
|
|
)
|
|
|
|
|
|
class BufferModePipeline:
|
|
"""All buffer-mode state plus the backup and load-back pipelines.
|
|
|
|
Constructed by ``UnifiedRadixCache.init_hicache`` when host memory mode
|
|
is ``buffer_only``; ``cache.buffer_pipeline is None`` elsewhere, which
|
|
the cache's mode branches use as the dispatch test.
|
|
|
|
Subclasses may replace the complete admission or anchor operations when
|
|
their resident layout differs. The allocation/queue, staged-record,
|
|
insertion and acknowledgment methods below preserve the shared ownership
|
|
lifecycle. A load receipt is passed from allocation through insertion to
|
|
acknowledgment; published device rows belong to the tree. An allocation
|
|
failure must roll back new device reservations before returning None.
|
|
Never free in-flight destinations after a publication failure.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
cache: UnifiedRadixCache,
|
|
swa_component: Optional[SWAComponent],
|
|
write_backlog_cap: int,
|
|
max_context_len: int = 0,
|
|
*,
|
|
auxiliary_swa_component: Optional[SWAComponent] = None,
|
|
):
|
|
self._cache = cache
|
|
self._swa_component = swa_component
|
|
self._auxiliary_swa_component = auxiliary_swa_component
|
|
# Metadata-only pending-write backlog cap, measured in unique FULL
|
|
# snapshot spans; beyond it new spans are dropped at admission.
|
|
self.write_backlog_cap = write_backlog_cap
|
|
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
|
|
|
kvcache = cache.token_to_kv_pool_allocator.get_kvcache()
|
|
full_pool = kvcache.full_kv_pool if isinstance(kvcache, SWAKVPool) else kvcache
|
|
# Clamp by admission headroom: pins must leave room for the largest
|
|
# allowed request, else a queued hold can wedge admission permanently
|
|
# (pool full, nothing retractable). No-headroom pools take no pins.
|
|
self.anchor_lock_cap_tokens = max(
|
|
0,
|
|
min(
|
|
int(envs.SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP.get() * full_pool.size),
|
|
full_pool.size - max_context_len,
|
|
),
|
|
)
|
|
if self.anchor_lock_cap_tokens == 0:
|
|
logger.warning(
|
|
"BufferModePipeline anchor_lock_cap_tokens=0 (pool=%d, "
|
|
"max_context_len=%d): every prefetch launches with its splice "
|
|
"base unpinned. Shrink --context-length or grow the KV pool.",
|
|
full_pool.size,
|
|
max_context_len,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"BufferModePipeline anchor_lock_cap_tokens=%d",
|
|
self.anchor_lock_cap_tokens,
|
|
)
|
|
self.reset()
|
|
|
|
# Arbitrary: a deferral means the admission budget and the allocator disagree
|
|
# about free slots, which waiting on decode rarely fixes.
|
|
max_staged_admission_defers: int = 32
|
|
|
|
def reset(self) -> None:
|
|
# Load pipeline: hits awaiting a staging grant (park-and-retry),
|
|
# enqueue-time prefix context, completed prefetches staged until
|
|
# prefill admission, and load-backs in flight (keyed by synthetic
|
|
# negative ack id).
|
|
self.pending_hit_allocs: deque = deque()
|
|
self._staged_admission_defers: dict[CacheRequestHandle, int] = {}
|
|
self._prefetch_prefix_ctx: dict[
|
|
CacheRequestHandle, tuple[list[int], Optional[str], Optional[str]]
|
|
] = {}
|
|
self.staged_prefetches: dict[CacheRequestHandle, _StagedPrefetch] = {}
|
|
self.ongoing_buffer_load_back: dict[int, _OngoingBufferLoadBack] = {}
|
|
# FIFO intents (one per pool of a node) awaiting a D2H slot, and the
|
|
# pools in flight per node (admission to storage ack).
|
|
self.pending_write_queue: deque[_UnifiedBackupIntent] = deque()
|
|
self._queued_span_refs: dict[tuple[NodeId, int], int] = {}
|
|
self.inflight_backup_pools: dict[NodeId, set[PoolName]] = {}
|
|
# Nodes admitted during the current insert walk (None outside one).
|
|
self._walk_admitted: Optional[set[NodeId]] = None
|
|
# Launched writes: D2H in flight (by ack id), then the storage write
|
|
# in flight (by operation id).
|
|
self.ongoing_write_through: dict[int, _UnifiedBufferBackupEntry] = {}
|
|
self.ongoing_backup: dict[int, _UnifiedBufferBackupEntry] = {}
|
|
self._backup_ack_seq = 0
|
|
self.write_staged_tokens_ = 0
|
|
self.write_backlog_tokens_ = 0
|
|
self._backlog_cap_hits = 0
|
|
# Attempt-keyed locks stop stale completions from unlocking retries.
|
|
self.anchor_locks: dict[CacheRequestHandle, _AnchorLock] = {}
|
|
self.anchor_locked_tokens_ = 0
|
|
self._anchor_lock_cap_skips = 0
|
|
|
|
def _swa_components(self):
|
|
return (
|
|
component
|
|
for component in (self._swa_component, self._auxiliary_swa_component)
|
|
if component is not None
|
|
)
|
|
|
|
def _shared_host_domain(self):
|
|
cc = self._cache.cache_controller
|
|
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
|
if not isinstance(anchor, HostKVCache):
|
|
return None
|
|
return anchor.shared_allocation_domain
|
|
|
|
def _transfer_tokens(self, transfer: PoolTransfer) -> int:
|
|
if transfer.host_indices is not None:
|
|
return len(transfer.host_indices)
|
|
if transfer.keys is None:
|
|
return 0
|
|
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
|
|
return 0 if entry is None else len(transfer.keys) * entry.host_pool.page_size
|
|
|
|
def host_allocation_units(
|
|
self,
|
|
host_indices: Optional[torch.Tensor],
|
|
aux_xfers: Optional[list[PoolTransfer]],
|
|
) -> int:
|
|
"""Host usage expressed in anchor-token units for scheduler accounting."""
|
|
if host_indices is None:
|
|
return 0
|
|
return self._host_request_units(len(host_indices), aux_xfers)
|
|
|
|
def _shared_host_requests(
|
|
self, kv_tokens: int, aux_xfers: Optional[list[PoolTransfer]]
|
|
) -> Optional[list[tuple[str, int]]]:
|
|
if self._shared_host_domain() is None:
|
|
return None
|
|
return [
|
|
(pool.pool_label, tokens)
|
|
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
|
|
]
|
|
|
|
def _host_staging_sizes(self, kv_tokens, aux_xfers):
|
|
cc = self._cache.cache_controller
|
|
yield cc.mem_pool_host.anchor_entry.host_pool, kv_tokens
|
|
for transfer in aux_xfers or ():
|
|
if transfer.indices_from_pool is not None:
|
|
continue
|
|
entry = cc.mem_pool_host.entry_map.get(transfer.name)
|
|
if entry is not None:
|
|
yield entry.host_pool, self._transfer_tokens(transfer)
|
|
|
|
def _host_request_units(
|
|
self, kv_tokens: int, aux_xfers: Optional[list[PoolTransfer]]
|
|
) -> int:
|
|
"""Host staging in anchor-token accounting units."""
|
|
cc = self._cache.cache_controller
|
|
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
|
if self._shared_host_domain() is None:
|
|
# Aux transfers staged through the anchor's own host pool (an
|
|
# entry aliasing it) occupy anchor tokens as well.
|
|
return kv_tokens + self._anchor_pool_aux_tokens(aux_xfers or [])
|
|
num_bytes = sum(
|
|
tokens * pool.size_per_token
|
|
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
|
|
)
|
|
return (num_bytes + anchor.size_per_token - 1) // anchor.size_per_token
|
|
|
|
def _shared_backup_fits(self, requests, *, empty: bool = False) -> bool:
|
|
return self._shared_host_domain().can_fit_many_then(
|
|
requests, self._shared_load_reserve_requests(), empty=empty
|
|
)
|
|
|
|
def _shared_load_reserve_requests(self) -> list[tuple[str, int]]:
|
|
cc = self._cache.cache_controller
|
|
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
|
min_full_tokens = _minimum_full_transfer_tokens(
|
|
anchor, self._cache.prefetch_threshold
|
|
)
|
|
requests = [(anchor.pool_label, min_full_tokens)]
|
|
for component in self._swa_components():
|
|
entry = cc.mem_pool_host.entry_map[component.pool_name]
|
|
requests.append(
|
|
(
|
|
entry.host_pool.pool_label,
|
|
component.full_window_pages * entry.host_pool.page_size,
|
|
)
|
|
)
|
|
return requests
|
|
|
|
@property
|
|
def inflight_backup_node_ids(self):
|
|
"""Nodes with a write of any pool queued, staged or awaiting its
|
|
storage ack."""
|
|
return self.inflight_backup_pools.keys()
|
|
|
|
def backup_pending(self, node_id: NodeId) -> bool:
|
|
"""A write of ``node_id`` is queued, staged or awaiting its storage
|
|
ack. A component that drops rows a pending write still reads asks
|
|
here first; the pipeline pins the node only from D2H launch to ack."""
|
|
return node_id in self.inflight_backup_pools
|
|
|
|
def is_idle(self) -> bool:
|
|
"""No queued, staged, or in-flight buffer-mode work or anchor pins."""
|
|
return not (
|
|
self.pending_hit_allocs
|
|
or self.staged_prefetches
|
|
or self.ongoing_buffer_load_back
|
|
or self.pending_write_queue
|
|
or self.inflight_backup_node_ids
|
|
or self.ongoing_write_through
|
|
or self.ongoing_backup
|
|
or self.anchor_locks
|
|
)
|
|
|
|
# ---- backup pipeline (device -> staging -> storage) ----
|
|
|
|
def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int:
|
|
"""Staging tokens aux transfers hold in the anchor's host pool."""
|
|
group = self._cache.cache_controller.mem_pool_host
|
|
anchor_pool = group.anchor_entry.host_pool
|
|
tokens = 0
|
|
for t in aux_xfers:
|
|
if t.indices_from_pool is not None:
|
|
continue
|
|
entry = group.entry_map.get(t.name)
|
|
if entry is not None and entry.host_pool is anchor_pool:
|
|
tokens += self._transfer_tokens(t)
|
|
return tokens
|
|
|
|
def _backup_parent_covered(self, state: BufferBackupState, node_id: NodeId) -> bool:
|
|
"""Only admit a node whose parent is stored/in-flight: writing above
|
|
a dropped parent creates a permanent longest-prefix hole. A component
|
|
may vouch for the parent through a pool of its own."""
|
|
if state.parent_is_root or PoolName.KV in self.inflight_backup_pools.get(
|
|
state.parent_node_id, ()
|
|
):
|
|
return True
|
|
if (
|
|
state.parent_last_hash is not None
|
|
and self._cache.storage_existence_cache.pool(PoolName.KV).contains(
|
|
state.parent_last_hash
|
|
)
|
|
):
|
|
return True
|
|
return any(
|
|
component.buffer_backup_parent_covered(state.parent_node_id, node_id)
|
|
for component in self._cache.components.values()
|
|
)
|
|
|
|
def _log_backup_dropped(self, num_tokens: int) -> None:
|
|
cache = self._cache
|
|
if cache.enable_storage_metrics and cache.storage_metrics_collector is not None:
|
|
cache.storage_metrics_collector.log_backup_dropped_tokens(num_tokens)
|
|
|
|
def _pool_covered(self, pool: PoolName, keys: list[str]) -> bool:
|
|
"""Beliefs plus content past a D2H launch (republished content must
|
|
not re-write while the original write drains)."""
|
|
return self._cache.storage_existence_cache.pool(pool).covered(keys)
|
|
|
|
def _transfer_keys(
|
|
self, transfer: PoolTransfer, hash_values: list[str]
|
|
) -> Optional[list[str]]:
|
|
"""The builder's keys, else the chain's trailing pages, one per staged
|
|
page (the SWA convention); None when the transfer stages nothing."""
|
|
if transfer.keys is not None:
|
|
return list(transfer.keys)
|
|
if transfer.indices_from_pool is not None or transfer.device_indices is None:
|
|
return None
|
|
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
|
|
if entry is None:
|
|
return None
|
|
num_pages = len(transfer.device_indices) // entry.host_pool.page_size
|
|
if num_pages == 0 or num_pages > len(hash_values):
|
|
return None
|
|
return list(hash_values[-num_pages:])
|
|
|
|
def _backup_pool_keys(
|
|
self, snapshot: BufferBackupSnapshot
|
|
) -> dict[PoolName, list[str]]:
|
|
"""Admission view of the keys a backup writes per pool: the KV chain,
|
|
each aux component's BACKUP_HOST transfers, component overrides on
|
|
top, every sidecar under its source's keys."""
|
|
hash_values = snapshot.hash_values
|
|
tree_core = self._cache.tree_core
|
|
pool_keys: dict[PoolName, list[str]] = {PoolName.KV: list(hash_values)}
|
|
overrides = tree_core.buffer_backup_pool_keys(snapshot.node_id, hash_values)
|
|
for ct in self._cache.components:
|
|
# A component that names its pools' keys spares the transfer build.
|
|
if ct == ComponentType.FULL or ct in overrides:
|
|
continue
|
|
transfers = tree_core.build_hicache_transfers(
|
|
ct, snapshot.node_id, CacheTransferPhase.BACKUP_HOST
|
|
)
|
|
for transfer in transfers or ():
|
|
keys = self._transfer_keys(transfer, hash_values)
|
|
if keys:
|
|
pool_keys.setdefault(transfer.name, []).extend(keys)
|
|
for named in overrides.values():
|
|
for pool, keys in named.items():
|
|
if keys:
|
|
pool_keys[pool] = list(keys)
|
|
else:
|
|
pool_keys.pop(pool, None)
|
|
for spec in self._cache.sidecar_pool_specs:
|
|
source = pool_keys.get(spec.indices_from_pool)
|
|
if source:
|
|
pool_keys[spec.pool_name] = list(source)
|
|
return pool_keys
|
|
|
|
@staticmethod
|
|
def _pool_order(pool: PoolName) -> tuple[bool, str]:
|
|
# KV first, then a fixed order: every rank must admit identically.
|
|
return (pool != PoolName.KV, pool.value)
|
|
|
|
def _write_intents(
|
|
self, snapshot: BufferBackupSnapshot
|
|
) -> list[tuple[PoolName, list[str]]]:
|
|
"""(pool, keys) of the pools not believed stored or in flight; a
|
|
sidecar riding a covered pool drives that pool too."""
|
|
pool_keys = self._backup_pool_keys(snapshot)
|
|
sidecars = {
|
|
spec.pool_name: spec.indices_from_pool
|
|
for spec in self._cache.sidecar_pool_specs
|
|
}
|
|
intents: list[tuple[PoolName, list[str]]] = []
|
|
for pool in sorted(pool_keys, key=self._pool_order):
|
|
keys = pool_keys[pool]
|
|
if pool in sidecars or not keys:
|
|
continue
|
|
riders = [
|
|
name
|
|
for name, source in sidecars.items()
|
|
if source == pool and pool_keys.get(name)
|
|
]
|
|
if all(
|
|
self._pool_covered(name, pool_keys[name]) for name in (pool, *riders)
|
|
):
|
|
continue
|
|
intents.append((pool, keys))
|
|
return intents
|
|
|
|
def _sizing(
|
|
self, pool: PoolName, keys: list[str]
|
|
) -> tuple[int, list[PoolTransfer]]:
|
|
"""(KV tokens, keys-only aux transfers) an intent stages, for the
|
|
oversize and budget gates."""
|
|
if pool == PoolName.KV:
|
|
return len(keys) * self._cache.page_size, []
|
|
return 0, [PoolTransfer(name=pool, keys=list(keys))]
|
|
|
|
def _finish_inflight(self, node_id: NodeId, pool: PoolName) -> None:
|
|
pools = self.inflight_backup_pools.get(node_id)
|
|
if pools is None:
|
|
return
|
|
pools.discard(pool)
|
|
if not pools:
|
|
del self.inflight_backup_pools[node_id]
|
|
|
|
@staticmethod
|
|
def _queued_span_key(intent: _UnifiedBackupIntent) -> tuple[NodeId, int]:
|
|
snapshot = intent.snapshot
|
|
return snapshot.node_id, len(snapshot.key)
|
|
|
|
def _retain_queued_span(self, intent: _UnifiedBackupIntent) -> None:
|
|
span = self._queued_span_key(intent)
|
|
refs = self._queued_span_refs.get(span, 0)
|
|
if refs == 0:
|
|
self.write_backlog_tokens_ += span[1]
|
|
self._queued_span_refs[span] = refs + 1
|
|
|
|
def _release_queued_span(self, intent: _UnifiedBackupIntent) -> None:
|
|
span = self._queued_span_key(intent)
|
|
refs = self._queued_span_refs[span]
|
|
if refs == 1:
|
|
del self._queued_span_refs[span]
|
|
self.write_backlog_tokens_ -= span[1]
|
|
else:
|
|
self._queued_span_refs[span] = refs - 1
|
|
|
|
def begin_insert_walk(self) -> None:
|
|
"""Admit each node once per insert walk: the trigger chains every
|
|
unbacked ancestor, so a depth-d path is presented d(d+1)/2 times."""
|
|
self._walk_admitted = set()
|
|
|
|
def end_insert_walk(self) -> None:
|
|
self._walk_admitted = None
|
|
|
|
def enqueue_backup_intent(self, node_id: NodeId) -> None:
|
|
"""Queue one intent per pool with something to write; rejected intents
|
|
are counted and the node re-triggers on a later hit."""
|
|
if not self._cache.enable_storage:
|
|
return
|
|
admitted = self._walk_admitted
|
|
if admitted is not None:
|
|
if node_id in admitted:
|
|
return
|
|
admitted.add(node_id)
|
|
snapshot = self._cache.tree_core.snapshot_buffer_backup(
|
|
node_id, self._cache.hicache_storage_pass_prefix_keys
|
|
)
|
|
if snapshot is None:
|
|
return
|
|
intents = self._write_intents(snapshot)
|
|
if not intents:
|
|
return
|
|
pending = self.inflight_backup_pools.get(node_id, set())
|
|
parent_covered = self._backup_parent_covered(
|
|
BufferBackupState(
|
|
parent_node_id=snapshot.parent_node_id,
|
|
parent_is_root=snapshot.parent_is_root,
|
|
parent_last_hash=snapshot.parent_last_hash,
|
|
),
|
|
node_id,
|
|
)
|
|
page_size = self._cache.page_size
|
|
span = (snapshot.node_id, len(snapshot.key))
|
|
for pool, keys in intents:
|
|
if pool in pending:
|
|
continue
|
|
intent_tokens = len(keys) * page_size
|
|
if (
|
|
span not in self._queued_span_refs
|
|
and self.write_backlog_tokens_ >= self.write_backlog_cap
|
|
):
|
|
# The cap sits at 2x the intrinsic live-backlog ceiling (see
|
|
# init_hicache), so reaching it means leaked accounting or a
|
|
# broken stale sweep — a bug, not load.
|
|
self._backlog_cap_hits += 1
|
|
if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0:
|
|
logger.error(
|
|
"HiCache write backlog cap hit (occurrence %d): "
|
|
"backlog=%d cap=%d queue=%d. Live backlog is bounded "
|
|
"by the device pool span, so this indicates a "
|
|
"stale-sweep or accounting leak.",
|
|
self._backlog_cap_hits,
|
|
self.write_backlog_tokens_,
|
|
self.write_backlog_cap,
|
|
len(self.pending_write_queue),
|
|
)
|
|
self._log_backup_dropped(intent_tokens)
|
|
continue
|
|
# A span larger than the pool's whole staging capacity can never
|
|
# stage; admitting it would wedge the head-of-line queue forever.
|
|
if not parent_covered or self._backup_oversize(*self._sizing(pool, keys)):
|
|
self._log_backup_dropped(intent_tokens)
|
|
continue
|
|
intent = _UnifiedBackupIntent(snapshot=snapshot, pool=pool, keys=list(keys))
|
|
self.pending_write_queue.append(intent)
|
|
self.inflight_backup_pools.setdefault(node_id, set()).add(pool)
|
|
self._retain_queued_span(intent)
|
|
|
|
def _backup_oversize(self, kv_tokens: int, aux_xfers: list[PoolTransfer]) -> bool:
|
|
"""True if any pool's staging need exceeds that pool's write-usable
|
|
capacity (total for KV, total minus the loads-priority margin for aux
|
|
pools — matching ``_aux_budget_blocked``'s admission ceiling): such an
|
|
intent could never stage and would wedge the FIFO head. Aux pools
|
|
staged through the anchor's host pool count against the KV total."""
|
|
cc = self._cache.cache_controller
|
|
shared_requests = self._shared_host_requests(kv_tokens, aux_xfers)
|
|
if shared_requests is not None:
|
|
pool_tokens = cc.mem_pool_host.size
|
|
max_write_units = pool_tokens - pool_tokens // 10
|
|
if self._host_request_units(kv_tokens, aux_xfers) > max_write_units:
|
|
return True
|
|
return not self._shared_backup_fits(shared_requests, empty=True)
|
|
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
|
|
anchor_need = kv_tokens
|
|
for t in aux_xfers:
|
|
entry = cc.mem_pool_host.entry_map.get(t.name)
|
|
if entry is None:
|
|
continue
|
|
need = len(t.keys) * entry.host_pool.page_size
|
|
if entry.host_pool is anchor_pool:
|
|
anchor_need += need
|
|
elif need > entry.host_pool.size - self._aux_loads_margin(
|
|
t.name, entry.host_pool
|
|
):
|
|
return True
|
|
return anchor_need > cc.mem_pool_host.size
|
|
|
|
def _aux_loads_margin(self, pool_name, host_pool) -> int:
|
|
"""Aux-pool tokens reserved for loads: at least one trailing window
|
|
(prepare_prefetch allocates its window here and a failed alloc
|
|
forfeits the whole prefetch), plus a 10% burst absorber mirroring
|
|
live_cap."""
|
|
window_pages = 0
|
|
if pool_name == PoolName.SWA and self._swa_component is not None:
|
|
window_pages = self._swa_component.full_window_pages
|
|
elif (
|
|
pool_name == PoolName.AUXILIARY_SWA
|
|
and self._auxiliary_swa_component is not None
|
|
):
|
|
window_pages = self._auxiliary_swa_component.full_window_pages
|
|
return max(window_pages * host_pool.page_size, host_pool.size // 10)
|
|
|
|
def _validate_backup_intent(
|
|
self, intent: _UnifiedBackupIntent
|
|
) -> Optional[BufferBackupState]:
|
|
# Arena-lookup failure = deleted, key-length mismatch vs the snapshot
|
|
# = split, a None FULL device value = evicted.
|
|
snapshot = intent.snapshot
|
|
return self._cache.tree_core.validate_buffer_backup(
|
|
snapshot.node_id, len(snapshot.key)
|
|
)
|
|
|
|
def _split_pieces(
|
|
self, snapshot: BufferBackupSnapshot
|
|
) -> Optional[list[BufferBackupSnapshot]]:
|
|
"""Resolve a split span, parents first, or return None if it is gone.
|
|
|
|
The original node retains the tail; its ancestors must cover exactly
|
|
the admitted span and end at the original parent.
|
|
"""
|
|
tree_core = self._cache.tree_core
|
|
pass_prefix_keys = self._cache.hicache_storage_pass_prefix_keys
|
|
pieces: list[BufferBackupSnapshot] = []
|
|
node_id, remaining = snapshot.node_id, len(snapshot.key)
|
|
while remaining > 0:
|
|
piece = tree_core.snapshot_buffer_backup(node_id, pass_prefix_keys)
|
|
if piece is None or len(piece.key) > remaining:
|
|
return None
|
|
pieces.append(piece)
|
|
remaining -= len(piece.key)
|
|
node_id = piece.parent_node_id
|
|
if node_id != snapshot.parent_node_id:
|
|
return None
|
|
pieces.reverse()
|
|
return pieces
|
|
|
|
def _refresh_pending_backup_intents(self) -> dict[NodeId, BufferBackupState]:
|
|
"""Drop dead spans and replace split intents in FIFO order.
|
|
|
|
The earliest intent for each (node, pool) wins, including split pieces.
|
|
"""
|
|
if not self.pending_write_queue:
|
|
return {}
|
|
page_size = self._cache.page_size
|
|
queued = {(i.snapshot.node_id, i.pool) for i in self.pending_write_queue}
|
|
survivors: dict[tuple[NodeId, PoolName], _UnifiedBackupIntent] = {}
|
|
states: dict[NodeId, BufferBackupState] = {}
|
|
swept_tokens = 0
|
|
for intent in self.pending_write_queue:
|
|
snapshot = intent.snapshot
|
|
key = (snapshot.node_id, intent.pool)
|
|
if key in survivors:
|
|
# The replacement retains this intent's inflight ownership.
|
|
self._release_queued_span(intent)
|
|
continue
|
|
state = self._validate_backup_intent(intent)
|
|
if state is not None:
|
|
survivors[key] = intent
|
|
states[snapshot.node_id] = state
|
|
continue
|
|
self._finish_inflight(snapshot.node_id, intent.pool)
|
|
self._release_queued_span(intent)
|
|
pieces = self._split_pieces(snapshot)
|
|
if pieces is None:
|
|
swept_tokens += len(intent.keys) * page_size
|
|
continue
|
|
for piece in pieces:
|
|
key = (piece.node_id, intent.pool)
|
|
if key in survivors:
|
|
continue
|
|
if (
|
|
intent.pool in self.inflight_backup_pools.get(piece.node_id, ())
|
|
and key not in queued
|
|
):
|
|
continue # staged or awaiting its storage ack
|
|
keys = dict(self._write_intents(piece)).get(intent.pool)
|
|
if not keys:
|
|
continue # covered since admission
|
|
repaired = _UnifiedBackupIntent(
|
|
snapshot=piece, pool=intent.pool, keys=list(keys)
|
|
)
|
|
survivors[key] = repaired
|
|
states[piece.node_id] = BufferBackupState(
|
|
parent_node_id=piece.parent_node_id,
|
|
parent_is_root=piece.parent_is_root,
|
|
parent_last_hash=piece.parent_last_hash,
|
|
)
|
|
self.inflight_backup_pools.setdefault(piece.node_id, set()).add(
|
|
intent.pool
|
|
)
|
|
self._retain_queued_span(repaired)
|
|
self.pending_write_queue = deque(survivors.values())
|
|
self._log_backup_dropped(swept_tokens)
|
|
return states
|
|
|
|
def flush_pending_writes(self) -> None:
|
|
"""Stage admitted intents, then submit their D2H as one operation.
|
|
|
|
Preparation must not allocate or free L1 KV slots: it only allocates
|
|
host staging, and buffer-mode evict_host is a no-op. Flush before
|
|
returning to scheduler admission, where L1 slots can be reused.
|
|
Each staged source stays locked until its D2H ack.
|
|
"""
|
|
if not self.pending_write_queue:
|
|
return
|
|
cc = self._cache.cache_controller
|
|
states = self._refresh_pending_backup_intents()
|
|
# Loads have priority (writes are deferrable): the write window is
|
|
# the pool minus prefetch occupancy minus a 10% margin, floored at
|
|
# the configured fraction.
|
|
pool_tokens = cc.mem_pool_host.size
|
|
live_cap = max(
|
|
int(HICACHE_WRITE_STAGING_POOL_FRACTION * pool_tokens),
|
|
pool_tokens - cc.prefetch_tokens_occupied - pool_tokens // 10,
|
|
)
|
|
while self.pending_write_queue:
|
|
intent = self.pending_write_queue[0]
|
|
snapshot = intent.snapshot
|
|
node_id = snapshot.node_id
|
|
state = states[node_id]
|
|
intent_tokens = len(intent.keys) * self._cache.page_size
|
|
if not self._backup_parent_covered(state, node_id):
|
|
# Cascade a dropped parent down the chain rather than creating
|
|
# a permanent storage hole.
|
|
self._drop_queue_head(intent_tokens, dropped=True)
|
|
continue
|
|
if (
|
|
self.write_staged_tokens_ >= live_cap
|
|
and self._host_request_units(*self._sizing(intent.pool, intent.keys))
|
|
> 0
|
|
):
|
|
# Wait for current copies to free staging before rebuilding the head.
|
|
break
|
|
launch = self._launch_transfers(intent)
|
|
# The pool may have become covered since admission (an earlier
|
|
# launch this round, a sibling's ack): nothing left to write
|
|
# retires the intent, not a drop.
|
|
if launch is None or all(
|
|
self._pool_covered(pool, keys) for pool, keys in launch[1].items()
|
|
):
|
|
self._drop_queue_head(intent_tokens, dropped=False)
|
|
continue
|
|
device_value, keys_by_pool, transfers = launch
|
|
sizing_xfers = [t for t in transfers if t.indices_from_pool is None]
|
|
kv_tokens = len(device_value)
|
|
if self._backup_oversize(kv_tokens, sizing_xfers):
|
|
# A permanently unstageable head must not block the queue.
|
|
self._drop_queue_head(intent_tokens, dropped=True)
|
|
continue
|
|
shared_domain = self._shared_host_domain()
|
|
staging_at_limit = (
|
|
self.write_staged_tokens_
|
|
+ self._host_request_units(kv_tokens, sizing_xfers)
|
|
> live_cap
|
|
if shared_domain is not None
|
|
else self.write_staged_tokens_ >= live_cap
|
|
and self._host_request_units(kv_tokens, sizing_xfers) > 0
|
|
)
|
|
if staging_at_limit:
|
|
# Yield to live fetch demand; retry next round.
|
|
break
|
|
if self._aux_budget_blocked(kv_tokens, sizing_xfers):
|
|
# An aux pool lacks staging headroom: yield at the gate
|
|
# instead of failing the alloc inside cc.write; acks free
|
|
# aux staging, retry next round.
|
|
break
|
|
if not self._stage_backup_intent(
|
|
intent, device_value, transfers, keys_by_pool
|
|
):
|
|
# Pool full of in-flight staging and nothing reclaimable
|
|
# (the tree never holds host values in buffer mode):
|
|
# defer, head-of-line; pending acks will free slots.
|
|
break
|
|
self.pending_write_queue.popleft()
|
|
|
|
# Submit earlier successes even if a later intent ran out of staging.
|
|
# Do not leave prepared copies deferred across scheduler admission.
|
|
cc.start_writing()
|
|
|
|
def _drop_queue_head(self, intent_tokens: int, *, dropped: bool) -> None:
|
|
"""Retire the FIFO head without a D2H launch; ``dropped`` feeds the
|
|
dropped-tokens metric (a pool with nothing left to write is not a
|
|
drop)."""
|
|
intent = self.pending_write_queue.popleft()
|
|
self._finish_inflight(intent.snapshot.node_id, intent.pool)
|
|
self._release_queued_span(intent)
|
|
if dropped:
|
|
self._log_backup_dropped(intent_tokens)
|
|
|
|
def _build_backup_transfers(
|
|
self, node_id: NodeId, *, kv_only: bool = False
|
|
) -> tuple[
|
|
torch.Tensor, dict[ComponentType, list[PoolTransfer]], Optional[list[str]]
|
|
]:
|
|
"""The node's D2H spec. A FULL component's BACKUP_HOST KV transfer
|
|
replaces the KV rows (its keys name their pages); its other transfers
|
|
ride as aux pools. A KV intent skips the other components' transfers."""
|
|
tree_core = self._cache.tree_core
|
|
if kv_only:
|
|
device_value = tree_core.get_component_device_value(
|
|
node_id, ComponentType.FULL
|
|
)
|
|
comp_xfers = {}
|
|
else:
|
|
device_value, comp_xfers = tree_core.build_backup_spec(node_id)
|
|
full_xfers = tree_core.build_hicache_transfers(
|
|
ComponentType.FULL, node_id, CacheTransferPhase.BACKUP_HOST
|
|
)
|
|
kv_keys = None
|
|
aux: list[PoolTransfer] = []
|
|
for transfer in full_xfers or ():
|
|
if transfer.name == PoolName.KV:
|
|
device_value = (
|
|
transfer.device_indices
|
|
if transfer.device_indices is not None
|
|
else device_value[:0]
|
|
)
|
|
kv_keys = list(transfer.keys or ())
|
|
else:
|
|
aux.append(transfer)
|
|
if aux:
|
|
comp_xfers[ComponentType.FULL] = aux
|
|
return device_value, comp_xfers, kv_keys
|
|
|
|
def _launch_transfers(
|
|
self, intent: _UnifiedBackupIntent
|
|
) -> Optional[tuple[torch.Tensor, dict[PoolName, list[str]], list[PoolTransfer]]]:
|
|
"""(KV rows, keys per written pool, transfers to stage) of one intent,
|
|
off the node's fresh spec; None when nothing is left to write."""
|
|
snapshot = intent.snapshot
|
|
hash_values = snapshot.hash_values
|
|
device_value, comp_xfers, kv_keys = self._build_backup_transfers(
|
|
snapshot.node_id, kv_only=intent.pool == PoolName.KV
|
|
)
|
|
transfers: list[PoolTransfer] = []
|
|
if intent.pool == PoolName.KV:
|
|
keys = hash_values if kv_keys is None else kv_keys
|
|
if not keys or len(device_value) == 0:
|
|
return None
|
|
comp_xfers = {}
|
|
else:
|
|
device_value = device_value[:0]
|
|
keys = []
|
|
for ct, xfers in comp_xfers.items():
|
|
kept = []
|
|
for transfer in xfers:
|
|
if transfer.name != intent.pool or transfer.indices_from_pool:
|
|
continue
|
|
t_keys = self._transfer_keys(transfer, hash_values)
|
|
if not t_keys:
|
|
continue
|
|
if transfer.keys is None:
|
|
transfer.keys = t_keys
|
|
transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES
|
|
kept.append(transfer)
|
|
keys.extend(t_keys)
|
|
if kept:
|
|
transfers.extend(kept)
|
|
comp_xfers[ct] = kept
|
|
if not transfers:
|
|
return None
|
|
comp_xfers = {ct: xfers for ct, xfers in comp_xfers.items() if xfers}
|
|
keys_by_pool = {intent.pool: list(keys)}
|
|
# Sidecars ride their source pool's slots and D2H operation; they
|
|
# allocate no staging of their own.
|
|
for sidecar in self._cache._build_backup_sidecar(device_value, comp_xfers):
|
|
if sidecar.indices_from_pool != intent.pool:
|
|
continue
|
|
sidecar.keys = list(keys)
|
|
keys_by_pool[sidecar.name] = list(keys)
|
|
transfers.append(sidecar)
|
|
return device_value, keys_by_pool, transfers
|
|
|
|
def _stage_backup_intent(
|
|
self,
|
|
intent: _UnifiedBackupIntent,
|
|
device_value: torch.Tensor,
|
|
transfers: list[PoolTransfer],
|
|
keys_by_pool: dict[PoolName, list[str]],
|
|
) -> bool:
|
|
"""Allocate host staging and pin one admitted intent's source.
|
|
|
|
The caller removes successful intents from pending_write_queue and
|
|
submits them together before returning. Return False when staging
|
|
cannot be allocated.
|
|
"""
|
|
cache = self._cache
|
|
cc = cache.cache_controller
|
|
snapshot = intent.snapshot
|
|
# One ack id per write: a node may have several pools in flight.
|
|
self._backup_ack_seq += 1
|
|
ack_id = -(1 << 40) - self._backup_ack_seq
|
|
host_indices = cc.write(
|
|
device_value,
|
|
node_id=ack_id,
|
|
extra_pools=transfers or None,
|
|
flush=False,
|
|
)
|
|
if host_indices is None:
|
|
self._backup_ack_seq -= 1
|
|
return False
|
|
beliefs = cache.storage_existence_cache
|
|
for pool, keys in keys_by_pool.items():
|
|
beliefs.pool(pool).track_inflight(keys)
|
|
# NOTE: no commit_backup — the node must never appear
|
|
# host-resident in buffer mode; staging slots live in the entry.
|
|
lock_params = cache.inc_lock_ref(snapshot.node_id).to_dec_params()
|
|
occupied_units = self.host_allocation_units(host_indices, transfers)
|
|
self.ongoing_write_through[ack_id] = _UnifiedBufferBackupEntry(
|
|
intent=intent,
|
|
host_indices=host_indices,
|
|
aux_xfers=transfers,
|
|
lock_params=lock_params,
|
|
occupied_units=occupied_units,
|
|
keys_by_pool=keys_by_pool,
|
|
)
|
|
self.write_staged_tokens_ += occupied_units
|
|
self._release_queued_span(intent)
|
|
return True
|
|
|
|
def _aux_budget_blocked(self, kv_tokens: int, aux: list[PoolTransfer]) -> bool:
|
|
"""True when an aux pool cannot stage this intent right now (free
|
|
minus the loads-priority margin falls short of the need): defer at
|
|
the gate instead of failing the alloc inside cc.write and blocking
|
|
pure-KV intents behind an unallocatable head. The margin enforces
|
|
loads-have-priority on aux pools the way live_cap does on the KV
|
|
pool; avail already reflects prefetch-held slots, so no occupancy
|
|
subtraction here. Aux pools staged through the anchor's host pool
|
|
are gated by live_cap and the allocation itself."""
|
|
cc = self._cache.cache_controller
|
|
shared_requests = self._shared_host_requests(kv_tokens, aux)
|
|
if shared_requests is not None:
|
|
return not self._shared_backup_fits(shared_requests)
|
|
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
|
|
for t in aux:
|
|
entry = cc.mem_pool_host.entry_map.get(t.name)
|
|
if entry is None or entry.host_pool is anchor_pool:
|
|
continue
|
|
need = len(t.keys) * entry.host_pool.page_size
|
|
headroom = entry.host_pool.available_size() - self._aux_loads_margin(
|
|
t.name, entry.host_pool
|
|
)
|
|
if need > headroom:
|
|
return True
|
|
return False
|
|
|
|
def finish_backup_ack(self, ack_id: int) -> None:
|
|
"""D2H confirmed: drop the device lock and enqueue the storage write
|
|
(which reads from the staging copy, so device eviction may proceed)."""
|
|
entry = self.ongoing_write_through.pop(ack_id)
|
|
intent = entry.intent
|
|
snapshot = intent.snapshot
|
|
self._cache.dec_lock_ref(snapshot.node_id, entry.lock_params)
|
|
|
|
storage_xfers = self._storage_transfers(entry.aux_xfers)
|
|
# Another pool's write carries no KV pages; its keys ride the transfer.
|
|
hash_value = (
|
|
list(entry.keys_by_pool[PoolName.KV]) if intent.pool == PoolName.KV else []
|
|
)
|
|
operation_id = self._cache.cache_controller.write_storage(
|
|
entry.host_indices,
|
|
snapshot.key.token_ids,
|
|
hash_value,
|
|
snapshot.prefix_keys,
|
|
extra_pools=storage_xfers or None,
|
|
)
|
|
self.ongoing_backup[operation_id] = entry
|
|
|
|
@staticmethod
|
|
def _storage_transfers(staged: list[PoolTransfer]) -> list[PoolTransfer]:
|
|
"""One storage transfer per pool: a pool staged in several transfers
|
|
becomes one contiguous key span over their slots; a derived sidecar
|
|
writes its source's key span over the source's slots."""
|
|
merged: dict[PoolName, PoolTransfer] = {}
|
|
for transfer in staged:
|
|
if transfer.indices_from_pool is not None:
|
|
merged[transfer.name] = PoolTransfer(
|
|
name=transfer.name,
|
|
keys=list(transfer.keys),
|
|
hit_policy=transfer.hit_policy,
|
|
indices_from_pool=transfer.indices_from_pool,
|
|
)
|
|
continue
|
|
earlier = merged.get(transfer.name)
|
|
if earlier is None:
|
|
merged[transfer.name] = PoolTransfer(
|
|
name=transfer.name,
|
|
host_indices=transfer.host_indices,
|
|
keys=list(transfer.keys),
|
|
hit_policy=transfer.hit_policy,
|
|
)
|
|
continue
|
|
earlier.keys.extend(transfer.keys)
|
|
earlier.host_indices = torch.cat(
|
|
[earlier.host_indices, transfer.host_indices]
|
|
)
|
|
return list(merged.values())
|
|
|
|
def finish_storage_write_ack(self, operation_id: int) -> None:
|
|
"""Storage ack (rank-synced drain): free the staging and feed each
|
|
written pool's beliefs unconditionally, keeping admission
|
|
TP-deterministic under per-rank backend failures."""
|
|
entry = self.ongoing_backup.pop(operation_id, None)
|
|
if entry is None:
|
|
return
|
|
beliefs = self._cache.storage_existence_cache
|
|
for pool, keys in entry.keys_by_pool.items():
|
|
pool_beliefs = beliefs.pool(pool)
|
|
pool_beliefs.add(keys)
|
|
pool_beliefs.untrack_inflight(keys)
|
|
self._free_staging_now(entry.host_indices, entry.aux_xfers)
|
|
self.write_staged_tokens_ -= entry.occupied_units
|
|
self._finish_inflight(entry.intent.snapshot.node_id, entry.intent.pool)
|
|
|
|
def _free_staging_now(
|
|
self, host_indices: torch.Tensor, aux_xfers: list[PoolTransfer]
|
|
) -> None:
|
|
"""Synchronously free a staging span (KV + aux pools) on the
|
|
scheduler thread; buffer-mode acks/drops all run here, so frees
|
|
land before the tick's next gate reads pool availability."""
|
|
cc = self._cache.cache_controller
|
|
if host_indices is not None and host_indices.numel() > 0:
|
|
cc.mem_pool_host.free(host_indices)
|
|
for t in aux_xfers or ():
|
|
if (
|
|
t.host_indices is None
|
|
or t.host_indices.numel() == 0
|
|
or t.indices_from_pool is not None
|
|
):
|
|
continue
|
|
entry = cc.mem_pool_host.entry_map.get(t.name)
|
|
if entry is not None:
|
|
entry.host_pool.free(t.host_indices)
|
|
|
|
# ---- load back pipeline (storage -> staging -> device) ----
|
|
|
|
def allocate_and_queue_prefetch(
|
|
self,
|
|
operation: PrefetchOperation,
|
|
info: _OngoingPrefetch,
|
|
hit_tokens: int,
|
|
aux_hit_tokens: int,
|
|
) -> bool:
|
|
"""Allocate the hit-sized bounce and queue its storage read.
|
|
|
|
False parks the operation for another allocation attempt; True means
|
|
the operation was queued or declined.
|
|
"""
|
|
cache = self._cache
|
|
cc = cache.cache_controller
|
|
if self._reject_oversized_prefetch(operation, hit_tokens):
|
|
return True
|
|
host_indices, alloc_len = self._allocate_prefetch_host_indices(
|
|
operation, hit_tokens
|
|
)
|
|
if host_indices is None:
|
|
return self._defer_prefetch_for_host_capacity(
|
|
operation.handle, max(alloc_len, aux_hit_tokens)
|
|
)
|
|
if not cache._alloc_prefetch_aux_staging(info, aux_hit_tokens):
|
|
# Same outcome as a KV shortfall: nothing stays staged.
|
|
cc.append_host_mem_release(host_indices=host_indices)
|
|
return self._defer_prefetch_for_host_capacity(
|
|
operation.handle, max(alloc_len, aux_hit_tokens)
|
|
)
|
|
self._queue_prefetch(operation, info, hit_tokens, host_indices, alloc_len)
|
|
return True
|
|
|
|
def _reject_oversized_prefetch(
|
|
self, operation: PrefetchOperation, tokens: int
|
|
) -> bool:
|
|
cache = self._cache
|
|
if cache.cache_controller.can_fit_prefetch_host_buffers(operation, tokens):
|
|
return False
|
|
cache._prefetch_outcome_stats["declined_host_oversize"] += 1
|
|
prefetch_logger.warning(
|
|
"HiCache buffer prefetch declined req=%s: the Full/sidecar "
|
|
"bounce cannot fit in an empty host arena",
|
|
operation.handle,
|
|
)
|
|
cache.revoke_pending_prefetch(operation.handle)
|
|
return True
|
|
|
|
def _allocate_prefetch_host_indices(
|
|
self, operation: PrefetchOperation, tokens: int
|
|
) -> tuple[Optional[torch.Tensor], int]:
|
|
cache = self._cache
|
|
return cache.cache_controller.allocate_storage_hit(
|
|
operation,
|
|
tokens,
|
|
allow_partial=False,
|
|
min_tokens=cache.prefetch_threshold,
|
|
evict_host=cache.evict_host,
|
|
)
|
|
|
|
def _defer_prefetch_for_host_capacity(
|
|
self, request: CacheRequestHandle, tokens: int
|
|
) -> bool:
|
|
# Parked ops hold no pin: release and re-take at the next attempt,
|
|
# which is also how a moved anchor gets noticed.
|
|
self.release_anchor_lock(request)
|
|
self._cache._log_storage_prefetch_deferred(tokens, "host_capacity")
|
|
return False
|
|
|
|
def _queue_prefetch(
|
|
self,
|
|
operation: PrefetchOperation,
|
|
info: _OngoingPrefetch,
|
|
hit_tokens: int,
|
|
host_indices: torch.Tensor,
|
|
alloc_len: int,
|
|
*,
|
|
span_tokens: Optional[int] = None,
|
|
) -> None:
|
|
# alloc_len counts FULL host slots; span_tokens counts accepted key tokens.
|
|
cache = self._cache
|
|
cc = cache.cache_controller
|
|
request = operation.handle
|
|
if span_tokens is None:
|
|
span_tokens = alloc_len
|
|
cache._resolve_storage_prefetch_tokens(
|
|
request, hit_tokens - span_tokens, reason="host_capacity"
|
|
)
|
|
operation.storage_hit_count = span_tokens
|
|
operation.hash_value = operation.hash_value[: alloc_len // cache.page_size]
|
|
operation.host_indices = host_indices
|
|
cache.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
|
operation.buffer_host_occupied_units = self.host_allocation_units(
|
|
host_indices, operation.pool_transfers
|
|
)
|
|
cc.prefetch_tokens_occupied += operation.buffer_host_occupied_units
|
|
cc.prefetch_buffer.put(operation)
|
|
|
|
def try_lock_anchor(
|
|
self, request: CacheRequestHandle, remaining_full_tokens: int
|
|
) -> tuple[str, int]:
|
|
"""Pin the staged prefetch's device anchor so eviction cannot
|
|
invalidate the splice, finding it by re-matching the live tree
|
|
(carried node ids go stale via splits and eviction; the walk is
|
|
O(request hit span)). Returns "locked", "no_anchor" (nothing to pin),
|
|
"cap_skip" (bigger than the whole cap; launches unlocked), "cap_busy"
|
|
(fits, but the budget is taken; the caller parks), "anchor_lost"
|
|
(splice base gone -- the caller re-plans), or "unservable" (resident
|
|
prefix is unreadable -- the caller recomputes)."""
|
|
assert request not in self.anchor_locks, (
|
|
f"prefetch anchor already locked: {request.rid}"
|
|
)
|
|
prefix_tokens, extra_key, cache_salt = self._prefetch_prefix_ctx[request]
|
|
matched_len = len(prefix_tokens)
|
|
assert matched_len + remaining_full_tokens > 0, (
|
|
f"empty prefetch span: {request.rid}"
|
|
)
|
|
full_key_tokens = array("q", prefix_tokens)
|
|
if remaining_full_tokens or self._cache.tree_core.is_eagle:
|
|
info = self._cache.ongoing_prefetch[request]
|
|
raw_len = remaining_full_tokens + int(info.prefetch_key.is_bigram)
|
|
full_key_tokens.extend(info.prefetch_key.token_ids[:raw_len])
|
|
cache = self._cache
|
|
full_key = RadixKey(
|
|
full_key_tokens,
|
|
extra_key=extra_key,
|
|
is_bigram=cache.tree_core.is_eagle,
|
|
cache_salt=cache_salt,
|
|
)
|
|
matched_full, anchor_node, anchor_tokens = (
|
|
cache.tree_core.match_full_device_prefix(full_key)
|
|
)
|
|
if matched_full < matched_len:
|
|
return "anchor_lost", matched_full
|
|
if anchor_tokens == 0:
|
|
return "no_anchor", matched_full
|
|
if self.anchor_locked_tokens_ + anchor_tokens > self.anchor_lock_cap_tokens:
|
|
# Parking for a pin no drain can ever satisfy deadlocks the
|
|
# request, so only a pin that still fits the cap is worth a wait.
|
|
over_cap = anchor_tokens > self.anchor_lock_cap_tokens
|
|
self._anchor_lock_cap_skips += 1
|
|
if (
|
|
self._anchor_lock_cap_skips <= 3
|
|
or self._anchor_lock_cap_skips % 1000 == 0
|
|
):
|
|
logger.warning(
|
|
"HiCache anchor-lock cap reached (skip %d): locked=%d "
|
|
"want=%d cap=%d; %s.",
|
|
self._anchor_lock_cap_skips,
|
|
self.anchor_locked_tokens_,
|
|
anchor_tokens,
|
|
self.anchor_lock_cap_tokens,
|
|
"launching unlocked" if over_cap else "parking",
|
|
)
|
|
return ("cap_skip" if over_cap else "cap_busy"), matched_full
|
|
cache.tree_core.inc_full_pin(anchor_node)
|
|
self.anchor_locks[request] = _AnchorLock(
|
|
node_id=anchor_node,
|
|
tokens=anchor_tokens,
|
|
)
|
|
self.anchor_locked_tokens_ += anchor_tokens
|
|
return "locked", matched_full
|
|
|
|
def release_anchor_lock(self, request: CacheRequestHandle) -> None:
|
|
"""Drop a staged prefetch's anchor lock (idempotent; called at every
|
|
consume/drop/abort exit)."""
|
|
lock = self.anchor_locks.pop(request, None)
|
|
if lock is None:
|
|
return
|
|
self._cache.tree_core.dec_full_pin(lock.node_id)
|
|
self.anchor_locked_tokens_ -= lock.tokens
|
|
assert self.anchor_locked_tokens_ >= 0, (
|
|
f"anchor-lock accounting corrupted: locked={self.anchor_locked_tokens_} "
|
|
f"after releasing {request.rid}"
|
|
)
|
|
|
|
def staged_span_covered(
|
|
self, request: CacheRequestHandle, span_tokens: int
|
|
) -> bool:
|
|
"""True when the live device tree already covers the fetch's whole
|
|
would-be span (prefix + the storage-hit tokens): nothing would be
|
|
left to splice at consumption, so the IO-commit caller cancels
|
|
before the bounce alloc and the storage read."""
|
|
info = self._cache.ongoing_prefetch.get(request)
|
|
if info is None or span_tokens <= 0:
|
|
return False
|
|
prefix_tokens, _, _ = self._prefetch_prefix_ctx[request]
|
|
span_key = info.prefetch_key
|
|
full_tokens = array("q", prefix_tokens)
|
|
full_tokens.extend(span_key[:span_tokens].token_ids)
|
|
key = RadixKey(
|
|
full_tokens,
|
|
extra_key=span_key.extra_key,
|
|
is_bigram=self._cache.tree_core.is_eagle,
|
|
cache_salt=span_key.cache_salt,
|
|
)
|
|
match = self._cache.match_prefix(MatchPrefixParams(key=key))
|
|
return match.device_prefix_len >= len(key)
|
|
|
|
def set_prefix_ctx(
|
|
self,
|
|
request: CacheRequestHandle,
|
|
matched_prefix_tokens,
|
|
extra_key: Optional[str] = None,
|
|
cache_salt: Optional[str] = None,
|
|
) -> None:
|
|
"""Record the device-matched prefix (and its tree-key namespace) at
|
|
prefetch enqueue; consumed at staging commit to build the full-span
|
|
tree key, and by try_lock_anchor to re-match a stale anchor."""
|
|
self._prefetch_prefix_ctx[request] = (
|
|
list(matched_prefix_tokens or []),
|
|
extra_key,
|
|
cache_salt,
|
|
)
|
|
|
|
def pop_prefix_ctx(self, request: CacheRequestHandle) -> None:
|
|
self._prefetch_prefix_ctx.pop(request, None)
|
|
|
|
def prefetch_prefix_key(self, request: CacheRequestHandle) -> RadixKey:
|
|
prefix_tokens, extra_key, cache_salt = self._prefetch_prefix_ctx[request]
|
|
return RadixKey(
|
|
array("q", prefix_tokens),
|
|
extra_key=extra_key,
|
|
is_bigram=self._cache.tree_core.is_eagle,
|
|
cache_salt=cache_salt,
|
|
)
|
|
|
|
def has_staged(self, request: CacheRequestHandle) -> bool:
|
|
return request in self.staged_prefetches
|
|
|
|
def prepare_staged_prefetch(self, req: Req) -> bool:
|
|
"""Rebuild the admission plan from this pass's joint component match."""
|
|
req.staged_prefetch_plan = None
|
|
f = self.staged_prefetches.get(req.cache_request_handle)
|
|
if f is None:
|
|
if not (req.host_hit_is_storage and req.host_loaded_length > 0):
|
|
self._clear_storage_hit(req)
|
|
return True
|
|
joint_len = req.prefix_len
|
|
if joint_len >= f.matched_len + f.num_tokens:
|
|
# The joint match already covers the staged span; a shorter FULL-only
|
|
# prefix would strand the slots recomputed below cache_protected_len.
|
|
self._resolve_device_covered(req)
|
|
return True
|
|
key = RadixKey(
|
|
f.key_tokens,
|
|
extra_key=f.extra_key,
|
|
is_bigram=self._cache.tree_core.is_eagle,
|
|
cache_salt=f.cache_salt,
|
|
)
|
|
matched_len, node_id, _ = self._cache.tree_core.match_full_device_prefix(key)
|
|
if matched_len < f.matched_len:
|
|
logger.warning(
|
|
"HiCache staged prefetch deferred req=%s reason=shrunk "
|
|
"matched=%d now=%d tokens=%d",
|
|
req.rid,
|
|
f.matched_len,
|
|
matched_len,
|
|
f.num_tokens,
|
|
)
|
|
self._refetch_staged(f)
|
|
return False
|
|
self._set_request_device_prefix(req, key, matched_len, node_id)
|
|
plan = self._build_staged_prefetch_plan(f, key, matched_len)
|
|
if plan is None:
|
|
self._resolve_device_covered(req)
|
|
return True
|
|
req.host_hit_length = plan.full_tokens
|
|
req.swa_host_hit_length = plan.window_tokens(ComponentType.SWA)
|
|
req.mamba_host_hit_length = plan.mamba_slots
|
|
# The device alone serves only this pass's joint match; the rest of the
|
|
# span, fetched FULL or resident FULL the aux tail unlocks, is storage's.
|
|
req.storage_hit_length = f.matched_len + f.num_tokens - joint_len
|
|
req.storage_hit_start = joint_len
|
|
req.host_hit_is_storage = True
|
|
req.staged_prefetch_plan = plan
|
|
return True
|
|
|
|
def _set_request_device_prefix(
|
|
self, req: Req, key: RadixKey, matched_len: int, node_id: NodeId
|
|
) -> None:
|
|
req.prefix_len = matched_len
|
|
req.last_node = node_id
|
|
req.kv.cache_protected_len = matched_len
|
|
|
|
def _build_staged_prefetch_plan(
|
|
self, f: _StagedPrefetch, key: RadixKey, matched_len: int
|
|
) -> Optional[StagedPrefetchPlan]:
|
|
full_tokens = max(0, f.matched_len + f.num_tokens - matched_len)
|
|
span_end = f.matched_len + f.num_tokens
|
|
windows = {}
|
|
for component in self._swa_components():
|
|
staged_tokens = sum(
|
|
len(t.host_indices)
|
|
for t in f.aux_xfers
|
|
if t.name == component.pool_name and t.host_indices is not None
|
|
)
|
|
window_start = span_end - staged_tokens
|
|
repair_end = min(matched_len, span_end)
|
|
missing_ranges = (
|
|
self._cache.tree_core.swa_tombstone_ranges(
|
|
key,
|
|
window_start,
|
|
repair_end,
|
|
component_type=component.component_type,
|
|
)
|
|
if window_start < repair_end
|
|
else []
|
|
)
|
|
tail_start = max(matched_len, window_start)
|
|
if tail_start < span_end:
|
|
missing_ranges.append((tail_start, span_end))
|
|
windows[component.component_type] = WindowLoadBackPlan(
|
|
window_start, span_end, tuple(missing_ranges)
|
|
)
|
|
mamba_slots = sum(
|
|
len(t.host_indices)
|
|
for t in f.aux_xfers
|
|
if t.name == PoolName.MAMBA and t.host_indices is not None
|
|
)
|
|
if (
|
|
full_tokens == 0
|
|
and mamba_slots == 0
|
|
and not any(w.num_tokens for w in windows.values())
|
|
):
|
|
# The device already covers every pool's remaining demand.
|
|
return None
|
|
return StagedPrefetchPlan(
|
|
f.operation_id, key, matched_len, full_tokens, windows, mamba_slots
|
|
)
|
|
|
|
def _resolve_device_covered(self, req: Req) -> None:
|
|
req.host_hit_length = 0
|
|
req.swa_host_hit_length = 0
|
|
req.mamba_host_hit_length = 0
|
|
self._clear_storage_hit(req)
|
|
self.release_staged_hold(req.cache_request_handle, reason="device_covered")
|
|
|
|
@staticmethod
|
|
def _clear_storage_hit(req: Req) -> None:
|
|
req.storage_hit_length = 0
|
|
req.storage_hit_start = None
|
|
req.host_hit_is_storage = False
|
|
|
|
def _refetch_staged(self, f: _StagedPrefetch) -> None:
|
|
self.release_staged_hold(f.request, reason="shrunk")
|
|
self._cache.storage_prefetch_retries.refetch(
|
|
f.request.rid, f.matched_len + f.num_tokens
|
|
)
|
|
|
|
def stage_completed_prefetch(
|
|
self,
|
|
request: CacheRequestHandle,
|
|
num_tokens: int,
|
|
hash_value: list[str],
|
|
) -> bool:
|
|
"""Park the completed fetch as a held bounce; the scheduler surfaces
|
|
it as host_hit_length and the adder consumes it via init_load_back.
|
|
Always returns True (ready is a stable, revisited state)."""
|
|
cache = self._cache
|
|
info = cache.ongoing_prefetch.pop(request)
|
|
host_indices = info.host_indices
|
|
operation = info.operation
|
|
comp_xfers = info.comp_xfers
|
|
cc = cache.cache_controller
|
|
prefix_ctx = self._prefetch_prefix_ctx.pop(request, None)
|
|
prefix_tokens = prefix_ctx[0] if prefix_ctx is not None else None
|
|
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
|
|
# Component transfers are already present in comp_xfers. Preserve the
|
|
# derived sidecars from the storage operation as well; cc.load resolves
|
|
# them against the freshly allocated source host/device indices.
|
|
aux_xfers.extend(
|
|
transfer
|
|
for transfer in operation.pool_transfers or ()
|
|
if transfer.indices_from_pool is not None
|
|
)
|
|
occupied_tokens = operation.buffer_host_occupied_units
|
|
if occupied_tokens is None:
|
|
occupied_tokens = self.host_allocation_units(host_indices, aux_xfers)
|
|
|
|
staged = self._build_staged_prefetch(
|
|
request,
|
|
info,
|
|
prefix_tokens,
|
|
num_tokens,
|
|
hash_value,
|
|
aux_xfers,
|
|
occupied_tokens,
|
|
)
|
|
if staged is None:
|
|
# Nothing usable fetched: recompute.
|
|
cache.discard_storage_prefetch_accounting(request)
|
|
self.release_anchor_lock(request)
|
|
cc.append_host_mem_release(
|
|
host_indices[:num_tokens], extra_pools=aux_xfers or None
|
|
)
|
|
cc.prefetch_tokens_occupied -= occupied_tokens
|
|
cache.prefetch_loaded_tokens_by_reqid[request] = 0
|
|
cache.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
|
return True
|
|
|
|
# The fetch is evidence for every pool it delivered (sound even if the
|
|
# staged prefetch is later dropped): KV-derived sidecars came with the
|
|
# staged pages, other pools with the keys their transfer carries.
|
|
beliefs = cache.storage_existence_cache
|
|
beliefs.pool(PoolName.KV).add(staged.hash_values)
|
|
for transfer in aux_xfers:
|
|
if transfer.indices_from_pool == PoolName.KV:
|
|
beliefs.pool(transfer.name).add(staged.hash_values)
|
|
elif transfer.keys:
|
|
beliefs.pool(transfer.name).add(transfer.keys)
|
|
self.staged_prefetches[request] = staged
|
|
cache.prefetch_loaded_tokens_by_reqid[request] = staged.num_tokens
|
|
cache.prefetch_loaded_storage_start_by_reqid[request] = operation.storage_start
|
|
return True
|
|
|
|
def _build_staged_prefetch(
|
|
self,
|
|
request: CacheRequestHandle,
|
|
info: _OngoingPrefetch,
|
|
prefix_tokens: Optional[list[int]],
|
|
num_tokens: int,
|
|
hash_values: list[str],
|
|
aux_xfers: list[PoolTransfer],
|
|
occupied_tokens: int,
|
|
) -> Optional[_StagedPrefetch]:
|
|
has_aux = any(
|
|
t.host_indices is not None and t.host_indices.numel() > 0 for t in aux_xfers
|
|
)
|
|
if (num_tokens == 0 and not has_aux) or prefix_tokens is None:
|
|
return None
|
|
key = info.prefetch_key
|
|
return _StagedPrefetch(
|
|
request=request,
|
|
key_tokens=array(
|
|
"q",
|
|
prefix_tokens + list(key.token_ids[: num_tokens + int(key.is_bigram)]),
|
|
),
|
|
extra_key=key.extra_key,
|
|
cache_salt=key.cache_salt,
|
|
matched_len=len(prefix_tokens),
|
|
num_tokens=num_tokens,
|
|
occupied_tokens=occupied_tokens,
|
|
host_indices=info.host_indices[:num_tokens],
|
|
aux_xfers=aux_xfers,
|
|
hash_values=hash_values[: num_tokens // self._cache.page_size],
|
|
operation_id=info.operation.id,
|
|
)
|
|
|
|
def defer_staged_admission(self, req: Req, pool: str) -> None:
|
|
"""Retain staging for a bounded number of capacity retries."""
|
|
request = req.cache_request_handle
|
|
f = self.staged_prefetches.get(request)
|
|
if f is None:
|
|
return
|
|
cache = self._cache
|
|
defers = self._staged_admission_defers.get(request, 0) + 1
|
|
self._staged_admission_defers[request] = defers
|
|
cache._log_storage_prefetch_deferred(f.num_tokens, "device_capacity")
|
|
if defers < self.max_staged_admission_defers:
|
|
logger.warning(
|
|
"HiCache staged prefetch deferred at admission req=%s "
|
|
"reason=device_capacity pool=%s tokens=%d defers=%d",
|
|
req.rid,
|
|
pool,
|
|
f.num_tokens,
|
|
defers,
|
|
)
|
|
return
|
|
# Still unmaterializable: drop the hold so the admission loop stops
|
|
# breaking on this request, which recomputes on its next pass.
|
|
logger.warning(
|
|
"HiCache staged prefetch dropped after %d device_capacity "
|
|
"deferrals req=%s pool=%s tokens=%d",
|
|
defers,
|
|
req.rid,
|
|
pool,
|
|
f.num_tokens,
|
|
)
|
|
self.release_staged_hold(request, reason="device_capacity")
|
|
req.staged_prefetch_plan = None
|
|
|
|
def init_load_back(
|
|
self, params: InitLoadBackParams
|
|
) -> Optional[tuple[int, NodeId]]:
|
|
"""Materialize a selected prefill under the caller's prefix lock.
|
|
|
|
The caller has finished selecting its prefill shape and must acquire
|
|
the request lock after success, without further admission gates. The
|
|
prefix lock protects allocation-time eviction. None retains staging
|
|
and its anchor for the next admission attempt.
|
|
|
|
cc.load queues H2D before insert. Insert must preserve all FULL
|
|
destinations queued for H2D; freeing them during dedup would let the
|
|
pending copy overwrite reused slots. Free redundant auxiliary
|
|
destinations only after the transfer ack."""
|
|
cache = self._cache
|
|
req = params.req
|
|
assert req is not None
|
|
request = req.cache_request_handle
|
|
unchanged = (0, req.last_node)
|
|
f = self.staged_prefetches.get(request)
|
|
if f is None:
|
|
self.release_anchor_lock(request)
|
|
return unchanged
|
|
plan = req.staged_prefetch_plan
|
|
assert plan is not None, f"staged prefetch was not planned for {req.rid}"
|
|
assert f.operation_id == plan.operation_id
|
|
assert (f.extra_key, f.cache_salt) == (req.extra_key, req.cache_salt)
|
|
assert (
|
|
req.host_hit_length,
|
|
req.swa_host_hit_length,
|
|
req.mamba_host_hit_length,
|
|
) == (
|
|
plan.full_tokens,
|
|
plan.window_tokens(ComponentType.SWA),
|
|
plan.mamba_slots,
|
|
), f"staged load-back budget changed for {req.rid}"
|
|
|
|
splice_base = plan.device_prefix_len
|
|
assert req.prefix_len == splice_base
|
|
prefix_indices = self._cache.prefix_device_indices(req)
|
|
trim_tokens = splice_base - f.matched_len
|
|
assert trim_tokens % cache.page_size == 0, (
|
|
f"staged splice trim not page-aligned req={req.rid}: "
|
|
f"matched={f.matched_len} splice_base={splice_base}"
|
|
)
|
|
|
|
key = plan.key
|
|
span_end = f.matched_len + f.num_tokens
|
|
load_back_id = -(f.operation_id) - 1
|
|
# Preserve the complete bounce in f until ack; H2D uses only the rows
|
|
# admitted by the plan, including repairs inside the resident FULL prefix.
|
|
load_xfers = list(f.aux_xfers)
|
|
for component in self._swa_components():
|
|
pool_name = component.pool_name
|
|
window = plan.windows[component.component_type]
|
|
if not window.missing_ranges:
|
|
load_xfers = [
|
|
t
|
|
for t in load_xfers
|
|
if t.name != pool_name and t.indices_from_pool != pool_name
|
|
]
|
|
continue
|
|
host_parts = [
|
|
slice(start - window.start, end - window.start)
|
|
for start, end in window.missing_ranges
|
|
]
|
|
for i, transfer in enumerate(load_xfers):
|
|
if transfer.name != pool_name:
|
|
continue
|
|
load_xfers[i] = replace(
|
|
transfer,
|
|
host_indices=torch.cat(
|
|
[transfer.host_indices[part] for part in host_parts]
|
|
),
|
|
)
|
|
|
|
state, failure_pool = self._allocate_and_queue_load_back(
|
|
req, f, plan, load_back_id, load_xfers
|
|
)
|
|
if state is None:
|
|
# load() allocates all pools atomically before queueing H2D, so the
|
|
# staged host buffers remain reusable after either pool is short.
|
|
return self.defer_staged_admission(req, failure_pool)
|
|
device_indices = state.device_indices
|
|
del self.staged_prefetches[request]
|
|
self._staged_admission_defers.pop(request, None)
|
|
req.staged_prefetch_plan = None
|
|
cache._settle_storage_prefetch_hit(
|
|
request, credited_tokens=req.storage_hit_length
|
|
)
|
|
|
|
full_indices = torch.cat([prefix_indices, device_indices])
|
|
for component in self._swa_components():
|
|
pool_name = component.pool_name
|
|
window_dev = next(
|
|
(
|
|
t.device_indices
|
|
for t in state.load_transfers
|
|
if t.name == pool_name
|
|
and t.device_indices is not None
|
|
and t.device_indices.numel() > 0
|
|
),
|
|
None,
|
|
)
|
|
if window_dev is None:
|
|
continue
|
|
ct = component.component_type
|
|
window = plan.windows[ct]
|
|
assert len(window_dev) == window.num_tokens
|
|
full_parts = []
|
|
offset = 0
|
|
for start, end in window.missing_ranges:
|
|
full_part = full_indices[start:end]
|
|
full_parts.append(full_part)
|
|
if end <= splice_base:
|
|
for action in cache.tree_core.attach_swa_window(
|
|
key,
|
|
start,
|
|
end,
|
|
window_dev[offset : offset + end - start],
|
|
component_type=ct,
|
|
):
|
|
cache._apply_cache_action(action)
|
|
offset += end - start
|
|
component.restore_buffer_mapping(torch.cat(full_parts), window_dev)
|
|
|
|
insert_result = self._insert_loaded_prefix(
|
|
state, req, plan, prefix_indices=prefix_indices
|
|
)
|
|
self.ongoing_buffer_load_back[load_back_id] = state
|
|
match = cache.match_prefix(MatchPrefixParams(key=key))
|
|
canonical = cache.path_device_indices(match.last_device_node)[
|
|
splice_base:span_end
|
|
]
|
|
self.release_anchor_lock(request)
|
|
owned = match.device_prefix_len >= span_end and torch.equal(
|
|
canonical, device_indices
|
|
)
|
|
if not owned:
|
|
# Fail-stop: the insert freed or replaced slots the in-flight H2D
|
|
# still targets; continuing risks silent KV corruption.
|
|
raise RuntimeError(
|
|
"HiCache buffer load-back ownership violation "
|
|
f"req={f.request.rid}: "
|
|
f"insert prefix_len={insert_result.prefix_len} "
|
|
f"expected={splice_base}, matched={match.device_prefix_len} "
|
|
f"span_end={span_end} splice_base={splice_base}; "
|
|
f"in-flight H2D targets freed slots"
|
|
)
|
|
return len(canonical), match.last_device_node
|
|
|
|
def _ensure_full_capacity(self, load_tokens: int) -> bool:
|
|
# Evict-before-alloc (mirrors _load_back_transfers): the budget gate
|
|
# counts evictable pages, but cc.load draws from free slots only.
|
|
cache = self._cache
|
|
if cache.supports_swa():
|
|
avail = cache.token_to_kv_pool_allocator.full_available_size()
|
|
else:
|
|
avail = cache.token_to_kv_pool_allocator.available_size()
|
|
if avail < load_tokens:
|
|
needed = load_tokens - avail
|
|
cache.evict_for_alloc(EvictParams(num_tokens=needed))
|
|
if cache.supports_swa():
|
|
avail = cache.token_to_kv_pool_allocator.full_available_size()
|
|
else:
|
|
avail = cache.token_to_kv_pool_allocator.available_size()
|
|
if avail < load_tokens:
|
|
return False
|
|
return True
|
|
|
|
def _allocate_and_queue_load_back(
|
|
self,
|
|
req: Req,
|
|
f: _StagedPrefetch,
|
|
plan: StagedPrefetchPlan,
|
|
load_back_id: int,
|
|
load_xfers: list[PoolTransfer],
|
|
) -> tuple[Optional[_OngoingBufferLoadBack], str]:
|
|
if not self._ensure_full_capacity(plan.full_tokens):
|
|
return None, "full"
|
|
cache = self._cache
|
|
load_contexts: list[BufferLoadBackContext] = []
|
|
for component in cache.components.values():
|
|
load_context = component.prepare_buffer_load_back(req, f.aux_xfers)
|
|
if load_context is None:
|
|
for prepared in load_contexts:
|
|
prepared.finalize_allocation(success=False)
|
|
return None, component.component_type.name.lower()
|
|
load_contexts.append(load_context)
|
|
# Not staging, so these stay out of the aux_xfers the ack frees.
|
|
load_xfers.extend(load_context.load_xfers)
|
|
trim_tokens = plan.device_prefix_len - f.matched_len
|
|
device_indices = cache.cache_controller.load(
|
|
host_indices=f.host_indices[trim_tokens:],
|
|
node_id=load_back_id,
|
|
extra_pools=load_xfers or None,
|
|
)
|
|
for load_context in load_contexts:
|
|
load_context.finalize_allocation(success=device_indices is not None)
|
|
if device_indices is None:
|
|
return None, "full_or_aux"
|
|
return (
|
|
_OngoingBufferLoadBack(
|
|
request=f.request,
|
|
num_tokens=plan.full_tokens,
|
|
occupied_tokens=f.occupied_tokens,
|
|
aux_xfers=f.aux_xfers,
|
|
# Keep the full staged bounce until ack, trimmed head included.
|
|
host_indices=f.host_indices,
|
|
hash_values=f.hash_values,
|
|
aux_device_releases=[],
|
|
device_indices=device_indices,
|
|
load_transfers=load_xfers,
|
|
load_contexts=load_contexts,
|
|
),
|
|
"",
|
|
)
|
|
|
|
def _insert_loaded_prefix(
|
|
self,
|
|
state: _OngoingBufferLoadBack,
|
|
req: Req,
|
|
plan: StagedPrefetchPlan,
|
|
*,
|
|
prefix_indices: torch.Tensor,
|
|
) -> InsertResult:
|
|
# Publish via a plain insert under the admission lock choreography;
|
|
# the caller's request lock then pins the span (load_back pattern).
|
|
# prev_prefix_len covers the already-device-resident head.
|
|
cache = self._cache
|
|
insert_fields = {}
|
|
for load_context in state.load_contexts:
|
|
insert_fields.update(load_context.get_insert_fields())
|
|
insert_result = cache.insert(
|
|
InsertParams(
|
|
key=plan.key,
|
|
value=torch.cat([prefix_indices, state.device_indices]),
|
|
prev_prefix_len=plan.device_prefix_len,
|
|
component_evicted_seqlens={
|
|
ct: window.start for ct, window in plan.windows.items()
|
|
},
|
|
**insert_fields,
|
|
)
|
|
)
|
|
for load_context in state.load_contexts:
|
|
state.aux_device_releases.extend(
|
|
load_context.get_redundant_device_slots(insert_result)
|
|
)
|
|
return insert_result
|
|
|
|
def try_finish_load_back(self, ack_id: int) -> bool:
|
|
"""Fill ack: free the host bounce and return True when the ack id is
|
|
a buffer-mode load-back. The span was published at admission; the
|
|
ack never touches the tree (existence beliefs were fed from the
|
|
storage-fetched pages at staging commit)."""
|
|
f = self.ongoing_buffer_load_back.pop(ack_id, None)
|
|
if f is None:
|
|
return False
|
|
cache = self._cache
|
|
cc = cache.cache_controller
|
|
|
|
self._free_completed_load_back_buffers(f)
|
|
|
|
cc.prefetch_tokens_occupied -= f.occupied_tokens
|
|
logger.info(
|
|
"HiCache prefetch fill committed req=%s filled=%d occupied=%d locked=%d",
|
|
f.request.rid,
|
|
f.num_tokens,
|
|
cc.prefetch_tokens_occupied,
|
|
self.anchor_locked_tokens_,
|
|
)
|
|
cache._finish_storage_prefetch(
|
|
f.request, fulfilled_tokens=f.num_tokens, reason=None
|
|
)
|
|
return True
|
|
|
|
def _free_completed_load_back_buffers(self, f: _OngoingBufferLoadBack) -> None:
|
|
cc = self._cache.cache_controller
|
|
# The H2D consumed the bounce buffers; free them outright.
|
|
self._free_staging_now(f.host_indices, f.aux_xfers)
|
|
for pool_name, device_indices in f.aux_device_releases:
|
|
entry = cc.mem_pool_host.entry_map[pool_name]
|
|
free_fn = entry.device_free_fn or entry.device_pool.free
|
|
free_fn(device_indices)
|
|
|
|
def release_staged_hold(
|
|
self, request: CacheRequestHandle, reason: Optional[str] = None
|
|
) -> bool:
|
|
"""Free a staged hold outright — anchor pin, host bounce (KV + aux),
|
|
occupancy grant; nothing device-side exists yet. Called for aborts
|
|
and for holds that can no longer splice. Returns True when a hold
|
|
existed."""
|
|
self.release_anchor_lock(request)
|
|
self._staged_admission_defers.pop(request, None)
|
|
staged = self.staged_prefetches.pop(request, None)
|
|
if staged is None:
|
|
return False
|
|
self._cache._finish_storage_prefetch(request, fulfilled_tokens=0, reason=reason)
|
|
self._free_staging_now(staged.host_indices, staged.aux_xfers)
|
|
self._cache.cache_controller.prefetch_tokens_occupied -= staged.occupied_tokens
|
|
return True
|