[HiCache] Manage buffer-mode backups per storage pool (#42633)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
xiezhq-hermann
parent
fd62ee84f7
commit
08aad0c196
@@ -1295,15 +1295,9 @@ class HiCacheController:
|
||||
)
|
||||
hash_value, storage_hit_count = [], 0
|
||||
|
||||
storage_hit_count_tensor = torch.tensor(
|
||||
storage_hit_count, dtype=torch.int
|
||||
storage_hit_count = self._sync_prefetch_hit_query(
|
||||
operation, storage_hit_count
|
||||
)
|
||||
self._all_reduce(
|
||||
storage_hit_count_tensor,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_hits_sync_groups,
|
||||
)
|
||||
storage_hit_count = storage_hit_count_tensor.item()
|
||||
|
||||
# Record the TP-synced hit count; the scheduler thread decides
|
||||
# at drain time whether to revoke (below threshold) or allocate.
|
||||
@@ -1316,6 +1310,17 @@ class HiCacheController:
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int:
|
||||
"""Rank-reduce a hit query (MIN) so every rank agrees on the usable
|
||||
prefix. Runs for every operation, hit or miss."""
|
||||
hit_count = torch.tensor(storage_hit_count, dtype=torch.int)
|
||||
self._all_reduce(
|
||||
hit_count,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_hits_sync_groups,
|
||||
)
|
||||
return int(hit_count.item())
|
||||
|
||||
def write_storage(
|
||||
self,
|
||||
host_indices: torch.Tensor,
|
||||
|
||||
@@ -81,23 +81,28 @@ def _minimum_full_transfer_tokens(
|
||||
|
||||
|
||||
class _UnifiedBackupIntent(msgspec.Struct):
|
||||
"""Buffer-mode backup intent, unpinned while queued.
|
||||
"""One pool of one node queued for a buffer-mode write (unpinned while
|
||||
queued); a derived sidecar rides its source pool's intent.
|
||||
|
||||
Snapshots node identity at enqueue time: a split rewrites the node's
|
||||
key/hash in place while these copies stay intact, so a key-length change
|
||||
detects a split and a missing FULL device value detects eviction
|
||||
(``_validate_backup_intent``).
|
||||
(``_validate_backup_intent``). ``keys`` is the admission view (it sizes
|
||||
drop metrics); the D2H launch re-reads rows and keys off the node.
|
||||
"""
|
||||
|
||||
snapshot: BufferBackupSnapshot
|
||||
pool: PoolName
|
||||
keys: list[str]
|
||||
|
||||
|
||||
class _UnifiedBufferBackupEntry(msgspec.Struct):
|
||||
"""A buffer-mode backup after its D2H launch: intent + staging slots.
|
||||
|
||||
``host_indices`` are FULL-pool staging slots; ``aux_xfers`` carry the
|
||||
staged aux-pool slots (e.g. the SWA window). All are freed at the
|
||||
storage-write ack — host memory is never retained as a cache tier.
|
||||
``host_indices`` are the KV staging slots (empty for another pool's
|
||||
intent); ``aux_xfers`` the staged transfers and riding sidecars; all are
|
||||
freed at the storage-write ack (host memory is never a cache tier).
|
||||
``keys_by_pool`` are the keys the write stores, learned at the ack.
|
||||
"""
|
||||
|
||||
intent: _UnifiedBackupIntent
|
||||
@@ -105,6 +110,7 @@ class _UnifiedBufferBackupEntry(msgspec.Struct):
|
||||
aux_xfers: list[PoolTransfer]
|
||||
lock_params: DecLockRefParams
|
||||
occupied_units: int
|
||||
keys_by_pool: dict[PoolName, list[str]]
|
||||
|
||||
|
||||
class _StagedPrefetch(msgspec.Struct):
|
||||
@@ -147,24 +153,6 @@ class _AnchorLock(msgspec.Struct):
|
||||
tokens: int
|
||||
|
||||
|
||||
def _track_content_refs(refs: dict[str, int], hash_values: list[str]) -> None:
|
||||
"""Add one content ref per page hash (at D2H launch). Refcounted,
|
||||
not a flag: several launched entries can carry the same content
|
||||
(duplicate staging of republished spans)."""
|
||||
for h in hash_values:
|
||||
refs[h] = refs.get(h, 0) + 1
|
||||
|
||||
|
||||
def _untrack_content_refs(refs: dict[str, int], hash_values: list[str]) -> None:
|
||||
"""Drop one content ref per page hash (at storage-ack)."""
|
||||
for h in hash_values:
|
||||
n = refs.get(h, 0) - 1
|
||||
if n <= 0:
|
||||
refs.pop(h, None)
|
||||
else:
|
||||
refs[h] = n
|
||||
|
||||
|
||||
def validate_buffer_only_stack(
|
||||
sidecar_pool_specs: list[SidecarPoolSpec],
|
||||
host_pool_group: HostPoolGroup,
|
||||
@@ -270,8 +258,8 @@ class BufferModePipeline:
|
||||
# a host pool (0 = KV-only: no trailing window staged). Static after
|
||||
# pool assembly.
|
||||
self._swa_window_pages = swa_window_pages
|
||||
# Metadata-only pending-write backlog cap; beyond it new intents
|
||||
# are dropped at admission (re-trigger on a later hit).
|
||||
# Metadata-only pending-write backlog cap, measured in unique FULL
|
||||
# snapshot spans; beyond it new spans are dropped at admission.
|
||||
self.write_backlog_cap = write_backlog_cap
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
|
||||
@@ -318,19 +306,18 @@ class BufferModePipeline:
|
||||
] = {}
|
||||
self.staged_prefetches: dict[CacheRequestHandle, _StagedPrefetch] = {}
|
||||
self.ongoing_buffer_load_back: dict[int, _OngoingBufferLoadBack] = {}
|
||||
# Backup pipeline: FIFO intents awaiting a D2H slot, node ids
|
||||
# anywhere in flight (dedupes re-triggers), and a content refcount
|
||||
# of every page hash between D2H launch and storage-ack — admission
|
||||
# skips content covered by beliefs + launched writes.
|
||||
# FIFO intents (one per pool of a node) awaiting a D2H slot, and the
|
||||
# pools in flight per node (admission to storage ack).
|
||||
self.pending_write_queue: deque[_UnifiedBackupIntent] = deque()
|
||||
self.inflight_backup_node_ids: set[int] = set()
|
||||
self.inflight_backup_hashes: dict[str, int] = {}
|
||||
# Backups between D2H launch and D2H ack (keyed by node id), then
|
||||
# between storage-write launch and storage ack (keyed by operation
|
||||
# id). Mirrors the cache-mode ongoing_write_through/ongoing_backup
|
||||
# stages, with buffer entries.
|
||||
self._queued_span_refs: dict[tuple[NodeId, int], int] = {}
|
||||
self.inflight_backup_pools: dict[NodeId, set[PoolName]] = {}
|
||||
# Nodes admitted during the current insert walk (None outside one).
|
||||
self._walk_admitted: Optional[set[NodeId]] = None
|
||||
# Launched writes: D2H in flight (by ack id), then the storage write
|
||||
# in flight (by operation id).
|
||||
self.ongoing_write_through: dict[int, _UnifiedBufferBackupEntry] = {}
|
||||
self.ongoing_backup: dict[int, _UnifiedBufferBackupEntry] = {}
|
||||
self._backup_ack_seq = 0
|
||||
self.write_staged_tokens_ = 0
|
||||
self.write_backlog_tokens_ = 0
|
||||
self._backlog_cap_hits = 0
|
||||
@@ -391,7 +378,9 @@ class BufferModePipeline:
|
||||
cc = self._cache.cache_controller
|
||||
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
||||
if self._shared_host_domain() is None:
|
||||
return kv_tokens
|
||||
# Aux transfers staged through the anchor's own host pool (an
|
||||
# entry aliasing it) occupy anchor tokens as well.
|
||||
return kv_tokens + self._anchor_pool_aux_tokens(aux_xfers or [])
|
||||
num_bytes = sum(
|
||||
tokens * pool.size_per_token
|
||||
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
|
||||
@@ -420,6 +409,18 @@ class BufferModePipeline:
|
||||
)
|
||||
return requests
|
||||
|
||||
@property
|
||||
def inflight_backup_node_ids(self):
|
||||
"""Nodes with a write of any pool queued, staged or awaiting its
|
||||
storage ack."""
|
||||
return self.inflight_backup_pools.keys()
|
||||
|
||||
def backup_pending(self, node_id: NodeId) -> bool:
|
||||
"""A write of ``node_id`` is queued, staged or awaiting its storage
|
||||
ack. A component that drops rows a pending write still reads asks
|
||||
here first; the pipeline pins the node only from D2H launch to ack."""
|
||||
return node_id in self.inflight_backup_pools
|
||||
|
||||
def is_idle(self) -> bool:
|
||||
"""No queued, staged, or in-flight buffer-mode work or anchor pins."""
|
||||
return not (
|
||||
@@ -444,19 +445,37 @@ class BufferModePipeline:
|
||||
|
||||
# ---- backup pipeline (device -> staging -> storage) ----
|
||||
|
||||
def _backup_parent_covered(self, state: BufferBackupState) -> bool:
|
||||
def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int:
|
||||
"""Staging tokens aux transfers hold in the anchor's host pool."""
|
||||
group = self._cache.cache_controller.mem_pool_host
|
||||
anchor_pool = group.anchor_entry.host_pool
|
||||
tokens = 0
|
||||
for t in aux_xfers:
|
||||
if t.indices_from_pool is not None:
|
||||
continue
|
||||
entry = group.entry_map.get(t.name)
|
||||
if entry is not None and entry.host_pool is anchor_pool:
|
||||
tokens += self._transfer_tokens(t)
|
||||
return tokens
|
||||
|
||||
def _backup_parent_covered(self, state: BufferBackupState, node_id: NodeId) -> bool:
|
||||
"""Only admit a node whose parent is stored/in-flight: writing above
|
||||
a dropped parent creates a permanent longest-prefix hole."""
|
||||
if (
|
||||
state.parent_is_root
|
||||
or state.parent_node_id in self.inflight_backup_node_ids
|
||||
a dropped parent creates a permanent longest-prefix hole. A component
|
||||
may vouch for the parent through a pool of its own."""
|
||||
if state.parent_is_root or PoolName.KV in self.inflight_backup_pools.get(
|
||||
state.parent_node_id, ()
|
||||
):
|
||||
return True
|
||||
return (
|
||||
if (
|
||||
state.parent_last_hash is not None
|
||||
and self._cache.storage_existence_cache.contains(
|
||||
PoolName.KV, state.parent_last_hash
|
||||
and self._cache.storage_existence_cache.pool(PoolName.KV).contains(
|
||||
state.parent_last_hash
|
||||
)
|
||||
):
|
||||
return True
|
||||
return any(
|
||||
component.buffer_backup_parent_covered(state.parent_node_id, node_id)
|
||||
for component in self._cache.components.values()
|
||||
)
|
||||
|
||||
def _log_backup_dropped(self, num_tokens: int) -> None:
|
||||
@@ -464,129 +483,229 @@ class BufferModePipeline:
|
||||
if cache.enable_storage_metrics and cache.storage_metrics_collector is not None:
|
||||
cache.storage_metrics_collector.log_backup_dropped_tokens(num_tokens)
|
||||
|
||||
def _pool_covered(self, pool: PoolName, keys: list[str]) -> bool:
|
||||
"""Beliefs plus content past a D2H launch (republished content must
|
||||
not re-write while the original write drains)."""
|
||||
return self._cache.storage_existence_cache.pool(pool).covered(keys)
|
||||
|
||||
def _transfer_keys(
|
||||
self, transfer: PoolTransfer, hash_values: list[str]
|
||||
) -> Optional[list[str]]:
|
||||
"""The builder's keys, else the chain's trailing pages, one per staged
|
||||
page (the SWA convention); None when the transfer stages nothing."""
|
||||
if transfer.keys is not None:
|
||||
return list(transfer.keys)
|
||||
if transfer.indices_from_pool is not None or transfer.device_indices is None:
|
||||
return None
|
||||
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
|
||||
if entry is None:
|
||||
return None
|
||||
num_pages = len(transfer.device_indices) // entry.host_pool.page_size
|
||||
if num_pages == 0 or num_pages > len(hash_values):
|
||||
return None
|
||||
return list(hash_values[-num_pages:])
|
||||
|
||||
def _backup_pool_keys(
|
||||
self, snapshot: BufferBackupSnapshot
|
||||
) -> dict[PoolName, list[str]]:
|
||||
"""Admission view of the keys a backup writes per pool: the KV chain,
|
||||
each aux component's BACKUP_HOST transfers, component overrides on
|
||||
top, every sidecar under its source's keys."""
|
||||
hash_values = snapshot.hash_values
|
||||
tree_core = self._cache.tree_core
|
||||
pool_keys: dict[PoolName, list[str]] = {PoolName.KV: list(hash_values)}
|
||||
overrides = tree_core.buffer_backup_pool_keys(snapshot.node_id, hash_values)
|
||||
for ct in self._cache.components:
|
||||
# A component that names its pools' keys spares the transfer build.
|
||||
if ct == ComponentType.FULL or ct in overrides:
|
||||
continue
|
||||
transfers = tree_core.build_hicache_transfers(
|
||||
ct, snapshot.node_id, CacheTransferPhase.BACKUP_HOST
|
||||
)
|
||||
for transfer in transfers or ():
|
||||
keys = self._transfer_keys(transfer, hash_values)
|
||||
if keys:
|
||||
pool_keys.setdefault(transfer.name, []).extend(keys)
|
||||
for named in overrides.values():
|
||||
for pool, keys in named.items():
|
||||
if keys:
|
||||
pool_keys[pool] = list(keys)
|
||||
else:
|
||||
pool_keys.pop(pool, None)
|
||||
for spec in self._cache.sidecar_pool_specs:
|
||||
source = pool_keys.get(spec.indices_from_pool)
|
||||
if source:
|
||||
pool_keys[spec.pool_name] = list(source)
|
||||
return pool_keys
|
||||
|
||||
@staticmethod
|
||||
def _pool_order(pool: PoolName) -> tuple[bool, str]:
|
||||
# KV first, then a fixed order: every rank must admit identically.
|
||||
return (pool != PoolName.KV, pool.value)
|
||||
|
||||
def _write_intents(
|
||||
self, snapshot: BufferBackupSnapshot
|
||||
) -> list[tuple[PoolName, list[str]]]:
|
||||
"""(pool, keys) of the pools not believed stored or in flight; a
|
||||
sidecar riding a covered pool drives that pool too."""
|
||||
pool_keys = self._backup_pool_keys(snapshot)
|
||||
sidecars = {
|
||||
spec.pool_name: spec.indices_from_pool
|
||||
for spec in self._cache.sidecar_pool_specs
|
||||
}
|
||||
intents: list[tuple[PoolName, list[str]]] = []
|
||||
for pool in sorted(pool_keys, key=self._pool_order):
|
||||
keys = pool_keys[pool]
|
||||
if pool in sidecars or not keys:
|
||||
continue
|
||||
riders = [
|
||||
name
|
||||
for name, source in sidecars.items()
|
||||
if source == pool and pool_keys.get(name)
|
||||
]
|
||||
if all(
|
||||
self._pool_covered(name, pool_keys[name]) for name in (pool, *riders)
|
||||
):
|
||||
continue
|
||||
intents.append((pool, keys))
|
||||
return intents
|
||||
|
||||
def _sizing(
|
||||
self, pool: PoolName, keys: list[str]
|
||||
) -> tuple[int, list[PoolTransfer]]:
|
||||
"""(KV tokens, keys-only aux transfers) an intent stages, for the
|
||||
oversize and budget gates."""
|
||||
if pool == PoolName.KV:
|
||||
return len(keys) * self._cache.page_size, []
|
||||
return 0, [PoolTransfer(name=pool, keys=list(keys))]
|
||||
|
||||
def _finish_inflight(self, node_id: NodeId, pool: PoolName) -> None:
|
||||
pools = self.inflight_backup_pools.get(node_id)
|
||||
if pools is None:
|
||||
return
|
||||
pools.discard(pool)
|
||||
if not pools:
|
||||
del self.inflight_backup_pools[node_id]
|
||||
|
||||
@staticmethod
|
||||
def _queued_span_key(intent: _UnifiedBackupIntent) -> tuple[NodeId, int]:
|
||||
snapshot = intent.snapshot
|
||||
return snapshot.node_id, len(snapshot.key)
|
||||
|
||||
def _retain_queued_span(self, intent: _UnifiedBackupIntent) -> None:
|
||||
span = self._queued_span_key(intent)
|
||||
refs = self._queued_span_refs.get(span, 0)
|
||||
if refs == 0:
|
||||
self.write_backlog_tokens_ += span[1]
|
||||
self._queued_span_refs[span] = refs + 1
|
||||
|
||||
def _release_queued_span(self, intent: _UnifiedBackupIntent) -> None:
|
||||
span = self._queued_span_key(intent)
|
||||
refs = self._queued_span_refs[span]
|
||||
if refs == 1:
|
||||
del self._queued_span_refs[span]
|
||||
self.write_backlog_tokens_ -= span[1]
|
||||
else:
|
||||
self._queued_span_refs[span] = refs - 1
|
||||
|
||||
def begin_insert_walk(self) -> None:
|
||||
"""Admit each node once per insert walk: the trigger chains every
|
||||
unbacked ancestor, so a depth-d path is presented d(d+1)/2 times."""
|
||||
self._walk_admitted = set()
|
||||
|
||||
def end_insert_walk(self) -> None:
|
||||
self._walk_admitted = None
|
||||
|
||||
def enqueue_backup_intent(self, node_id: NodeId) -> None:
|
||||
"""Snapshot a backup intent and commit it to the write queue.
|
||||
Admission gates: belief skip, parent-cover, backlog cap, oversize.
|
||||
Rejected intents are counted; the node re-triggers on a later hit."""
|
||||
"""Queue one intent per pool with something to write; rejected intents
|
||||
are counted and the node re-triggers on a later hit."""
|
||||
if not self._cache.enable_storage:
|
||||
return
|
||||
if node_id in self.inflight_backup_node_ids:
|
||||
return
|
||||
admitted = self._walk_admitted
|
||||
if admitted is not None:
|
||||
if node_id in admitted:
|
||||
return
|
||||
admitted.add(node_id)
|
||||
snapshot = self._cache.tree_core.snapshot_buffer_backup(
|
||||
node_id, self._cache.hicache_storage_pass_prefix_keys
|
||||
)
|
||||
if snapshot is None:
|
||||
return
|
||||
# Admission cover: beliefs plus content past its D2H launch. The
|
||||
# launched cover keeps republished content (fill inserts under new
|
||||
# node ids) from re-writing while the original write drains.
|
||||
if self._cache.storage_existence_cache.covers_all(
|
||||
PoolName.KV,
|
||||
snapshot.hash_values,
|
||||
extra_cover=self.inflight_backup_hashes,
|
||||
):
|
||||
intents = self._write_intents(snapshot)
|
||||
if not intents:
|
||||
return
|
||||
intent_tokens = len(snapshot.hash_values) * self._cache.page_size
|
||||
if self.write_backlog_tokens_ >= self.write_backlog_cap:
|
||||
# The cap sits at 2x the intrinsic live-backlog ceiling (see
|
||||
# init_hicache), so reaching it means leaked accounting or a
|
||||
# broken stale sweep — a bug, not load.
|
||||
self._backlog_cap_hits += 1
|
||||
if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0:
|
||||
logger.error(
|
||||
"HiCache write backlog cap hit (occurrence %d): "
|
||||
"backlog=%d cap=%d queue=%d. Live backlog is bounded "
|
||||
"by the device pool span, so this indicates a "
|
||||
"stale-sweep or accounting leak.",
|
||||
self._backlog_cap_hits,
|
||||
self.write_backlog_tokens_,
|
||||
self.write_backlog_cap,
|
||||
len(self.pending_write_queue),
|
||||
)
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
return
|
||||
# A span larger than any pool's whole staging capacity can never
|
||||
# stage; admitting it would wedge the head-of-line queue forever.
|
||||
state = BufferBackupState(
|
||||
parent_node_id=snapshot.parent_node_id,
|
||||
parent_is_root=snapshot.parent_is_root,
|
||||
parent_last_hash=snapshot.parent_last_hash,
|
||||
pending = self.inflight_backup_pools.get(node_id, set())
|
||||
parent_covered = self._backup_parent_covered(
|
||||
BufferBackupState(
|
||||
parent_node_id=snapshot.parent_node_id,
|
||||
parent_is_root=snapshot.parent_is_root,
|
||||
parent_last_hash=snapshot.parent_last_hash,
|
||||
),
|
||||
node_id,
|
||||
)
|
||||
if not self._backup_parent_covered(state) or self._backup_oversize(
|
||||
snapshot.node_id, snapshot.hash_values, intent_tokens
|
||||
):
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
return
|
||||
|
||||
intent = _UnifiedBackupIntent(snapshot=snapshot)
|
||||
self.pending_write_queue.append(intent)
|
||||
self.inflight_backup_node_ids.add(snapshot.node_id)
|
||||
self.write_backlog_tokens_ += intent_tokens
|
||||
|
||||
def _build_aux_staging_transfers(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
hash_values: list[str],
|
||||
comp_xfers: Optional[dict[ComponentType, list[PoolTransfer]]] = None,
|
||||
) -> list[PoolTransfer]:
|
||||
"""Keys-only aux transfers mirroring what BACKUP_STORAGE would write;
|
||||
sizes the per-pool oversize gate (beliefs do not consult these)."""
|
||||
transfers: list[PoolTransfer] = []
|
||||
if ComponentType.SWA in self._cache.components:
|
||||
current = (
|
||||
comp_xfers.get(ComponentType.SWA)
|
||||
if comp_xfers is not None
|
||||
else self._cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.SWA,
|
||||
node_id,
|
||||
CacheTransferPhase.BACKUP_HOST,
|
||||
)
|
||||
)
|
||||
for transfer in current or ():
|
||||
if transfer.device_indices is None:
|
||||
continue
|
||||
num_pages = len(transfer.device_indices) // self._cache.page_size
|
||||
if num_pages > 0:
|
||||
transfers.append(
|
||||
PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
keys=hash_values[-num_pages:],
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
page_size = self._cache.page_size
|
||||
span = (snapshot.node_id, len(snapshot.key))
|
||||
for pool, keys in intents:
|
||||
if pool in pending:
|
||||
continue
|
||||
intent_tokens = len(keys) * page_size
|
||||
if (
|
||||
span not in self._queued_span_refs
|
||||
and self.write_backlog_tokens_ >= self.write_backlog_cap
|
||||
):
|
||||
# The cap sits at 2x the intrinsic live-backlog ceiling (see
|
||||
# init_hicache), so reaching it means leaked accounting or a
|
||||
# broken stale sweep — a bug, not load.
|
||||
self._backlog_cap_hits += 1
|
||||
if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0:
|
||||
logger.error(
|
||||
"HiCache write backlog cap hit (occurrence %d): "
|
||||
"backlog=%d cap=%d queue=%d. Live backlog is bounded "
|
||||
"by the device pool span, so this indicates a "
|
||||
"stale-sweep or accounting leak.",
|
||||
self._backlog_cap_hits,
|
||||
self.write_backlog_tokens_,
|
||||
self.write_backlog_cap,
|
||||
len(self.pending_write_queue),
|
||||
)
|
||||
return transfers
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
continue
|
||||
# A span larger than the pool's whole staging capacity can never
|
||||
# stage; admitting it would wedge the head-of-line queue forever.
|
||||
if not parent_covered or self._backup_oversize(*self._sizing(pool, keys)):
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
continue
|
||||
intent = _UnifiedBackupIntent(snapshot=snapshot, pool=pool, keys=list(keys))
|
||||
self.pending_write_queue.append(intent)
|
||||
self.inflight_backup_pools.setdefault(node_id, set()).add(pool)
|
||||
self._retain_queued_span(intent)
|
||||
|
||||
def _backup_oversize(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
hash_values: list[str],
|
||||
intent_tokens: int,
|
||||
aux_xfers: Optional[list[PoolTransfer]] = None,
|
||||
) -> bool:
|
||||
def _backup_oversize(self, kv_tokens: int, aux_xfers: list[PoolTransfer]) -> bool:
|
||||
"""True if any pool's staging need exceeds that pool's write-usable
|
||||
capacity (total for KV, total minus the loads-priority margin for aux
|
||||
pools — matching ``_aux_budget_blocked``'s admission ceiling): such an
|
||||
intent could never stage and would wedge the FIFO head."""
|
||||
if aux_xfers is None:
|
||||
aux_xfers = self._build_aux_staging_transfers(node_id, hash_values)
|
||||
intent could never stage and would wedge the FIFO head. Aux pools
|
||||
staged through the anchor's host pool count against the KV total."""
|
||||
cc = self._cache.cache_controller
|
||||
shared_requests = self._shared_host_requests(intent_tokens, aux_xfers)
|
||||
shared_requests = self._shared_host_requests(kv_tokens, aux_xfers)
|
||||
if shared_requests is not None:
|
||||
pool_tokens = cc.mem_pool_host.size
|
||||
max_write_units = pool_tokens - pool_tokens // 10
|
||||
if self._host_request_units(intent_tokens, aux_xfers) > max_write_units:
|
||||
if self._host_request_units(kv_tokens, aux_xfers) > max_write_units:
|
||||
return True
|
||||
return not self._shared_backup_fits(shared_requests, empty=True)
|
||||
if intent_tokens > cc.mem_pool_host.size:
|
||||
return True
|
||||
for t in aux_xfers or ():
|
||||
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
|
||||
anchor_need = kv_tokens
|
||||
for t in aux_xfers:
|
||||
entry = cc.mem_pool_host.entry_map.get(t.name)
|
||||
if entry is not None and (
|
||||
len(t.keys) * entry.host_pool.page_size
|
||||
> entry.host_pool.size - self._aux_loads_margin(entry.host_pool)
|
||||
):
|
||||
if entry is None:
|
||||
continue
|
||||
need = len(t.keys) * entry.host_pool.page_size
|
||||
if entry.host_pool is anchor_pool:
|
||||
anchor_need += need
|
||||
elif need > entry.host_pool.size - self._aux_loads_margin(entry.host_pool):
|
||||
return True
|
||||
return False
|
||||
return anchor_need > cc.mem_pool_host.size
|
||||
|
||||
def _aux_loads_margin(self, host_pool) -> int:
|
||||
"""Aux-pool tokens reserved for loads: at least one trailing window
|
||||
@@ -623,9 +742,9 @@ class BufferModePipeline:
|
||||
snapshot = intent.snapshot
|
||||
state = self._validate_backup_intent(intent)
|
||||
if state is None:
|
||||
self.inflight_backup_node_ids.discard(snapshot.node_id)
|
||||
intent_tokens = len(snapshot.hash_values) * page_size
|
||||
self.write_backlog_tokens_ -= intent_tokens
|
||||
self._finish_inflight(snapshot.node_id, intent.pool)
|
||||
intent_tokens = len(intent.keys) * page_size
|
||||
self._release_queued_span(intent)
|
||||
swept_tokens += intent_tokens
|
||||
continue
|
||||
survivors.append(intent)
|
||||
@@ -657,54 +776,57 @@ class BufferModePipeline:
|
||||
while self.pending_write_queue:
|
||||
intent = self.pending_write_queue[0]
|
||||
snapshot = intent.snapshot
|
||||
state = states[snapshot.node_id]
|
||||
intent_tokens = len(snapshot.hash_values) * self._cache.page_size
|
||||
if not self._backup_parent_covered(state):
|
||||
node_id = snapshot.node_id
|
||||
state = states[node_id]
|
||||
intent_tokens = len(intent.keys) * self._cache.page_size
|
||||
if not self._backup_parent_covered(state, node_id):
|
||||
# Cascade a dropped parent down the chain rather than creating
|
||||
# a permanent storage hole.
|
||||
self.pending_write_queue.popleft()
|
||||
self.inflight_backup_node_ids.discard(snapshot.node_id)
|
||||
self.write_backlog_tokens_ -= intent_tokens
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
self._drop_queue_head(intent_tokens, dropped=True)
|
||||
continue
|
||||
if self.write_staged_tokens_ >= live_cap:
|
||||
if (
|
||||
self.write_staged_tokens_ >= live_cap
|
||||
and self._host_request_units(*self._sizing(intent.pool, intent.keys))
|
||||
> 0
|
||||
):
|
||||
# Wait for current copies to free staging before rebuilding the head.
|
||||
break
|
||||
device_value, comp_xfers = self._cache.tree_core.build_backup_spec(
|
||||
snapshot.node_id
|
||||
)
|
||||
sizing_xfers = self._build_aux_staging_transfers(
|
||||
snapshot.node_id, snapshot.hash_values, comp_xfers
|
||||
)
|
||||
if self._backup_oversize(
|
||||
snapshot.node_id,
|
||||
snapshot.hash_values,
|
||||
intent_tokens,
|
||||
sizing_xfers,
|
||||
launch = self._launch_transfers(intent)
|
||||
# The pool may have become covered since admission (an earlier
|
||||
# launch this round, a sibling's ack): nothing left to write
|
||||
# retires the intent, not a drop.
|
||||
if launch is None or all(
|
||||
self._pool_covered(pool, keys) for pool, keys in launch[1].items()
|
||||
):
|
||||
self._drop_queue_head(intent_tokens, dropped=False)
|
||||
continue
|
||||
device_value, keys_by_pool, transfers = launch
|
||||
sizing_xfers = [t for t in transfers if t.indices_from_pool is None]
|
||||
kv_tokens = len(device_value)
|
||||
if self._backup_oversize(kv_tokens, sizing_xfers):
|
||||
# A permanently unstageable head must not block the queue.
|
||||
self.pending_write_queue.popleft()
|
||||
self.inflight_backup_node_ids.discard(snapshot.node_id)
|
||||
self.write_backlog_tokens_ -= intent_tokens
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
self._drop_queue_head(intent_tokens, dropped=True)
|
||||
continue
|
||||
shared_domain = self._shared_host_domain()
|
||||
staging_at_limit = (
|
||||
self.write_staged_tokens_
|
||||
+ self._host_request_units(intent_tokens, sizing_xfers)
|
||||
+ self._host_request_units(kv_tokens, sizing_xfers)
|
||||
> live_cap
|
||||
if shared_domain is not None
|
||||
else self.write_staged_tokens_ >= live_cap
|
||||
and self._host_request_units(kv_tokens, sizing_xfers) > 0
|
||||
)
|
||||
if staging_at_limit:
|
||||
# Yield to live fetch demand; retry next round.
|
||||
break
|
||||
if self._aux_budget_blocked(intent, sizing_xfers):
|
||||
if self._aux_budget_blocked(kv_tokens, sizing_xfers):
|
||||
# An aux pool lacks staging headroom: yield at the gate
|
||||
# instead of failing the alloc inside cc.write; acks free
|
||||
# aux staging, retry next round.
|
||||
break
|
||||
if not self._stage_backup_intent(intent, device_value, comp_xfers):
|
||||
if not self._stage_backup_intent(
|
||||
intent, device_value, transfers, keys_by_pool
|
||||
):
|
||||
# Pool full of in-flight staging and nothing reclaimable
|
||||
# (the tree never holds host values in buffer mode):
|
||||
# defer, head-of-line; pending acks will free slots.
|
||||
@@ -715,11 +837,106 @@ class BufferModePipeline:
|
||||
# Do not leave prepared copies deferred across scheduler admission.
|
||||
cc.start_writing()
|
||||
|
||||
def _drop_queue_head(self, intent_tokens: int, *, dropped: bool) -> None:
|
||||
"""Retire the FIFO head without a D2H launch; ``dropped`` feeds the
|
||||
dropped-tokens metric (a pool with nothing left to write is not a
|
||||
drop)."""
|
||||
intent = self.pending_write_queue.popleft()
|
||||
self._finish_inflight(intent.snapshot.node_id, intent.pool)
|
||||
self._release_queued_span(intent)
|
||||
if dropped:
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
|
||||
def _build_backup_transfers(
|
||||
self, node_id: NodeId, *, kv_only: bool = False
|
||||
) -> tuple[
|
||||
torch.Tensor, dict[ComponentType, list[PoolTransfer]], Optional[list[str]]
|
||||
]:
|
||||
"""The node's D2H spec. A FULL component's BACKUP_HOST KV transfer
|
||||
replaces the KV rows (its keys name their pages); its other transfers
|
||||
ride as aux pools. A KV intent skips the other components' transfers."""
|
||||
tree_core = self._cache.tree_core
|
||||
if kv_only:
|
||||
device_value = tree_core.get_component_device_value(
|
||||
node_id, ComponentType.FULL
|
||||
)
|
||||
comp_xfers = {}
|
||||
else:
|
||||
device_value, comp_xfers = tree_core.build_backup_spec(node_id)
|
||||
full_xfers = tree_core.build_hicache_transfers(
|
||||
ComponentType.FULL, node_id, CacheTransferPhase.BACKUP_HOST
|
||||
)
|
||||
kv_keys = None
|
||||
aux: list[PoolTransfer] = []
|
||||
for transfer in full_xfers or ():
|
||||
if transfer.name == PoolName.KV:
|
||||
device_value = (
|
||||
transfer.device_indices
|
||||
if transfer.device_indices is not None
|
||||
else device_value[:0]
|
||||
)
|
||||
kv_keys = list(transfer.keys or ())
|
||||
else:
|
||||
aux.append(transfer)
|
||||
if aux:
|
||||
comp_xfers[ComponentType.FULL] = aux
|
||||
return device_value, comp_xfers, kv_keys
|
||||
|
||||
def _launch_transfers(
|
||||
self, intent: _UnifiedBackupIntent
|
||||
) -> Optional[tuple[torch.Tensor, dict[PoolName, list[str]], list[PoolTransfer]]]:
|
||||
"""(KV rows, keys per written pool, transfers to stage) of one intent,
|
||||
off the node's fresh spec; None when nothing is left to write."""
|
||||
snapshot = intent.snapshot
|
||||
hash_values = snapshot.hash_values
|
||||
device_value, comp_xfers, kv_keys = self._build_backup_transfers(
|
||||
snapshot.node_id, kv_only=intent.pool == PoolName.KV
|
||||
)
|
||||
transfers: list[PoolTransfer] = []
|
||||
if intent.pool == PoolName.KV:
|
||||
keys = hash_values if kv_keys is None else kv_keys
|
||||
if not keys or len(device_value) == 0:
|
||||
return None
|
||||
comp_xfers = {}
|
||||
else:
|
||||
device_value = device_value[:0]
|
||||
keys = []
|
||||
for ct, xfers in comp_xfers.items():
|
||||
kept = []
|
||||
for transfer in xfers:
|
||||
if transfer.name != intent.pool or transfer.indices_from_pool:
|
||||
continue
|
||||
t_keys = self._transfer_keys(transfer, hash_values)
|
||||
if not t_keys:
|
||||
continue
|
||||
if transfer.keys is None:
|
||||
transfer.keys = t_keys
|
||||
transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES
|
||||
kept.append(transfer)
|
||||
keys.extend(t_keys)
|
||||
if kept:
|
||||
transfers.extend(kept)
|
||||
comp_xfers[ct] = kept
|
||||
if not transfers:
|
||||
return None
|
||||
comp_xfers = {ct: xfers for ct, xfers in comp_xfers.items() if xfers}
|
||||
keys_by_pool = {intent.pool: list(keys)}
|
||||
# Sidecars ride their source pool's slots and D2H operation; they
|
||||
# allocate no staging of their own.
|
||||
for sidecar in self._cache._build_backup_sidecar(device_value, comp_xfers):
|
||||
if sidecar.indices_from_pool != intent.pool:
|
||||
continue
|
||||
sidecar.keys = list(keys)
|
||||
keys_by_pool[sidecar.name] = list(keys)
|
||||
transfers.append(sidecar)
|
||||
return device_value, keys_by_pool, transfers
|
||||
|
||||
def _stage_backup_intent(
|
||||
self,
|
||||
intent: _UnifiedBackupIntent,
|
||||
device_value: torch.Tensor,
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]],
|
||||
transfers: list[PoolTransfer],
|
||||
keys_by_pool: dict[PoolName, list[str]],
|
||||
) -> bool:
|
||||
"""Allocate host staging and pin one admitted intent's source.
|
||||
|
||||
@@ -730,64 +947,54 @@ class BufferModePipeline:
|
||||
cache = self._cache
|
||||
cc = cache.cache_controller
|
||||
snapshot = intent.snapshot
|
||||
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
|
||||
# Sidecars reuse the source pool's transient host/device indices. This
|
||||
# includes both KV-derived pools and SWA-derived DSV4 state pools. They
|
||||
# allocate no additional staging, but must ride the same D2H operation
|
||||
# so their bytes are present when the storage write starts.
|
||||
aux_xfers.extend(cache._build_backup_sidecar(device_value, comp_xfers))
|
||||
# One ack id per write: a node may have several pools in flight.
|
||||
self._backup_ack_seq += 1
|
||||
ack_id = -(1 << 40) - self._backup_ack_seq
|
||||
host_indices = cc.write(
|
||||
device_value,
|
||||
node_id=snapshot.node_id,
|
||||
extra_pools=aux_xfers or None,
|
||||
node_id=ack_id,
|
||||
extra_pools=transfers or None,
|
||||
flush=False,
|
||||
)
|
||||
if host_indices is None:
|
||||
self._backup_ack_seq -= 1
|
||||
return False
|
||||
_track_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
|
||||
beliefs = cache.storage_existence_cache
|
||||
for pool, keys in keys_by_pool.items():
|
||||
beliefs.pool(pool).track_inflight(keys)
|
||||
# NOTE: no commit_backup — the node must never appear
|
||||
# host-resident in buffer mode; staging slots live in the entry.
|
||||
lock_params = cache.inc_lock_ref(snapshot.node_id).to_dec_params()
|
||||
occupied_units = self.host_allocation_units(host_indices, aux_xfers)
|
||||
self.ongoing_write_through[snapshot.node_id] = _UnifiedBufferBackupEntry(
|
||||
occupied_units = self.host_allocation_units(host_indices, transfers)
|
||||
self.ongoing_write_through[ack_id] = _UnifiedBufferBackupEntry(
|
||||
intent=intent,
|
||||
host_indices=host_indices,
|
||||
aux_xfers=aux_xfers,
|
||||
aux_xfers=transfers,
|
||||
lock_params=lock_params,
|
||||
occupied_units=occupied_units,
|
||||
keys_by_pool=keys_by_pool,
|
||||
)
|
||||
self.write_staged_tokens_ += occupied_units
|
||||
self.write_backlog_tokens_ -= len(snapshot.hash_values) * cache.page_size
|
||||
self._release_queued_span(intent)
|
||||
return True
|
||||
|
||||
def _aux_budget_blocked(
|
||||
self,
|
||||
intent: _UnifiedBackupIntent,
|
||||
aux: Optional[list[PoolTransfer]] = None,
|
||||
) -> bool:
|
||||
def _aux_budget_blocked(self, kv_tokens: int, aux: list[PoolTransfer]) -> bool:
|
||||
"""True when an aux pool cannot stage this intent right now (free
|
||||
minus the loads-priority margin falls short of the need): defer at
|
||||
the gate instead of failing the alloc inside cc.write and blocking
|
||||
pure-KV intents behind an unallocatable head. The margin enforces
|
||||
loads-have-priority on aux pools the way live_cap does on the KV
|
||||
pool; avail already reflects prefetch-held slots, so no occupancy
|
||||
subtraction here."""
|
||||
snapshot = intent.snapshot
|
||||
if aux is None:
|
||||
aux = self._build_aux_staging_transfers(
|
||||
snapshot.node_id, snapshot.hash_values
|
||||
)
|
||||
subtraction here. Aux pools staged through the anchor's host pool
|
||||
are gated by live_cap and the allocation itself."""
|
||||
cc = self._cache.cache_controller
|
||||
shared_requests = self._shared_host_requests(
|
||||
len(snapshot.hash_values) * self._cache.page_size, aux
|
||||
)
|
||||
shared_requests = self._shared_host_requests(kv_tokens, aux)
|
||||
if shared_requests is not None:
|
||||
return not self._shared_backup_fits(shared_requests)
|
||||
if not aux:
|
||||
return False
|
||||
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
|
||||
for t in aux:
|
||||
entry = cc.mem_pool_host.entry_map.get(t.name)
|
||||
if entry is None:
|
||||
if entry is None or entry.host_pool is anchor_pool:
|
||||
continue
|
||||
need = len(t.keys) * entry.host_pool.page_size
|
||||
headroom = entry.host_pool.available_size() - self._aux_loads_margin(
|
||||
@@ -797,23 +1004,6 @@ class BufferModePipeline:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _aux_window_keys(
|
||||
self, hash_values: list[str], transfer: PoolTransfer
|
||||
) -> Optional[list[str]]:
|
||||
"""Trailing KV page hashes keying an aux transfer's staged window
|
||||
(one key per aux-pool page)."""
|
||||
if transfer.host_indices is None or transfer.host_indices.numel() == 0:
|
||||
return None
|
||||
if transfer.indices_from_pool is not None:
|
||||
return None # sidecar rides another pool's slots; nothing to key
|
||||
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
|
||||
if entry is None:
|
||||
return None
|
||||
num_keys = len(transfer.host_indices) // entry.host_pool.page_size
|
||||
if num_keys == 0 or num_keys > len(hash_values):
|
||||
return None
|
||||
return hash_values[-num_keys:]
|
||||
|
||||
def finish_backup_ack(self, ack_id: int) -> None:
|
||||
"""D2H confirmed: drop the device lock and enqueue the storage write
|
||||
(which reads from the staging copy, so device eviction may proceed)."""
|
||||
@@ -822,73 +1012,65 @@ class BufferModePipeline:
|
||||
snapshot = intent.snapshot
|
||||
self._cache.dec_lock_ref(snapshot.node_id, entry.lock_params)
|
||||
|
||||
# Every independently staged aux pool writes a trailing snapshot keyed
|
||||
# by the last KV page hashes it covers. A derived sidecar writes the
|
||||
# exact key span of its source pool while reusing that source's slots.
|
||||
storage_xfers: list[PoolTransfer] = []
|
||||
storage_sources = {
|
||||
PoolName.KV: PoolTransfer(
|
||||
name=PoolName.KV,
|
||||
host_indices=entry.host_indices,
|
||||
keys=snapshot.hash_values,
|
||||
)
|
||||
}
|
||||
for staged in entry.aux_xfers:
|
||||
if staged.indices_from_pool is not None:
|
||||
continue
|
||||
keys = self._aux_window_keys(snapshot.hash_values, staged)
|
||||
if keys is None:
|
||||
continue
|
||||
transfer = PoolTransfer(
|
||||
name=staged.name,
|
||||
host_indices=staged.host_indices,
|
||||
keys=keys,
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
storage_xfers.append(transfer)
|
||||
storage_sources.setdefault(staged.name, transfer)
|
||||
for staged in entry.aux_xfers:
|
||||
if staged.indices_from_pool is None:
|
||||
continue
|
||||
source = storage_sources.get(staged.indices_from_pool)
|
||||
if source is None:
|
||||
raise AssertionError(
|
||||
"Buffer-mode storage sidecar source missing: "
|
||||
f"{staged.name} from {staged.indices_from_pool}."
|
||||
)
|
||||
storage_xfers.append(
|
||||
PoolTransfer(
|
||||
name=staged.name,
|
||||
keys=source.keys,
|
||||
hit_policy=staged.hit_policy,
|
||||
indices_from_pool=staged.indices_from_pool,
|
||||
)
|
||||
)
|
||||
storage_xfers = self._storage_transfers(entry.aux_xfers)
|
||||
# Another pool's write carries no KV pages; its keys ride the transfer.
|
||||
hash_value = (
|
||||
list(entry.keys_by_pool[PoolName.KV]) if intent.pool == PoolName.KV else []
|
||||
)
|
||||
operation_id = self._cache.cache_controller.write_storage(
|
||||
entry.host_indices,
|
||||
snapshot.key.token_ids,
|
||||
snapshot.hash_values,
|
||||
hash_value,
|
||||
snapshot.prefix_keys,
|
||||
extra_pools=storage_xfers or None,
|
||||
)
|
||||
self.ongoing_backup[operation_id] = entry
|
||||
|
||||
@staticmethod
|
||||
def _storage_transfers(staged: list[PoolTransfer]) -> list[PoolTransfer]:
|
||||
"""One storage transfer per pool: a pool staged in several transfers
|
||||
becomes one contiguous key span over their slots; a derived sidecar
|
||||
writes its source's key span over the source's slots."""
|
||||
merged: dict[PoolName, PoolTransfer] = {}
|
||||
for transfer in staged:
|
||||
if transfer.indices_from_pool is not None:
|
||||
merged[transfer.name] = PoolTransfer(
|
||||
name=transfer.name,
|
||||
keys=list(transfer.keys),
|
||||
hit_policy=transfer.hit_policy,
|
||||
indices_from_pool=transfer.indices_from_pool,
|
||||
)
|
||||
continue
|
||||
earlier = merged.get(transfer.name)
|
||||
if earlier is None:
|
||||
merged[transfer.name] = PoolTransfer(
|
||||
name=transfer.name,
|
||||
host_indices=transfer.host_indices,
|
||||
keys=list(transfer.keys),
|
||||
hit_policy=transfer.hit_policy,
|
||||
)
|
||||
continue
|
||||
earlier.keys.extend(transfer.keys)
|
||||
earlier.host_indices = torch.cat(
|
||||
[earlier.host_indices, transfer.host_indices]
|
||||
)
|
||||
return list(merged.values())
|
||||
|
||||
def finish_storage_write_ack(self, operation_id: int) -> None:
|
||||
"""Storage write acked (rank-synced drain): free the entry's staging
|
||||
outright. Existence entries are added unconditionally
|
||||
(completed_tokens can diverge across ranks under backend failure) to
|
||||
keep admission decisions TP-deterministic. No-op for operations this
|
||||
pipeline does not own (e.g. acks for already-reset state)."""
|
||||
"""Storage ack (rank-synced drain): free the staging and feed each
|
||||
written pool's beliefs unconditionally, keeping admission
|
||||
TP-deterministic under per-rank backend failures."""
|
||||
entry = self.ongoing_backup.pop(operation_id, None)
|
||||
if entry is None:
|
||||
return
|
||||
intent = entry.intent
|
||||
snapshot = intent.snapshot
|
||||
self._cache.storage_existence_cache.add(PoolName.KV, snapshot.hash_values)
|
||||
beliefs = self._cache.storage_existence_cache
|
||||
for pool, keys in entry.keys_by_pool.items():
|
||||
pool_beliefs = beliefs.pool(pool)
|
||||
pool_beliefs.add(keys)
|
||||
pool_beliefs.untrack_inflight(keys)
|
||||
self._free_staging_now(entry.host_indices, entry.aux_xfers)
|
||||
self.write_staged_tokens_ -= entry.occupied_units
|
||||
self.inflight_backup_node_ids.discard(snapshot.node_id)
|
||||
_untrack_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
|
||||
self._finish_inflight(entry.intent.snapshot.node_id, entry.intent.pool)
|
||||
|
||||
def _free_staging_now(
|
||||
self, host_indices: torch.Tensor, aux_xfers: list[PoolTransfer]
|
||||
@@ -1163,10 +1345,16 @@ class BufferModePipeline:
|
||||
staged_pages = num_tokens // cache.page_size
|
||||
staged_hashes = hash_value[:staged_pages]
|
||||
staged_kv = host_indices[:num_tokens]
|
||||
# Feed existence beliefs from the storage-fetched pages: the fetch
|
||||
# itself is the evidence, so feeding is sound even if this staged
|
||||
# prefetch is later dropped unconsumed.
|
||||
cache.storage_existence_cache.add(PoolName.KV, list(staged_hashes))
|
||||
# The fetch is evidence for every pool it delivered (sound even if the
|
||||
# staged prefetch is later dropped): KV-derived sidecars came with the
|
||||
# staged pages, other pools with the keys their transfer carries.
|
||||
beliefs = cache.storage_existence_cache
|
||||
beliefs.pool(PoolName.KV).add(staged_hashes)
|
||||
for transfer in aux_xfers:
|
||||
if transfer.indices_from_pool == PoolName.KV:
|
||||
beliefs.pool(transfer.name).add(staged_hashes)
|
||||
elif transfer.keys:
|
||||
beliefs.pool(transfer.name).add(transfer.keys)
|
||||
self.staged_prefetches[request] = _StagedPrefetch(
|
||||
request=request,
|
||||
key_tokens=array(
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
"""Local existence cache for HiCache buffer_only mode.
|
||||
"""Local existence beliefs for HiCache buffer_only mode.
|
||||
|
||||
In buffer mode host memory holds no persistent copy, so without a local
|
||||
existence signal every re-insert of a hot prefix re-stages and re-writes to
|
||||
L3 storage. This cache is that signal: a bounded LRU of (pool, page-hash)
|
||||
entries *believed* present in storage.
|
||||
L3 storage. The beliefs are that signal: per storage pool, a bounded LRU of
|
||||
page hashes *believed* present in storage, kept and healed independently so
|
||||
a node is written pool by pool (only the pools not believed stored).
|
||||
|
||||
Semantics are advisory, not authoritative:
|
||||
|
||||
- A hit skips the redundant D2H + storage write.
|
||||
- A hit skips the pool's redundant D2H staging + storage write.
|
||||
- A stale positive (backend evicted the data) costs skipped write-backs until
|
||||
a prefetch hit-query shortfall invalidates the entries; the next insert
|
||||
then writes the data back. Never a correctness issue — at worst one cold
|
||||
recompute, the same as any cache miss.
|
||||
a prefetch hit-query shortfall invalidates the pool's entries; the next
|
||||
insert then writes that pool back. Never a correctness issue — at worst
|
||||
one cold recompute, the same as any cache miss.
|
||||
- A miss (entry LRU-evicted or never seen) just costs one redundant write
|
||||
(idempotent: storage keys are content-addressed).
|
||||
|
||||
@@ -21,6 +22,10 @@ recompute (same tokens => same chain). Page hashes are chained and the write
|
||||
path is prefix-contiguous (parent-cover gate), so a node's own page set is
|
||||
the only thing a caller needs to check.
|
||||
|
||||
A pool also tracks its in-flight content (D2H launch to storage ack), which
|
||||
admission treats as covered; its ``PoolBeliefPolicy`` says how its keys sit
|
||||
on the chain (a component may supply one via ``buffer_belief_policy``).
|
||||
|
||||
TP determinism: replicas stay identical because every mutation happens on the
|
||||
scheduler thread at lockstep points with cross-rank-reduced inputs
|
||||
(storage-ack drain, prefetch-hit drain, fill commit). Do not touch it from
|
||||
@@ -30,82 +35,165 @@ anywhere else.
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Container, Iterable, Sequence
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
# ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers roughly
|
||||
# 32M tokens at page size 64 across all pools.
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy
|
||||
|
||||
# Per pool: ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers
|
||||
# roughly 32M tokens at page size 64.
|
||||
HICACHE_EXISTENCE_CACHE_MAX_ENTRIES = 512 * 1024
|
||||
|
||||
|
||||
class StorageExistenceCache:
|
||||
class PoolExistenceBeliefs:
|
||||
"""One pool's LRU of page hashes believed stored, plus its refcounted
|
||||
in-flight content (D2H launch to storage ack)."""
|
||||
|
||||
def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES):
|
||||
self.max_entries = max_entries
|
||||
self._entries: OrderedDict[tuple[str, str], None] = OrderedDict()
|
||||
self._entries: OrderedDict[str, None] = OrderedDict()
|
||||
self._inflight: dict[str, int] = {}
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._entries)
|
||||
|
||||
def add(self, pool: str, hashes: Iterable[str]) -> None:
|
||||
def add(self, hashes: Iterable[str]) -> None:
|
||||
entries = self._entries
|
||||
move_to_end = entries.move_to_end
|
||||
for h in hashes:
|
||||
key = (pool, h)
|
||||
entries[key] = None
|
||||
move_to_end(key)
|
||||
entries[h] = None
|
||||
move_to_end(h)
|
||||
pop_oldest = entries.popitem
|
||||
for _ in range(len(entries) - self.max_entries):
|
||||
pop_oldest(last=False)
|
||||
|
||||
def contains(self, pool: str, page_hash: str) -> bool:
|
||||
def contains(self, page_hash: str) -> bool:
|
||||
entries = self._entries
|
||||
key = (pool, page_hash)
|
||||
if key not in entries:
|
||||
if page_hash not in entries:
|
||||
return False
|
||||
entries.move_to_end(key)
|
||||
entries.move_to_end(page_hash)
|
||||
return True
|
||||
|
||||
def contains_all(self, pool: str, hashes: Iterable[str]) -> bool:
|
||||
def contains_all(self, hashes: Iterable[str]) -> bool:
|
||||
entries = self._entries
|
||||
move_to_end = entries.move_to_end
|
||||
for h in hashes:
|
||||
key = (pool, h)
|
||||
try:
|
||||
move_to_end(key)
|
||||
move_to_end(h)
|
||||
except KeyError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def covers_all(
|
||||
self,
|
||||
pool: str,
|
||||
hashes: Iterable[str],
|
||||
extra_cover: Container[str] = frozenset(),
|
||||
) -> bool:
|
||||
"""True when every page is believed stored or sits in
|
||||
``extra_cover`` (e.g. content past its D2H launch, which always
|
||||
reaches its storage-ack). LRU-touches the believed entries."""
|
||||
def covered(self, hashes: Iterable[str]) -> bool:
|
||||
"""Every page believed stored or in flight; LRU-touches believed entries."""
|
||||
inflight = self._inflight
|
||||
if not inflight:
|
||||
return self.contains_all(hashes)
|
||||
entries = self._entries
|
||||
move_to_end = entries.move_to_end
|
||||
if extra_cover:
|
||||
for h in hashes:
|
||||
key = (pool, h)
|
||||
if key in entries:
|
||||
move_to_end(key)
|
||||
elif h not in extra_cover:
|
||||
return False
|
||||
return True
|
||||
return self.contains_all(pool, hashes)
|
||||
for h in hashes:
|
||||
if h in entries:
|
||||
move_to_end(h)
|
||||
elif h not in inflight:
|
||||
return False
|
||||
return True
|
||||
|
||||
def invalidate_beyond(
|
||||
self, pool: str, hashes: Sequence[str], keep_pages: int
|
||||
) -> None:
|
||||
"""Ground-truth heal from a prefetch hit query: discard beliefs
|
||||
beyond the leading ``keep_pages`` of a hash chain (the folded
|
||||
usable cut). The next insert re-writes the discarded span, closing
|
||||
stale positives and aux holes at the cut."""
|
||||
def track_inflight(self, hashes: Iterable[str]) -> None:
|
||||
"""One ref per page hash at its D2H launch (several launched writes
|
||||
can carry the same content)."""
|
||||
refs = self._inflight
|
||||
for h in hashes:
|
||||
refs[h] = refs.get(h, 0) + 1
|
||||
|
||||
def untrack_inflight(self, hashes: Iterable[str]) -> None:
|
||||
"""Drop one ref per page hash at the storage ack (or a failed launch)."""
|
||||
refs = self._inflight
|
||||
for h in hashes:
|
||||
n = refs.get(h, 0) - 1
|
||||
if n <= 0:
|
||||
refs.pop(h, None)
|
||||
else:
|
||||
refs[h] = n
|
||||
|
||||
def invalidate_beyond(self, hashes: Sequence[str], keep_pages: int) -> None:
|
||||
"""Discard the beliefs beyond the leading ``keep_pages`` of ``hashes``;
|
||||
the next insert re-writes that span."""
|
||||
pop = self._entries.pop
|
||||
for h in hashes[keep_pages:]:
|
||||
pop((pool, h), None)
|
||||
pop(h, None)
|
||||
|
||||
def clear(self) -> None:
|
||||
self._entries.clear()
|
||||
self._inflight.clear()
|
||||
|
||||
def clear_inflight(self) -> None:
|
||||
self._inflight.clear()
|
||||
|
||||
|
||||
class StorageExistenceCache:
|
||||
"""The beliefs of every storage pool, created on first use."""
|
||||
|
||||
def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES):
|
||||
self.max_entries = max_entries
|
||||
self._pools: dict[str, PoolExistenceBeliefs] = {}
|
||||
|
||||
def pool(self, name: str) -> PoolExistenceBeliefs:
|
||||
beliefs = self._pools.get(name)
|
||||
if beliefs is None:
|
||||
beliefs = self._pools[name] = PoolExistenceBeliefs(self.max_entries)
|
||||
return beliefs
|
||||
|
||||
def clear(self) -> None:
|
||||
for beliefs in self._pools.values():
|
||||
beliefs.clear()
|
||||
|
||||
def clear_inflight(self) -> None:
|
||||
for beliefs in self._pools.values():
|
||||
beliefs.clear_inflight()
|
||||
|
||||
|
||||
class PoolBeliefPolicy:
|
||||
"""How a pool's keys sit on a node's page-hash chain, hence how a storage
|
||||
verdict heals its beliefs: a rank-agreed hit-query boundary (``evidence``
|
||||
= the query reported this pool) or a transfer's delivered page count."""
|
||||
|
||||
def heal_from_query(
|
||||
self,
|
||||
beliefs: PoolExistenceBeliefs,
|
||||
chain: Sequence[str],
|
||||
keys: Sequence[str],
|
||||
hit_pages: int,
|
||||
*,
|
||||
evidence: bool,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def heal_from_transfer(
|
||||
self,
|
||||
beliefs: PoolExistenceBeliefs,
|
||||
keys: Sequence[str],
|
||||
delivered_pages: int,
|
||||
) -> None:
|
||||
beliefs.invalidate_beyond(keys, delivered_pages)
|
||||
|
||||
|
||||
class ChainPagesPolicy(PoolBeliefPolicy):
|
||||
"""One key per chain page (ALL_PAGES). The agreed boundary is the MIN of
|
||||
per-rank prefix hits, so every rank verified the chain up to it."""
|
||||
|
||||
def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence):
|
||||
beliefs.invalidate_beyond(chain, hit_pages)
|
||||
if evidence:
|
||||
beliefs.add(chain[:hit_pages])
|
||||
|
||||
|
||||
class TrailingWindowPolicy(PoolBeliefPolicy):
|
||||
"""Keys are a trailing window of the chain (TRAILING_PAGES). A rank only
|
||||
verified the window ending at its own boundary, so the query only
|
||||
invalidates; acks and fills teach presence."""
|
||||
|
||||
def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence):
|
||||
beliefs.invalidate_beyond(chain, hit_pages)
|
||||
|
||||
|
||||
def default_belief_policy(hit_policy: PoolHitPolicy) -> PoolBeliefPolicy:
|
||||
"""The built-in policy for a pool no component claims."""
|
||||
if hit_policy == PoolHitPolicy.TRAILING_PAGES:
|
||||
return TrailingWindowPolicy()
|
||||
return ChainPagesPolicy()
|
||||
|
||||
@@ -51,6 +51,10 @@ from sglang.srt.utils import broadcast_pyobj
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Pool boundaries are nonnegative; an omitted rank must dominate MIN so
|
||||
# a peer cannot establish presence of that rank's storage shard.
|
||||
_NO_POOL_VERDICT = -1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PPPrefetchPoolSpec:
|
||||
@@ -165,6 +169,10 @@ class PrefetchOperation(StorageOperation):
|
||||
pool_transfers=pool_transfers,
|
||||
)
|
||||
self.pool_transfers_done = not bool(pool_transfers)
|
||||
# The hit query's verdict per pool, as the chain prefix length (in
|
||||
# pages) up to which the pool is present; rank-reduced with the
|
||||
# folded hit count. Empty until the query ran on this rank.
|
||||
self.query_pool_hit_pages: dict[PoolName, int] = {}
|
||||
self.buffer_host_occupied_units: Optional[int] = None
|
||||
# The Python transfer worker leaves the unfinished tail to the ACK drain;
|
||||
# a controller that releases it itself must set this False.
|
||||
@@ -1348,9 +1356,11 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
if operation.assume_stored:
|
||||
# A prior hit on a suffix of this span proved it stored, and writes
|
||||
# are prefix-covered, so re-querying only adds a round trip.
|
||||
# are prefix-covered, so re-querying only adds a round trip. Nothing
|
||||
# is verified now, so no pool gets a verdict to learn from.
|
||||
kv_hit_pages = len(hash_value)
|
||||
operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages)
|
||||
operation.query_pool_hit_pages = {}
|
||||
return hash_value, kv_hit_pages * self.page_size
|
||||
|
||||
extra_info = HiCacheStorageExtraInfo(
|
||||
@@ -1369,12 +1379,45 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
kv_hit_pages = hit_result.kv_hit_pages
|
||||
operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages)
|
||||
# Each pool's own boundary (the KV pool's where the backend reports
|
||||
# it, else the folded cut); the beliefs heal pool by pool. A pool the
|
||||
# backend did not report carries no verdict.
|
||||
pool_hits = hit_result.extra_pool_hit_pages
|
||||
operation.query_pool_hit_pages = {
|
||||
PoolName.KV: pool_hits.get(PoolName.KV, kv_hit_pages)
|
||||
}
|
||||
for transfer in operation.pool_transfers or ():
|
||||
if transfer.name in pool_hits:
|
||||
operation.query_pool_hit_pages[transfer.name] = pool_hits[transfer.name]
|
||||
|
||||
return (
|
||||
hash_value[:kv_hit_pages],
|
||||
kv_hit_pages * self.page_size,
|
||||
)
|
||||
|
||||
def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int:
|
||||
"""One MIN collective for the folded hit count and every pool's own
|
||||
boundary, so per-pool beliefs stay rank-identical. A pool omitted
|
||||
by any rank heals from the folded cut without learning presence."""
|
||||
pools = list(PoolName)
|
||||
local = operation.query_pool_hit_pages
|
||||
packed = torch.tensor(
|
||||
[storage_hit_count] + [local.get(pool, _NO_POOL_VERDICT) for pool in pools],
|
||||
dtype=torch.int,
|
||||
)
|
||||
self._all_reduce(
|
||||
packed,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_hits_sync_groups,
|
||||
)
|
||||
reduced = packed.tolist()
|
||||
operation.query_pool_hit_pages = {
|
||||
pool: pages
|
||||
for pool, pages in zip(pools, reduced[1:], strict=True)
|
||||
if pages != _NO_POOL_VERDICT
|
||||
}
|
||||
return int(reduced[0])
|
||||
|
||||
def _move_pool_indices(
|
||||
self, host_pool, host_indices, device_indices, *, write_back_jit: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -17,7 +17,11 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
PoolTransferResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401
|
||||
BASE_COMPONENT_TYPE,
|
||||
ComponentType,
|
||||
@@ -25,7 +29,11 @@ from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
|
||||
PoolBeliefPolicy,
|
||||
)
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.pool_host.group import HostPoolGroup, PoolEntry
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
CacheAction,
|
||||
ComponentAction,
|
||||
@@ -699,6 +707,40 @@ class TreeComponent(ABC):
|
||||
"""Allocate prefetch staging sized by prepare_prefetch, once the hit is known."""
|
||||
return None
|
||||
|
||||
# ---- Buffer-mode backup hooks (hicache_host_memory_mode=buffer_only) ----
|
||||
# Buffer mode writes a node pool by pool: keys per pool come from
|
||||
# ``buffer_backup_keys`` or the BACKUP_HOST transfers, beliefs heal through
|
||||
# ``buffer_belief_policy``; the pipeline pins the node from D2H launch to
|
||||
# ack, so a component dropping rows asks ``buffer_backup_pending`` first.
|
||||
|
||||
def buffer_mode_host_pool_entries(
|
||||
self, host_pool_group: HostPoolGroup
|
||||
) -> list[PoolEntry]:
|
||||
"""Extra host pool entries this component stages through (an entry
|
||||
may alias the anchor's host pool)."""
|
||||
return []
|
||||
|
||||
def buffer_backup_keys(
|
||||
self, node: UnifiedTreeNode, hash_values: list[str]
|
||||
) -> dict[PoolName, list[str]]:
|
||||
"""Per-pool storage keys a backup of ``node`` writes for this component
|
||||
(empty list: withhold the pool; empty dict: derive them from its
|
||||
BACKUP_HOST transfers). A component naming any pool names all of them."""
|
||||
return {}
|
||||
|
||||
def buffer_backup_parent_covered(
|
||||
self, parent_node_id: NodeId, node_id: NodeId
|
||||
) -> bool:
|
||||
"""Whether this component covers the parent of ``node_id`` without
|
||||
the parent's KV pages in storage (no prefix hole for a reader)."""
|
||||
return False
|
||||
|
||||
def buffer_belief_policy(self, pool: PoolName) -> Optional[PoolBeliefPolicy]:
|
||||
"""The policy healing ``pool``'s existence beliefs, for a pool this
|
||||
component owns; None picks the built-in one for the transfer's hit
|
||||
policy."""
|
||||
return None
|
||||
|
||||
def build_hicache_transfers(
|
||||
self,
|
||||
node: UnifiedTreeNode,
|
||||
|
||||
@@ -1055,6 +1055,26 @@ class SWAComponent(TreeComponent):
|
||||
host_indices = self._swa_kv_pool_host.alloc(num_tokens)
|
||||
return host_indices
|
||||
|
||||
def buffer_backup_keys(
|
||||
self, node: UnifiedTreeNode, hash_values: list[str]
|
||||
) -> dict[PoolName, list[str]]:
|
||||
# Buffer mode stages the node's own SWA rows, one key per page from
|
||||
# the chain's tail, so admission needs no transfer (and no device op).
|
||||
if not self.tree_core.is_host_memory_buffer_only:
|
||||
return {}
|
||||
cd = node.component_data[self.component_type]
|
||||
if not self.tree_core.has_swa_host_pool or cd.value is None:
|
||||
return {PoolName.SWA: []}
|
||||
rows = (
|
||||
node.component_data[BASE_COMPONENT_TYPE].value
|
||||
if self._unified_allocator() is not None
|
||||
else cd.value
|
||||
)
|
||||
num_pages = len(rows) // self._swa_kv_pool_host.page_size
|
||||
if num_pages == 0 or num_pages > len(hash_values):
|
||||
return {PoolName.SWA: []}
|
||||
return {PoolName.SWA: list(hash_values[-num_pages:])}
|
||||
|
||||
def build_hicache_transfers(
|
||||
self,
|
||||
node: UnifiedTreeNode,
|
||||
|
||||
@@ -2381,6 +2381,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
"""Read a node's device->host backup spec (device value + component transfers) now."""
|
||||
return self._build_backup_spec(self.node_by_id(node_id))
|
||||
|
||||
def buffer_backup_pool_keys(
|
||||
self, node_id: NodeId, hash_values: list[str]
|
||||
) -> dict[ComponentType, dict[PoolName, list[str]]]:
|
||||
node = self.node_by_id(node_id)
|
||||
named: dict[ComponentType, dict[PoolName, list[str]]] = {}
|
||||
for comp in self.components:
|
||||
pool_keys = comp.buffer_backup_keys(node, hash_values)
|
||||
if pool_keys:
|
||||
named[comp.component_type] = pool_keys
|
||||
return named
|
||||
|
||||
def build_backup_kv_action(self, node_id: NodeId) -> BackupKV:
|
||||
return self._build_backup_kv_action(self.node_by_id(node_id))
|
||||
|
||||
def _build_backup_spec(self, node: UnifiedTreeNode):
|
||||
"""Gather missing Full and component transfers for Host backup."""
|
||||
device_value = node.component_data[BASE_COMPONENT_TYPE].value
|
||||
|
||||
@@ -127,7 +127,11 @@ if TYPE_CHECKING:
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.events import KVCacheEventRecorder
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
PoolTransferResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
BackupKV,
|
||||
@@ -549,6 +553,20 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""Read a node's device->host backup spec (device value + transfers) now."""
|
||||
...
|
||||
|
||||
def buffer_backup_pool_keys(
|
||||
self, node_id: NodeId, hash_values: list[str]
|
||||
) -> dict[ComponentType, dict[PoolName, list[str]]]:
|
||||
"""Per component, the storage keys per pool a buffer-mode backup of
|
||||
the node writes (``TreeComponent.buffer_backup_keys``); Rust cores name none."""
|
||||
return {}
|
||||
|
||||
def build_backup_kv_action(self, node_id: NodeId) -> BackupKV:
|
||||
"""The write-through backup action for a node and its unstored
|
||||
ancestors; only component-requested buffer backups need it."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not expose build_backup_kv_action"
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def build_storage_backup_spec(
|
||||
self, node_id: NodeId, pass_prefix_keys: bool
|
||||
|
||||
@@ -36,10 +36,17 @@ from sglang.srt.mem_cache.buffer_mode.pipeline import (
|
||||
validate_buffer_only_stack,
|
||||
)
|
||||
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
|
||||
PoolBeliefPolicy,
|
||||
StorageExistenceCache,
|
||||
default_belief_policy,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import RetractionBackup
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer, SidecarPoolSpec
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolHitPolicy,
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
SidecarPoolSpec,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
PPPrefetchDecision,
|
||||
@@ -391,6 +398,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.ongoing_backup: dict[int, tuple[NodeId, DecLockRefParams]] = {}
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.reset()
|
||||
# The dropped writes' content no longer reaches a storage ack.
|
||||
self.storage_existence_cache.clear_inflight()
|
||||
|
||||
if self.cache_controller is not None:
|
||||
self.cache_controller.reset()
|
||||
@@ -487,10 +496,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if swa is not None and self.tree_core.has_swa_host_pool
|
||||
else 0
|
||||
),
|
||||
# Leak backstop only: live queued tokens are intrinsically
|
||||
# bounded by the FULL device pool (one intent per node, stale
|
||||
# intents swept per tick), so a cap that binds on live
|
||||
# content would drop-newest and punch storage holes.
|
||||
# Leak backstop only: the queue is charged once per unique node
|
||||
# span across pools; a cap binding on live content would
|
||||
# drop-newest and punch storage holes.
|
||||
write_backlog_cap=2 * self.token_to_kv_pool_allocator.size_full,
|
||||
)
|
||||
# State initialization
|
||||
@@ -498,6 +506,13 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.cache_controller.host_write_staged_tokens_fn = lambda: (
|
||||
self.buffer_pipeline.write_staged_tokens_
|
||||
)
|
||||
# Pools a component's backups stage through besides the stock
|
||||
# ones (an entry may alias the anchor's host pool).
|
||||
for component in self.components.values():
|
||||
for entry in component.buffer_mode_host_pool_entries(
|
||||
self.host_pool_group
|
||||
):
|
||||
self.cache_controller.register_host_pool_entry(entry)
|
||||
self.write_through_threshold = (
|
||||
1 if get_memory().hicache_write_policy == "write_through" else 2
|
||||
)
|
||||
@@ -596,6 +611,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return InsertResult(prefix_len=0)
|
||||
# Fail fast on re-entrancy without touching the in-flight walk.
|
||||
assert not self.tree_core.has_ongoing_insert(), "re-entrant insert"
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.begin_insert_walk()
|
||||
# Pump the resumable insert, applying each step's actions at its barrier.
|
||||
try:
|
||||
step = self.tree_core.begin_insert(params)
|
||||
@@ -609,6 +626,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
finally:
|
||||
# Drain still-pending actions so frees reach the allocator on abort.
|
||||
self._apply_cache_actions(self.tree_core.end_insert())
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.end_insert_walk()
|
||||
|
||||
@rank_consensus(same_params=True, same_results=True)
|
||||
def evict(self, params: EvictParams) -> EvictResult:
|
||||
@@ -1577,6 +1596,22 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.writing_check(write_back=True)
|
||||
return True
|
||||
|
||||
def buffer_backup_pending(self, node_id: NodeId) -> bool:
|
||||
"""Whether a buffer-mode write of ``node_id`` is queued, staged or
|
||||
awaiting its storage ack (see ``BufferModePipeline.backup_pending``)."""
|
||||
return self.buffer_pipeline is not None and self.buffer_pipeline.backup_pending(
|
||||
node_id
|
||||
)
|
||||
|
||||
def request_buffer_backup(self, node_id: NodeId) -> None:
|
||||
"""Queue a buffer-mode backup of ``node_id`` (and its unstored
|
||||
ancestors) outside the insert trigger; no-op without buffer storage."""
|
||||
if self.buffer_pipeline is None or not self.enable_storage:
|
||||
return
|
||||
self._execute_and_commit_kv_backup(
|
||||
self.tree_core.build_backup_kv_action(node_id)
|
||||
)
|
||||
|
||||
def _execute_and_commit_kv_backup(
|
||||
self, action: BackupKV, write_back: bool = False
|
||||
) -> int:
|
||||
@@ -2408,9 +2443,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
)
|
||||
if pool_transfers and not all_succeeded:
|
||||
# Drop the KV beliefs from the first page any pool failed to serve;
|
||||
# the next insert then re-writes that span through one FULL check,
|
||||
# restoring the missing aux pages.
|
||||
# Drop each pool's beliefs beyond what it served (KV and its
|
||||
# sidecars beyond the completed span); the next insert re-writes
|
||||
# only the missing pools.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
@@ -2418,18 +2453,29 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
beliefs = self.storage_existence_cache
|
||||
if keep_pages < len(invalidation_hashes):
|
||||
self._belief_policy(
|
||||
PoolName.KV, PoolHitPolicy.ALL_PAGES
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(PoolName.KV), invalidation_hashes, keep_pages
|
||||
)
|
||||
for transfer in operation.pool_transfers or ():
|
||||
if transfer.indices_from_pool == PoolName.KV:
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name),
|
||||
invalidation_hashes,
|
||||
keep_pages,
|
||||
)
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if transfer.keys is None:
|
||||
keep_pages = 0
|
||||
elif count < len(transfer.keys):
|
||||
# Aux transfers key the chain's trailing pages.
|
||||
keep_pages = min(
|
||||
keep_pages,
|
||||
max(0, len(invalidation_hashes) - len(transfer.keys)),
|
||||
if transfer.keys is not None and count < len(transfer.keys):
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name), transfer.keys, count
|
||||
)
|
||||
self.storage_existence_cache.invalidate_beyond(
|
||||
PoolName.KV, invalidation_hashes, keep_pages=keep_pages
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason="storage_transfer"
|
||||
)
|
||||
@@ -2444,11 +2490,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
|
||||
"kv_beliefs_kept_pages=%d",
|
||||
"kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s",
|
||||
request.rid,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
keep_pages,
|
||||
{
|
||||
getattr(transfer.name, "value", transfer.name): count
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
},
|
||||
)
|
||||
return False
|
||||
return True
|
||||
@@ -2637,18 +2687,47 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
),
|
||||
)
|
||||
|
||||
def _belief_policy(
|
||||
self, pool: PoolName, hit_policy: PoolHitPolicy
|
||||
) -> PoolBeliefPolicy:
|
||||
"""The policy healing ``pool``'s existence beliefs: the owning
|
||||
component's, else the built-in one for the transfer's hit policy."""
|
||||
for component in self.components.values():
|
||||
policy = component.buffer_belief_policy(pool)
|
||||
if policy is not None:
|
||||
return policy
|
||||
return default_belief_policy(hit_policy)
|
||||
|
||||
def _invalidate_absent_from_hit_query(self, operation) -> None:
|
||||
"""Drop KV beliefs beyond the folded usable cut (rank-synced): the
|
||||
next insert then re-writes the node (all pools), healing stale
|
||||
positives and aux holes at the cut through one FULL check."""
|
||||
"""Heal each pool's beliefs beyond its rank-agreed boundary (a
|
||||
chain-page pool also learns presence up to it); a pool without a
|
||||
verdict heals from the folded cut."""
|
||||
if self.host_memory_mode != "buffer_only":
|
||||
return
|
||||
chain = operation.all_hash_values
|
||||
if chain is None:
|
||||
return
|
||||
self.storage_existence_cache.invalidate_beyond(
|
||||
PoolName.KV, chain, keep_pages=operation.storage_hit_count // self.page_size
|
||||
beliefs = self.storage_existence_cache
|
||||
verdicts = operation.query_pool_hit_pages
|
||||
folded_pages = operation.storage_hit_count // self.page_size
|
||||
|
||||
def verdict(pool: PoolName) -> tuple[int, bool]:
|
||||
pages = verdicts.get(pool)
|
||||
return (folded_pages, False) if pages is None else (pages, True)
|
||||
|
||||
kv_pages, kv_evidence = verdict(PoolName.KV)
|
||||
self._belief_policy(PoolName.KV, PoolHitPolicy.ALL_PAGES).heal_from_query(
|
||||
beliefs.pool(PoolName.KV), chain, chain, kv_pages, evidence=kv_evidence
|
||||
)
|
||||
for transfer in operation.pool_transfers or ():
|
||||
hit_pages, evidence = verdict(transfer.name)
|
||||
self._belief_policy(transfer.name, transfer.hit_policy).heal_from_query(
|
||||
beliefs.pool(transfer.name),
|
||||
chain,
|
||||
list(transfer.keys or ()),
|
||||
hit_pages,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
def _account_prefetch_outcome(self, operation, revoked: bool) -> None:
|
||||
"""Feed the cumulative prefetch-outcome counters at the (rank-synced)
|
||||
|
||||
@@ -2,31 +2,47 @@
|
||||
|
||||
import unittest
|
||||
from array import array
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import (
|
||||
BufferModePipeline,
|
||||
_UnifiedBackupIntent,
|
||||
validate_buffer_only_stack,
|
||||
)
|
||||
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
|
||||
StorageExistenceCache,
|
||||
)
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
HiCacheFile,
|
||||
HiCacheStorageConfig,
|
||||
PoolHitPolicy,
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
SidecarPoolSpec,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
PrefetchOperation,
|
||||
)
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
ComponentType,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
BufferBackupSnapshot,
|
||||
BufferBackupState,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_radix_cache import _OngoingPrefetch
|
||||
from sglang.srt.mem_cache.unified_radix_cache import (
|
||||
UnifiedRadixCache,
|
||||
_OngoingPrefetch,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
@@ -44,6 +60,171 @@ class TestBufferModeSidecar(unittest.TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_file_query_missing_rank_preserves_swa_repair(self):
|
||||
snapshot = self._backup_snapshot()
|
||||
token_ids = list(snapshot.key.token_ids)
|
||||
chain = get_storage_hash_str(token_ids, page_size=2)
|
||||
snapshot = msgspec.structs.replace(snapshot, hash_values=chain)
|
||||
swa = PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
keys=chain[-1:],
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
for peer_swa_pages in (0, 1):
|
||||
with (
|
||||
TemporaryDirectory() as directory,
|
||||
patch.dict(
|
||||
"os.environ", {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": directory}
|
||||
),
|
||||
):
|
||||
ranks, packed = [], []
|
||||
|
||||
def capture(values, reduce_op, groups):
|
||||
self.assertEqual(reduce_op, torch.distributed.ReduceOp.MIN)
|
||||
packed.append(values.clone())
|
||||
|
||||
for rank, swa_pages in enumerate((2, peer_swa_pages)):
|
||||
backend = HiCacheFile(
|
||||
HiCacheStorageConfig(
|
||||
tp_rank=rank,
|
||||
tp_size=2,
|
||||
pp_rank=0,
|
||||
pp_size=1,
|
||||
attn_cp_rank=0,
|
||||
attn_cp_size=1,
|
||||
is_mla_model=False,
|
||||
enable_storage_metrics=False,
|
||||
is_page_first_layout=False,
|
||||
model_name="swa-repair",
|
||||
extra_config={"enable_metadata_cache": False},
|
||||
)
|
||||
)
|
||||
for pool, keys in (
|
||||
(PoolName.KV, chain),
|
||||
(PoolName.SWA, chain[:swa_pages]),
|
||||
):
|
||||
for key in keys:
|
||||
Path(
|
||||
directory,
|
||||
backend._get_component_key(key, pool) + ".bin",
|
||||
).touch()
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("repair", 0), token_ids, pool_transfers=[swa]
|
||||
)
|
||||
controller = SimpleNamespace(
|
||||
storage_backend=backend,
|
||||
page_size=2,
|
||||
prefetch_hits_sync_groups=[],
|
||||
_all_reduce=capture,
|
||||
)
|
||||
_, hit_tokens = HybridCacheController._storage_hit_query(
|
||||
controller, operation
|
||||
)
|
||||
self.assertEqual(operation.all_hash_values, chain)
|
||||
self.assertEqual(
|
||||
operation.query_pool_hit_pages,
|
||||
{
|
||||
PoolName.KV: 2,
|
||||
**({PoolName.SWA: swa_pages} if swa_pages else {}),
|
||||
},
|
||||
)
|
||||
HybridCacheController._sync_prefetch_hit_query(
|
||||
controller, operation, hit_tokens
|
||||
)
|
||||
ranks.append((controller, operation, hit_tokens))
|
||||
|
||||
# Only transport is simulated: reduce the tensors built by the
|
||||
# actual controller, then deliver the same MIN to both ranks.
|
||||
agreed = torch.stack(packed).amin(dim=0)
|
||||
for rank, (controller, operation, hit_tokens) in enumerate(ranks):
|
||||
with self.subTest(peer_swa_pages=peer_swa_pages, rank=rank):
|
||||
controller._all_reduce = lambda values, *_: values.copy_(agreed)
|
||||
operation.storage_hit_count = (
|
||||
HybridCacheController._sync_prefetch_hit_query(
|
||||
controller, operation, hit_tokens
|
||||
)
|
||||
)
|
||||
tree = MagicMock()
|
||||
tree.build_hicache_transfers.return_value = [swa]
|
||||
tree.buffer_backup_pool_keys.return_value = {}
|
||||
pipeline = self._launch_pipeline(tree, [])
|
||||
cache = pipeline._cache
|
||||
cache.host_memory_mode, cache.page_size = "buffer_only", 2
|
||||
cache.components = {
|
||||
ct: SimpleNamespace(buffer_belief_policy=lambda _: None)
|
||||
for ct in (ComponentType.FULL, ComponentType.SWA)
|
||||
}
|
||||
cache._belief_policy = UnifiedRadixCache._belief_policy.__get__(
|
||||
cache
|
||||
)
|
||||
for pool in (PoolName.KV, PoolName.SWA):
|
||||
cache.storage_existence_cache.pool(pool).add(chain)
|
||||
UnifiedRadixCache._invalidate_absent_from_hit_query(
|
||||
cache, operation
|
||||
)
|
||||
intents = pipeline._write_intents(snapshot)
|
||||
# Retain FULL while making only the missing SWA tail
|
||||
# eligible, including when one rank omitted its verdict.
|
||||
self.assertEqual(intents, [(PoolName.SWA, chain[-1:])])
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.pool(
|
||||
PoolName.KV
|
||||
).contains_all(chain)
|
||||
)
|
||||
self.assertEqual(
|
||||
operation.storage_hit_count, peer_swa_pages * 2
|
||||
)
|
||||
self.assertEqual(
|
||||
[
|
||||
cache.storage_existence_cache.pool(
|
||||
PoolName.SWA
|
||||
).contains(h)
|
||||
for h in chain
|
||||
],
|
||||
[i < peer_swa_pages for i in range(2)],
|
||||
)
|
||||
|
||||
# An assumed-stored retry verifies nothing now, so it carries no pool
|
||||
# verdict and re-learns nothing a shortfall heal dropped.
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("assumed", 0),
|
||||
token_ids,
|
||||
pool_transfers=[swa],
|
||||
assume_stored=True,
|
||||
)
|
||||
controller = SimpleNamespace(
|
||||
storage_backend=None,
|
||||
page_size=2,
|
||||
prefetch_hits_sync_groups=[],
|
||||
_all_reduce=lambda values, *_: None,
|
||||
)
|
||||
_, hit_tokens = HybridCacheController._storage_hit_query(controller, operation)
|
||||
operation.storage_hit_count = HybridCacheController._sync_prefetch_hit_query(
|
||||
controller, operation, hit_tokens
|
||||
)
|
||||
self.assertEqual(
|
||||
(operation.storage_hit_count, operation.query_pool_hit_pages), (4, {})
|
||||
)
|
||||
pipeline = self._launch_pipeline(MagicMock(), [])
|
||||
cache = pipeline._cache
|
||||
cache.host_memory_mode, cache.page_size = "buffer_only", 2
|
||||
cache.components = {
|
||||
ct: SimpleNamespace(buffer_belief_policy=lambda _: None)
|
||||
for ct in (ComponentType.FULL, ComponentType.SWA)
|
||||
}
|
||||
cache._belief_policy = UnifiedRadixCache._belief_policy.__get__(cache)
|
||||
cache.storage_existence_cache.pool(PoolName.KV).add(chain)
|
||||
UnifiedRadixCache._invalidate_absent_from_hit_query(cache, operation)
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.pool(PoolName.KV).contains_all(chain)
|
||||
)
|
||||
self.assertFalse(
|
||||
any(
|
||||
cache.storage_existence_cache.pool(PoolName.SWA).contains(h)
|
||||
for h in chain
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _dsv4_specs():
|
||||
return [
|
||||
@@ -120,7 +301,39 @@ class TestBufferModeSidecar(unittest.TestCase):
|
||||
swa_component=self._swa_component(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _backup_snapshot():
|
||||
return BufferBackupSnapshot(
|
||||
node_id=7,
|
||||
parent_node_id=0,
|
||||
parent_is_root=True,
|
||||
parent_last_hash=None,
|
||||
hash_values=["page-0", "page-1"],
|
||||
key=RadixKey(array("q", [1, 2, 3, 4])),
|
||||
prefix_keys=["page-p"],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _launch_pipeline(tree, sidecar_pool_specs):
|
||||
cache = SimpleNamespace(
|
||||
tree_core=tree,
|
||||
components={ComponentType.FULL: SimpleNamespace()},
|
||||
sidecar_pool_specs=sidecar_pool_specs,
|
||||
storage_existence_cache=StorageExistenceCache(),
|
||||
)
|
||||
cache._build_sidecar_transfers = (
|
||||
UnifiedRadixCache._build_sidecar_transfers.__get__(cache)
|
||||
)
|
||||
cache._build_backup_sidecar = UnifiedRadixCache._build_backup_sidecar.__get__(
|
||||
cache
|
||||
)
|
||||
pipeline = BufferModePipeline.__new__(BufferModePipeline)
|
||||
pipeline._cache = cache
|
||||
return pipeline
|
||||
|
||||
def test_write_stages_and_persists_dsv4_full_and_swa_sidecars(self):
|
||||
"""Each pool of a node is a write of its own; every sidecar rides its
|
||||
source pool's write under the source's keys."""
|
||||
page_size = 2
|
||||
device_indices = torch.arange(4, dtype=torch.int64)
|
||||
host_indices = torch.arange(10, 14, dtype=torch.int64)
|
||||
@@ -139,49 +352,68 @@ class TestBufferModeSidecar(unittest.TestCase):
|
||||
)
|
||||
for spec in self._dsv4_specs()
|
||||
]
|
||||
kv_sidecars = [s for s in sidecars if s.indices_from_pool == PoolName.KV]
|
||||
swa_sidecars = [s for s in sidecars if s.indices_from_pool == PoolName.SWA]
|
||||
|
||||
controller = MagicMock()
|
||||
controller.mem_pool_host.anchor_entry.host_pool.shared_allocation_domain = None
|
||||
controller.mem_pool_host.entry_map = {
|
||||
PoolName.SWA: SimpleNamespace(
|
||||
host_pool=SimpleNamespace(page_size=page_size)
|
||||
host_pool=SimpleNamespace(
|
||||
page_size=page_size,
|
||||
size=8,
|
||||
available_size=lambda: 8,
|
||||
free=MagicMock(),
|
||||
)
|
||||
)
|
||||
}
|
||||
controller.mem_pool_host.size = 32
|
||||
controller.mem_pool_host.anchor_entry.host_pool = SimpleNamespace()
|
||||
controller.prefetch_tokens_occupied = 0
|
||||
ack_ids = []
|
||||
|
||||
def _write(device_value, node_id, extra_pools, flush):
|
||||
self.assertEqual(node_id, 7)
|
||||
self.assertFalse(flush)
|
||||
self.assertEqual(
|
||||
[transfer.name for transfer in extra_pools],
|
||||
[PoolName.SWA, *[transfer.name for transfer in sidecars]],
|
||||
)
|
||||
ack_ids.append(node_id)
|
||||
# HostPoolGroup.resolve_host_transfers gives derived pools their
|
||||
# source pool's indices without allocating another staging span.
|
||||
swa.host_indices = swa_host_indices
|
||||
for sidecar in sidecars:
|
||||
if sidecar.indices_from_pool == PoolName.KV:
|
||||
if device_value.numel():
|
||||
self.assertEqual(extra_pools, kv_sidecars)
|
||||
for sidecar in extra_pools:
|
||||
sidecar.host_indices = host_indices
|
||||
sidecar.device_indices = device_value
|
||||
else:
|
||||
sidecar.host_indices = swa_host_indices
|
||||
sidecar.device_indices = swa_device_indices
|
||||
return host_indices
|
||||
return host_indices
|
||||
self.assertEqual(extra_pools, [swa, *swa_sidecars])
|
||||
swa.host_indices = swa_host_indices
|
||||
for sidecar in swa_sidecars:
|
||||
sidecar.host_indices = swa_host_indices
|
||||
sidecar.device_indices = swa_device_indices
|
||||
return device_value
|
||||
|
||||
controller.write.side_effect = _write
|
||||
controller.write_storage.return_value = 99
|
||||
controller.write_storage.side_effect = [98, 99]
|
||||
|
||||
cache = MagicMock()
|
||||
cache.cache_controller = controller
|
||||
cache.page_size = page_size
|
||||
cache.enable_storage = True
|
||||
cache.enable_storage_metrics = False
|
||||
cache.hicache_storage_pass_prefix_keys = True
|
||||
cache.components = {ComponentType.FULL: MagicMock()}
|
||||
cache.sidecar_pool_specs = self._dsv4_specs()
|
||||
cache._build_backup_sidecar.return_value = sidecars
|
||||
cache.storage_existence_cache = StorageExistenceCache()
|
||||
|
||||
pipeline = BufferModePipeline.__new__(BufferModePipeline)
|
||||
pipeline._cache = cache
|
||||
pipeline.ongoing_write_through = {}
|
||||
pipeline.ongoing_backup = {}
|
||||
pipeline.inflight_backup_hashes = {}
|
||||
pipeline.write_staged_tokens_ = 0
|
||||
pipeline.write_backlog_tokens_ = len(device_indices)
|
||||
pipeline._swa_window_pages = 1
|
||||
pipeline.write_backlog_cap = 8
|
||||
pipeline.reset()
|
||||
pipeline._build_backup_transfers = lambda node_id, *, kv_only=False: (
|
||||
device_indices,
|
||||
{ComponentType.SWA: [swa]},
|
||||
None,
|
||||
)
|
||||
|
||||
hashes = ["page-0", "page-1"]
|
||||
snapshot = BufferBackupSnapshot(
|
||||
@@ -191,24 +423,51 @@ class TestBufferModeSidecar(unittest.TestCase):
|
||||
parent_last_hash=None,
|
||||
hash_values=hashes,
|
||||
key=RadixKey(array("q", [1, 2, 3, 4])),
|
||||
prefix_keys=None,
|
||||
prefix_keys=["page-p"],
|
||||
)
|
||||
intent = _UnifiedBackupIntent(snapshot=snapshot)
|
||||
|
||||
self.assertTrue(
|
||||
pipeline._stage_backup_intent(
|
||||
intent,
|
||||
device_indices,
|
||||
comp_xfers={ComponentType.SWA: [swa]},
|
||||
)
|
||||
tree = cache.tree_core
|
||||
tree.snapshot_buffer_backup.return_value = snapshot
|
||||
tree.validate_buffer_backup.return_value = BufferBackupState(0, True, None)
|
||||
tree.buffer_backup_pool_keys.return_value = {
|
||||
ComponentType.SWA: {PoolName.SWA: hashes[-1:]}
|
||||
}
|
||||
pipeline.enqueue_backup_intent(7)
|
||||
self.assertEqual(
|
||||
[intent.pool for intent in pipeline.pending_write_queue],
|
||||
[PoolName.KV, PoolName.SWA],
|
||||
)
|
||||
self.assertEqual(pipeline.ongoing_write_through[7].aux_xfers, [swa, *sidecars])
|
||||
pipeline.flush_pending_writes()
|
||||
self.assertFalse(pipeline.pending_write_queue)
|
||||
kv_ack, swa_ack = ack_ids
|
||||
self.assertEqual(pipeline.ongoing_write_through[kv_ack].aux_xfers, kv_sidecars)
|
||||
self.assertEqual(
|
||||
pipeline.ongoing_write_through[swa_ack].aux_xfers, [swa, *swa_sidecars]
|
||||
)
|
||||
expected_keys = {PoolName.KV: hashes, PoolName.SWA: hashes[-1:]}
|
||||
for spec in self._dsv4_specs():
|
||||
expected_keys[spec.pool_name] = expected_keys[spec.indices_from_pool]
|
||||
for pool, keys in expected_keys.items():
|
||||
# In flight (covered for admission) but not yet believed stored.
|
||||
pool_beliefs = cache.storage_existence_cache.pool(pool)
|
||||
self.assertTrue(pool_beliefs.covered(keys))
|
||||
self.assertFalse(pool_beliefs.contains_all(keys))
|
||||
self.assertEqual(pipeline.write_backlog_tokens_, 0)
|
||||
|
||||
pipeline.finish_backup_ack(7)
|
||||
pipeline.finish_backup_ack(kv_ack)
|
||||
pipeline.finish_backup_ack(swa_ack)
|
||||
|
||||
kv_call, swa_call = controller.write_storage.call_args_list
|
||||
self.assertTrue(torch.equal(kv_call.args[0], host_indices))
|
||||
self.assertEqual(kv_call.args[2], hashes)
|
||||
self.assertEqual(kv_call.args[3], ["page-p"])
|
||||
# No KV page in the SWA write; the window's keys ride the transfer.
|
||||
self.assertEqual(swa_call.args[0].numel(), 0)
|
||||
self.assertEqual(swa_call.args[2], [])
|
||||
self.assertEqual(swa_call.args[3], ["page-p"])
|
||||
storage_transfers = {
|
||||
transfer.name: transfer
|
||||
for transfer in controller.write_storage.call_args.kwargs["extra_pools"]
|
||||
for call in (kv_call, swa_call)
|
||||
for transfer in call.kwargs["extra_pools"]
|
||||
}
|
||||
self.assertEqual(storage_transfers[PoolName.SWA].keys, [hashes[-1]])
|
||||
self.assertTrue(
|
||||
@@ -216,14 +475,16 @@ class TestBufferModeSidecar(unittest.TestCase):
|
||||
)
|
||||
for spec in self._dsv4_specs():
|
||||
storage_sidecar = storage_transfers[spec.pool_name]
|
||||
expected_keys = (
|
||||
hashes if spec.indices_from_pool == PoolName.KV else [hashes[-1]]
|
||||
)
|
||||
self.assertEqual(storage_sidecar.keys, expected_keys)
|
||||
self.assertEqual(storage_sidecar.keys, expected_keys[spec.pool_name])
|
||||
self.assertEqual(storage_sidecar.hit_policy, spec.hit_policy)
|
||||
self.assertEqual(storage_sidecar.indices_from_pool, spec.indices_from_pool)
|
||||
self.assertIsNone(storage_sidecar.host_indices)
|
||||
self.assertIn(99, pipeline.ongoing_backup)
|
||||
self.assertEqual(set(pipeline.ongoing_backup), {98, 99})
|
||||
pipeline.finish_storage_write_ack(98)
|
||||
self.assertTrue(pipeline.backup_pending(7))
|
||||
pipeline.finish_storage_write_ack(99)
|
||||
self.assertFalse(pipeline.backup_pending(7))
|
||||
self.assertEqual(pipeline.write_staged_tokens_, 0)
|
||||
|
||||
def test_completed_prefetch_keeps_dsv4_full_and_swa_sidecars_for_h2d(self):
|
||||
swa = PoolTransfer(
|
||||
|
||||
@@ -62,6 +62,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
MHATokenToKVPool,
|
||||
ReqToTokenPool,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host import PoolEntry
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
@@ -77,11 +78,13 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
SWARebuild,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
BASE_COMPONENT_TYPE,
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
TreeComponent,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components.full import FullComponent
|
||||
from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment
|
||||
from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
|
||||
_TREE_CORE_REGISTRY,
|
||||
@@ -238,6 +241,46 @@ class _FakeFullComponent(TreeComponent):
|
||||
pass
|
||||
|
||||
|
||||
class _ExtraPoolFullComponent(FullComponent):
|
||||
"""FULL component that also stores every page under a pool of its own,
|
||||
staged through the anchor's host pool."""
|
||||
|
||||
EXTRA_POOL = PoolName.DRAFT
|
||||
|
||||
def buffer_mode_host_pool_entries(self, host_pool_group):
|
||||
anchor = host_pool_group.anchor_entry
|
||||
return [
|
||||
PoolEntry(
|
||||
name=self.EXTRA_POOL,
|
||||
host_pool=anchor.host_pool,
|
||||
device_pool=anchor.device_pool,
|
||||
layer_mapper=anchor.layer_mapper,
|
||||
)
|
||||
]
|
||||
|
||||
def buffer_backup_keys(self, node, hash_values):
|
||||
return {self.EXTRA_POOL: list(hash_values)}
|
||||
|
||||
def build_hicache_transfers(self, node, phase, **kwargs):
|
||||
if phase == CacheTransferPhase.BACKUP_HOST:
|
||||
# Two transfers of the pool, each keyed by its own pages: the
|
||||
# storage write must keep the keys per transfer.
|
||||
value = node.component_data[BASE_COMPONENT_TYPE].value
|
||||
keys = list(node.hash_value)
|
||||
split = len(keys) // 2
|
||||
page_size = self.tree_core.page_size
|
||||
halves = [
|
||||
(value[: split * page_size], keys[:split]),
|
||||
(value[split * page_size :], keys[split:]),
|
||||
]
|
||||
return [
|
||||
PoolTransfer(name=self.EXTRA_POOL, device_indices=rows, keys=part)
|
||||
for rows, part in halves
|
||||
if part
|
||||
]
|
||||
return super().build_hicache_transfers(node, phase, **kwargs)
|
||||
|
||||
|
||||
def _drop_hicache_atexit_pin(cache):
|
||||
"""init_hicache registers ``cache.shutdown`` with atexit, which pins the
|
||||
fixture's device and host pools until process exit; hundreds of HiCache
|
||||
@@ -468,6 +511,9 @@ def build_fixture(
|
||||
enable_session_radix_cache: bool = False,
|
||||
tree_page_size: Optional[int] = None,
|
||||
mamba_cache_chunk_size: Optional[int] = None,
|
||||
component_registry_override: Optional[
|
||||
dict[ComponentType, type[TreeComponent]]
|
||||
] = None,
|
||||
tree_core_backend: Optional[str] = None,
|
||||
):
|
||||
"""Create (tree, allocator, req_to_token_pool) from a CacheConfig.
|
||||
@@ -602,6 +648,7 @@ def build_fixture(
|
||||
enable_session_radix_cache=enable_session_radix_cache,
|
||||
eviction_policy=cfg.eviction_policy,
|
||||
is_eagle=cfg.is_eagle,
|
||||
component_registry_override=component_registry_override,
|
||||
)
|
||||
requested_backend = tree_core_backend or _selected_tree_core_test_backend()
|
||||
selected_backend = resolve_tree_core_backend(requested_backend, cache_init_params)
|
||||
@@ -1135,13 +1182,18 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase):
|
||||
# every in-flight backup.
|
||||
lock_params = cache.inc_lock_ref(leaf_id).to_dec_params()
|
||||
pipeline = BufferModePipeline.__new__(BufferModePipeline)
|
||||
keys = list(snapshot.hash_values)
|
||||
# Entries are keyed by the write's ack id, not by the node.
|
||||
pipeline.ongoing_write_through = {
|
||||
leaf_id: _UnifiedBufferBackupEntry(
|
||||
intent=_UnifiedBackupIntent(snapshot=snapshot),
|
||||
-1: _UnifiedBufferBackupEntry(
|
||||
intent=_UnifiedBackupIntent(
|
||||
snapshot=snapshot, pool=PoolName.KV, keys=keys
|
||||
),
|
||||
host_indices=torch.empty(0, dtype=torch.int64),
|
||||
aux_xfers=[],
|
||||
lock_params=lock_params,
|
||||
occupied_units=0,
|
||||
keys_by_pool={PoolName.KV: keys},
|
||||
)
|
||||
}
|
||||
cache.buffer_pipeline = pipeline
|
||||
@@ -4031,8 +4083,14 @@ class UnifiedRadixCacheSuite:
|
||||
pipeline = cache.buffer_pipeline
|
||||
controller = cache.cache_controller
|
||||
_write_backup(cache, leaf)
|
||||
node_ids = [intent.snapshot.node_id for intent in pipeline.pending_write_queue]
|
||||
# One intent per pool of every node on the path.
|
||||
intents = list(pipeline.pending_write_queue)
|
||||
node_ids = sorted({intent.snapshot.node_id for intent in intents})
|
||||
self.assertGreaterEqual(len(node_ids), 2)
|
||||
self.assertEqual(
|
||||
pipeline.inflight_backup_pools[leaf],
|
||||
{intent.pool for intent in intents if intent.snapshot.node_id == leaf},
|
||||
)
|
||||
|
||||
device_allocators = [allocator]
|
||||
if self.cfg.has_swa:
|
||||
@@ -4064,9 +4122,19 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertFalse(pipeline.pending_write_queue)
|
||||
self.assertFalse(controller.write_queue)
|
||||
self.assertEqual(len(controller.ack_write_queue), 1)
|
||||
# Every intent is a write of its own, acked under its own id.
|
||||
ack = controller.ack_write_queue[0]
|
||||
self.assertEqual(ack.node_ids, node_ids)
|
||||
self.assertEqual(set(pipeline.ongoing_write_through), set(node_ids))
|
||||
self.assertEqual(len(ack.node_ids), len(intents))
|
||||
self.assertEqual(list(ack.node_ids), list(pipeline.ongoing_write_through))
|
||||
self.assertEqual(
|
||||
sorted(
|
||||
{
|
||||
entry.intent.snapshot.node_id
|
||||
for entry in pipeline.ongoing_write_through.values()
|
||||
}
|
||||
),
|
||||
node_ids,
|
||||
)
|
||||
staged_avail = self._host_avail_sizes(cache)
|
||||
self.assertNotEqual(staged_avail, avail0)
|
||||
|
||||
@@ -4076,7 +4144,7 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertGreater(_device_lock_ref(cache, node_id, ComponentType.FULL), 0)
|
||||
cache.writing_check(finish_count=1)
|
||||
self.assertFalse(pipeline.ongoing_write_through)
|
||||
self.assertEqual(len(pipeline.ongoing_backup), len(node_ids))
|
||||
self.assertEqual(len(pipeline.ongoing_backup), len(intents))
|
||||
for node_id in node_ids:
|
||||
self.assertEqual(_device_lock_ref(cache, node_id, ComponentType.FULL), 0)
|
||||
# Each storage write still owns its staging until its separate ACK.
|
||||
@@ -4103,13 +4171,259 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
for n in chain:
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.contains_all(
|
||||
PoolName.KV, cache.tree_core.get_hash_values(n)
|
||||
cache.storage_existence_cache.pool(PoolName.KV).contains_all(
|
||||
cache.tree_core.get_hash_values(n)
|
||||
)
|
||||
)
|
||||
# Re-hit absorbed by the (FULL-focused) belief skip.
|
||||
for transfer in _aux_storage_key_transfers(cache, n) or ():
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.pool(transfer.name).contains_all(
|
||||
transfer.keys
|
||||
)
|
||||
)
|
||||
# Re-hit absorbed when every pool is believed present.
|
||||
_write_backup(cache, leaf)
|
||||
self.assertNotIn(leaf, cache.buffer_pipeline.inflight_backup_node_ids)
|
||||
|
||||
if self.cfg.has_swa:
|
||||
# Only the SWA window dropped: the next insert queues the SWA pool
|
||||
# alone. A private SWA arena's write proceeds even with FULL staging
|
||||
# at the live cap (shared or anchor-aliased layouts keep the gate).
|
||||
swa_transfer = _aux_storage_key_transfers(cache, leaf)[0]
|
||||
cache.storage_existence_cache.pool(PoolName.SWA).invalidate_beyond(
|
||||
swa_transfer.keys, keep_pages=0
|
||||
)
|
||||
host_group = controller.mem_pool_host
|
||||
private_swa = (
|
||||
pipeline._shared_host_domain() is None
|
||||
and host_group.entry_map[PoolName.SWA].host_pool
|
||||
is not host_group.anchor_entry.host_pool
|
||||
)
|
||||
staged_before = pipeline.write_staged_tokens_
|
||||
if private_swa:
|
||||
pipeline.write_staged_tokens_ = host_group.size
|
||||
with (
|
||||
mock.patch.object(controller, "write", wraps=controller.write) as stage,
|
||||
mock.patch.object(
|
||||
controller, "write_storage", wraps=controller.write_storage
|
||||
) as write_storage,
|
||||
mock.patch.object(
|
||||
controller, "page_set_func", wraps=controller.page_set_func
|
||||
) as write_full,
|
||||
mock.patch.object(
|
||||
pipeline, "_write_intents", wraps=pipeline._write_intents
|
||||
) as admissions,
|
||||
):
|
||||
# The fixture parks the stock trigger; arm it here.
|
||||
cache.write_through_threshold = 1
|
||||
self._insert(cache, allocator, req_to_token_pool, seq_ab)
|
||||
self.assertEqual(
|
||||
[
|
||||
(i.snapshot.node_id, i.pool)
|
||||
for i in pipeline.pending_write_queue
|
||||
],
|
||||
[(leaf, PoolName.SWA)],
|
||||
)
|
||||
# One admission per node per walk.
|
||||
self.assertEqual(admissions.call_count, len(node_ids))
|
||||
self.assertTrue(cache.buffer_backup_pending(leaf))
|
||||
self._pump_hicache_until(
|
||||
cache,
|
||||
lambda: (
|
||||
not pipeline.inflight_backup_node_ids
|
||||
and not pipeline.ongoing_backup
|
||||
),
|
||||
"selective buffer backup did not drain",
|
||||
)
|
||||
self.assertFalse(cache.buffer_backup_pending(leaf))
|
||||
if private_swa:
|
||||
pipeline.write_staged_tokens_ -= host_group.size - staged_before
|
||||
self.assertTrue(stage.called)
|
||||
self.assertTrue(
|
||||
all(call.args[0].numel() == 0 for call in stage.call_args_list)
|
||||
)
|
||||
# SWA writes carry no KV page; the window's keys ride the transfer.
|
||||
self.assertTrue(write_storage.called)
|
||||
snapshot = cache.tree_core.snapshot_buffer_backup(
|
||||
leaf, cache.hicache_storage_pass_prefix_keys
|
||||
)
|
||||
for call in write_storage.call_args_list:
|
||||
host_indices, _, hash_value, prefix_keys = call.args[:4]
|
||||
self.assertEqual(host_indices.numel(), 0)
|
||||
self.assertEqual(hash_value, [])
|
||||
transfers = call.kwargs["extra_pools"]
|
||||
self.assertEqual(
|
||||
[t.name for t in transfers if t.indices_from_pool is None],
|
||||
[PoolName.SWA],
|
||||
)
|
||||
self.assertEqual(prefix_keys, snapshot.prefix_keys)
|
||||
write_full.assert_not_called()
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.pool(PoolName.SWA).contains_all(
|
||||
swa_transfer.keys
|
||||
)
|
||||
)
|
||||
self.assertEqual(self._host_avail_sizes(cache), avail0)
|
||||
cache.sanity_check()
|
||||
|
||||
@staticmethod
|
||||
def _writes_by_pool(write_storage, extra):
|
||||
"""Key tuples of the KV writes and of the ``extra`` pool's writes
|
||||
among the storage writes recorded by ``write_storage``."""
|
||||
kv_writes, extra_writes = set(), set()
|
||||
for call in write_storage.call_args_list:
|
||||
host_indices, _, hash_value = call.args[:3]
|
||||
transfers = call.kwargs.get("extra_pools") or []
|
||||
names = [transfer.name for transfer in transfers]
|
||||
if hash_value:
|
||||
assert extra not in names
|
||||
kv_writes.add(tuple(hash_value))
|
||||
elif extra in names:
|
||||
# The pool's transfers reach storage as one contiguous span.
|
||||
assert host_indices.numel() == 0 and names == [extra]
|
||||
extra_writes.add(tuple(transfers[0].keys))
|
||||
return kv_writes, extra_writes
|
||||
|
||||
def test_buffer_only_writes_component_pools_independently(self):
|
||||
"""A component's own pool (aliased host entry, per-pool keys,
|
||||
BACKUP_HOST transfers) is written, believed and re-written
|
||||
independently of the KV pool."""
|
||||
self._skip_unsupported_hicache_test()
|
||||
storage_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, storage_dir, ignore_errors=True)
|
||||
|
||||
# Component buffer-mode hooks are Python-core only.
|
||||
cache, allocator, req_to_token_pool = build_fixture(
|
||||
self.cfg,
|
||||
component_registry_override={ComponentType.FULL: _ExtraPoolFullComponent},
|
||||
tree_core_backend="python",
|
||||
)
|
||||
self._init_buffer_hicache(cache, storage_dir)
|
||||
extra = _ExtraPoolFullComponent.EXTRA_POOL
|
||||
controller = cache.cache_controller
|
||||
self.assertIn(extra, controller.mem_pool_host.entry_map)
|
||||
avail0 = self._host_avail_sizes(cache)
|
||||
|
||||
seq = self._buffer_swa_seq()
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
leaf = cache.match_prefix(
|
||||
MatchPrefixParams(key=RadixKey(array("q", seq)))
|
||||
).last_device_node
|
||||
chain = self._path_chain(cache, leaf)
|
||||
pipeline = cache.buffer_pipeline
|
||||
expected_intents = []
|
||||
expected_backlog = 0
|
||||
for node_id in chain:
|
||||
snapshot = cache.tree_core.snapshot_buffer_backup(
|
||||
node_id, cache.hicache_storage_pass_prefix_keys
|
||||
)
|
||||
expected_backlog += len(snapshot.key)
|
||||
expected_intents.extend(
|
||||
(node_id, pool) for pool, _ in pipeline._write_intents(snapshot)
|
||||
)
|
||||
pipeline.write_backlog_cap = expected_backlog
|
||||
if self.cfg.has_swa:
|
||||
self.assertEqual(
|
||||
{pool for node_id, pool in expected_intents if node_id == leaf},
|
||||
{PoolName.KV, PoolName.DRAFT, PoolName.SWA},
|
||||
)
|
||||
self.assertFalse(cache.buffer_backup_pending(leaf))
|
||||
with mock.patch.object(
|
||||
controller, "write_storage", wraps=controller.write_storage
|
||||
) as write_storage:
|
||||
_write_backup(cache, leaf)
|
||||
self.assertEqual(
|
||||
[
|
||||
(intent.snapshot.node_id, intent.pool)
|
||||
for intent in pipeline.pending_write_queue
|
||||
],
|
||||
expected_intents,
|
||||
)
|
||||
self.assertEqual(pipeline.write_backlog_tokens_, expected_backlog)
|
||||
self.assertEqual(pipeline._backlog_cap_hits, 0)
|
||||
self.assertTrue(cache.buffer_backup_pending(leaf))
|
||||
self._pump_hicache_until(
|
||||
cache,
|
||||
lambda: (
|
||||
not cache.buffer_pipeline.inflight_backup_node_ids
|
||||
and not cache.buffer_pipeline.ongoing_backup
|
||||
),
|
||||
"buffer backup pipeline did not drain",
|
||||
)
|
||||
self.assertFalse(cache.buffer_backup_pending(leaf))
|
||||
# Each pool of each node is a write of its own: the KV chain with its
|
||||
# pages, the component's pool with no KV rows and its transfers
|
||||
# keyed by their own pages.
|
||||
expected = {tuple(cache.tree_core.get_hash_values(n)) for n in chain}
|
||||
kv_writes, extra_writes = self._writes_by_pool(write_storage, extra)
|
||||
self.assertEqual(kv_writes, expected)
|
||||
self.assertEqual(extra_writes, expected)
|
||||
|
||||
page_hashes = self._all_page_hashes(cache, leaf)
|
||||
self.assertEqual(
|
||||
self._storage_exists_count(
|
||||
cache, page_hashes, [PoolTransfer(name=extra, keys=page_hashes)]
|
||||
),
|
||||
len(page_hashes),
|
||||
)
|
||||
for n in chain:
|
||||
keys = cache.tree_core.get_hash_values(n)
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.pool(PoolName.KV).contains_all(keys)
|
||||
)
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.pool(extra).contains_all(keys)
|
||||
)
|
||||
self.assertEqual(self._host_avail_sizes(cache), avail0)
|
||||
|
||||
# Only the component's pool dropped from the beliefs: its requested
|
||||
# backup writes that pool alone, staged through the anchor's host
|
||||
# pool with no KV rows.
|
||||
for n in chain:
|
||||
cache.storage_existence_cache.pool(extra).invalidate_beyond(
|
||||
cache.tree_core.get_hash_values(n), keep_pages=0
|
||||
)
|
||||
with (
|
||||
mock.patch.object(controller, "write", wraps=controller.write) as stage,
|
||||
mock.patch.object(
|
||||
controller, "write_storage", wraps=controller.write_storage
|
||||
) as write_storage,
|
||||
mock.patch.object(
|
||||
controller, "page_set_func", wraps=controller.page_set_func
|
||||
) as write_full,
|
||||
):
|
||||
cache.request_buffer_backup(leaf)
|
||||
intents = list(pipeline.pending_write_queue)
|
||||
self.assertEqual(
|
||||
[(intent.snapshot.node_id, intent.pool) for intent in intents],
|
||||
[(n, extra) for n in chain],
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cache,
|
||||
lambda: (
|
||||
not pipeline.inflight_backup_node_ids
|
||||
and not pipeline.ongoing_backup
|
||||
),
|
||||
"component-pool backup did not drain",
|
||||
)
|
||||
self.assertTrue(stage.called)
|
||||
self.assertTrue(all(call.args[0].numel() == 0 for call in stage.call_args_list))
|
||||
self.assertEqual(len(write_storage.call_args_list), len(chain))
|
||||
kv_writes, extra_writes = self._writes_by_pool(write_storage, extra)
|
||||
self.assertEqual(kv_writes, set())
|
||||
self.assertEqual(extra_writes, expected)
|
||||
write_full.assert_not_called()
|
||||
for n in chain:
|
||||
self.assertTrue(
|
||||
cache.storage_existence_cache.pool(extra).contains_all(
|
||||
cache.tree_core.get_hash_values(n)
|
||||
)
|
||||
)
|
||||
self.assertEqual(self._host_avail_sizes(cache), avail0)
|
||||
|
||||
# Everything believed stored: a requested backup is absorbed.
|
||||
cache.request_buffer_backup(leaf)
|
||||
self.assertNotIn(leaf, pipeline.inflight_backup_node_ids)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_buffer_only_read_path_roundtrip(self):
|
||||
@@ -4228,10 +4542,18 @@ class UnifiedRadixCacheSuite:
|
||||
for component_type in cons.tree_components:
|
||||
self.assertIsNone(_host_value(cons, cur, component_type))
|
||||
self.assertTrue(
|
||||
cons.storage_existence_cache.contains_all(
|
||||
PoolName.KV, self._all_page_hashes(cons, leaf)
|
||||
cons.storage_existence_cache.pool(PoolName.KV).contains_all(
|
||||
self._all_page_hashes(cons, leaf)
|
||||
)
|
||||
)
|
||||
# The fetch is evidence for every pool it delivered, so the
|
||||
# consumer's own re-write of the span is skipped pool by pool.
|
||||
for transfer in _aux_storage_key_transfers(cons, leaf) or ():
|
||||
self.assertTrue(
|
||||
cons.storage_existence_cache.pool(transfer.name).contains_all(
|
||||
transfer.keys
|
||||
)
|
||||
)
|
||||
loaded_k, loaded_v = self._snapshot_full_kv(cons_alloc, mc.device_indices)
|
||||
self.assertTrue(torch.equal(loaded_k, expected_k))
|
||||
self.assertTrue(torch.equal(loaded_v, expected_v))
|
||||
|
||||
Reference in New Issue
Block a user