[HiCache] Manage buffer-mode backups per storage pool (#42633)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-06 16:45:05 -07:00
committed by GitHub
co-authored by merrymercy xiezhq-hermann
parent fd62ee84f7
commit 08aad0c196
11 changed files with 1495 additions and 415 deletions
+13 -8
View File
@@ -1295,15 +1295,9 @@ class HiCacheController:
)
hash_value, storage_hit_count = [], 0
storage_hit_count_tensor = torch.tensor(
storage_hit_count, dtype=torch.int
storage_hit_count = self._sync_prefetch_hit_query(
operation, storage_hit_count
)
self._all_reduce(
storage_hit_count_tensor,
torch.distributed.ReduceOp.MIN,
self.prefetch_hits_sync_groups,
)
storage_hit_count = storage_hit_count_tensor.item()
# Record the TP-synced hit count; the scheduler thread decides
# at drain time whether to revoke (below threshold) or allocate.
@@ -1316,6 +1310,17 @@ class HiCacheController:
except Empty:
continue
def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int:
"""Rank-reduce a hit query (MIN) so every rank agrees on the usable
prefix. Runs for every operation, hit or miss."""
hit_count = torch.tensor(storage_hit_count, dtype=torch.int)
self._all_reduce(
hit_count,
torch.distributed.ReduceOp.MIN,
self.prefetch_hits_sync_groups,
)
return int(hit_count.item())
def write_storage(
self,
host_indices: torch.Tensor,
@@ -81,23 +81,28 @@ def _minimum_full_transfer_tokens(
class _UnifiedBackupIntent(msgspec.Struct):
"""Buffer-mode backup intent, unpinned while queued.
"""One pool of one node queued for a buffer-mode write (unpinned while
queued); a derived sidecar rides its source pool's intent.
Snapshots node identity at enqueue time: a split rewrites the node's
key/hash in place while these copies stay intact, so a key-length change
detects a split and a missing FULL device value detects eviction
(``_validate_backup_intent``).
(``_validate_backup_intent``). ``keys`` is the admission view (it sizes
drop metrics); the D2H launch re-reads rows and keys off the node.
"""
snapshot: BufferBackupSnapshot
pool: PoolName
keys: list[str]
class _UnifiedBufferBackupEntry(msgspec.Struct):
"""A buffer-mode backup after its D2H launch: intent + staging slots.
``host_indices`` are FULL-pool staging slots; ``aux_xfers`` carry the
staged aux-pool slots (e.g. the SWA window). All are freed at the
storage-write ack — host memory is never retained as a cache tier.
``host_indices`` are the KV staging slots (empty for another pool's
intent); ``aux_xfers`` the staged transfers and riding sidecars; all are
freed at the storage-write ack (host memory is never a cache tier).
``keys_by_pool`` are the keys the write stores, learned at the ack.
"""
intent: _UnifiedBackupIntent
@@ -105,6 +110,7 @@ class _UnifiedBufferBackupEntry(msgspec.Struct):
aux_xfers: list[PoolTransfer]
lock_params: DecLockRefParams
occupied_units: int
keys_by_pool: dict[PoolName, list[str]]
class _StagedPrefetch(msgspec.Struct):
@@ -147,24 +153,6 @@ class _AnchorLock(msgspec.Struct):
tokens: int
def _track_content_refs(refs: dict[str, int], hash_values: list[str]) -> None:
"""Add one content ref per page hash (at D2H launch). Refcounted,
not a flag: several launched entries can carry the same content
(duplicate staging of republished spans)."""
for h in hash_values:
refs[h] = refs.get(h, 0) + 1
def _untrack_content_refs(refs: dict[str, int], hash_values: list[str]) -> None:
"""Drop one content ref per page hash (at storage-ack)."""
for h in hash_values:
n = refs.get(h, 0) - 1
if n <= 0:
refs.pop(h, None)
else:
refs[h] = n
def validate_buffer_only_stack(
sidecar_pool_specs: list[SidecarPoolSpec],
host_pool_group: HostPoolGroup,
@@ -270,8 +258,8 @@ class BufferModePipeline:
# a host pool (0 = KV-only: no trailing window staged). Static after
# pool assembly.
self._swa_window_pages = swa_window_pages
# Metadata-only pending-write backlog cap; beyond it new intents
# are dropped at admission (re-trigger on a later hit).
# Metadata-only pending-write backlog cap, measured in unique FULL
# snapshot spans; beyond it new spans are dropped at admission.
self.write_backlog_cap = write_backlog_cap
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -318,19 +306,18 @@ class BufferModePipeline:
] = {}
self.staged_prefetches: dict[CacheRequestHandle, _StagedPrefetch] = {}
self.ongoing_buffer_load_back: dict[int, _OngoingBufferLoadBack] = {}
# Backup pipeline: FIFO intents awaiting a D2H slot, node ids
# anywhere in flight (dedupes re-triggers), and a content refcount
# of every page hash between D2H launch and storage-ack — admission
# skips content covered by beliefs + launched writes.
# FIFO intents (one per pool of a node) awaiting a D2H slot, and the
# pools in flight per node (admission to storage ack).
self.pending_write_queue: deque[_UnifiedBackupIntent] = deque()
self.inflight_backup_node_ids: set[int] = set()
self.inflight_backup_hashes: dict[str, int] = {}
# Backups between D2H launch and D2H ack (keyed by node id), then
# between storage-write launch and storage ack (keyed by operation
# id). Mirrors the cache-mode ongoing_write_through/ongoing_backup
# stages, with buffer entries.
self._queued_span_refs: dict[tuple[NodeId, int], int] = {}
self.inflight_backup_pools: dict[NodeId, set[PoolName]] = {}
# Nodes admitted during the current insert walk (None outside one).
self._walk_admitted: Optional[set[NodeId]] = None
# Launched writes: D2H in flight (by ack id), then the storage write
# in flight (by operation id).
self.ongoing_write_through: dict[int, _UnifiedBufferBackupEntry] = {}
self.ongoing_backup: dict[int, _UnifiedBufferBackupEntry] = {}
self._backup_ack_seq = 0
self.write_staged_tokens_ = 0
self.write_backlog_tokens_ = 0
self._backlog_cap_hits = 0
@@ -391,7 +378,9 @@ class BufferModePipeline:
cc = self._cache.cache_controller
anchor = cc.mem_pool_host.anchor_entry.host_pool
if self._shared_host_domain() is None:
return kv_tokens
# Aux transfers staged through the anchor's own host pool (an
# entry aliasing it) occupy anchor tokens as well.
return kv_tokens + self._anchor_pool_aux_tokens(aux_xfers or [])
num_bytes = sum(
tokens * pool.size_per_token
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
@@ -420,6 +409,18 @@ class BufferModePipeline:
)
return requests
@property
def inflight_backup_node_ids(self):
"""Nodes with a write of any pool queued, staged or awaiting its
storage ack."""
return self.inflight_backup_pools.keys()
def backup_pending(self, node_id: NodeId) -> bool:
"""A write of ``node_id`` is queued, staged or awaiting its storage
ack. A component that drops rows a pending write still reads asks
here first; the pipeline pins the node only from D2H launch to ack."""
return node_id in self.inflight_backup_pools
def is_idle(self) -> bool:
"""No queued, staged, or in-flight buffer-mode work or anchor pins."""
return not (
@@ -444,19 +445,37 @@ class BufferModePipeline:
# ---- backup pipeline (device -> staging -> storage) ----
def _backup_parent_covered(self, state: BufferBackupState) -> bool:
def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int:
"""Staging tokens aux transfers hold in the anchor's host pool."""
group = self._cache.cache_controller.mem_pool_host
anchor_pool = group.anchor_entry.host_pool
tokens = 0
for t in aux_xfers:
if t.indices_from_pool is not None:
continue
entry = group.entry_map.get(t.name)
if entry is not None and entry.host_pool is anchor_pool:
tokens += self._transfer_tokens(t)
return tokens
def _backup_parent_covered(self, state: BufferBackupState, node_id: NodeId) -> bool:
"""Only admit a node whose parent is stored/in-flight: writing above
a dropped parent creates a permanent longest-prefix hole."""
if (
state.parent_is_root
or state.parent_node_id in self.inflight_backup_node_ids
a dropped parent creates a permanent longest-prefix hole. A component
may vouch for the parent through a pool of its own."""
if state.parent_is_root or PoolName.KV in self.inflight_backup_pools.get(
state.parent_node_id, ()
):
return True
return (
if (
state.parent_last_hash is not None
and self._cache.storage_existence_cache.contains(
PoolName.KV, state.parent_last_hash
and self._cache.storage_existence_cache.pool(PoolName.KV).contains(
state.parent_last_hash
)
):
return True
return any(
component.buffer_backup_parent_covered(state.parent_node_id, node_id)
for component in self._cache.components.values()
)
def _log_backup_dropped(self, num_tokens: int) -> None:
@@ -464,129 +483,229 @@ class BufferModePipeline:
if cache.enable_storage_metrics and cache.storage_metrics_collector is not None:
cache.storage_metrics_collector.log_backup_dropped_tokens(num_tokens)
def _pool_covered(self, pool: PoolName, keys: list[str]) -> bool:
"""Beliefs plus content past a D2H launch (republished content must
not re-write while the original write drains)."""
return self._cache.storage_existence_cache.pool(pool).covered(keys)
def _transfer_keys(
self, transfer: PoolTransfer, hash_values: list[str]
) -> Optional[list[str]]:
"""The builder's keys, else the chain's trailing pages, one per staged
page (the SWA convention); None when the transfer stages nothing."""
if transfer.keys is not None:
return list(transfer.keys)
if transfer.indices_from_pool is not None or transfer.device_indices is None:
return None
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
return None
num_pages = len(transfer.device_indices) // entry.host_pool.page_size
if num_pages == 0 or num_pages > len(hash_values):
return None
return list(hash_values[-num_pages:])
def _backup_pool_keys(
self, snapshot: BufferBackupSnapshot
) -> dict[PoolName, list[str]]:
"""Admission view of the keys a backup writes per pool: the KV chain,
each aux component's BACKUP_HOST transfers, component overrides on
top, every sidecar under its source's keys."""
hash_values = snapshot.hash_values
tree_core = self._cache.tree_core
pool_keys: dict[PoolName, list[str]] = {PoolName.KV: list(hash_values)}
overrides = tree_core.buffer_backup_pool_keys(snapshot.node_id, hash_values)
for ct in self._cache.components:
# A component that names its pools' keys spares the transfer build.
if ct == ComponentType.FULL or ct in overrides:
continue
transfers = tree_core.build_hicache_transfers(
ct, snapshot.node_id, CacheTransferPhase.BACKUP_HOST
)
for transfer in transfers or ():
keys = self._transfer_keys(transfer, hash_values)
if keys:
pool_keys.setdefault(transfer.name, []).extend(keys)
for named in overrides.values():
for pool, keys in named.items():
if keys:
pool_keys[pool] = list(keys)
else:
pool_keys.pop(pool, None)
for spec in self._cache.sidecar_pool_specs:
source = pool_keys.get(spec.indices_from_pool)
if source:
pool_keys[spec.pool_name] = list(source)
return pool_keys
@staticmethod
def _pool_order(pool: PoolName) -> tuple[bool, str]:
# KV first, then a fixed order: every rank must admit identically.
return (pool != PoolName.KV, pool.value)
def _write_intents(
self, snapshot: BufferBackupSnapshot
) -> list[tuple[PoolName, list[str]]]:
"""(pool, keys) of the pools not believed stored or in flight; a
sidecar riding a covered pool drives that pool too."""
pool_keys = self._backup_pool_keys(snapshot)
sidecars = {
spec.pool_name: spec.indices_from_pool
for spec in self._cache.sidecar_pool_specs
}
intents: list[tuple[PoolName, list[str]]] = []
for pool in sorted(pool_keys, key=self._pool_order):
keys = pool_keys[pool]
if pool in sidecars or not keys:
continue
riders = [
name
for name, source in sidecars.items()
if source == pool and pool_keys.get(name)
]
if all(
self._pool_covered(name, pool_keys[name]) for name in (pool, *riders)
):
continue
intents.append((pool, keys))
return intents
def _sizing(
self, pool: PoolName, keys: list[str]
) -> tuple[int, list[PoolTransfer]]:
"""(KV tokens, keys-only aux transfers) an intent stages, for the
oversize and budget gates."""
if pool == PoolName.KV:
return len(keys) * self._cache.page_size, []
return 0, [PoolTransfer(name=pool, keys=list(keys))]
def _finish_inflight(self, node_id: NodeId, pool: PoolName) -> None:
pools = self.inflight_backup_pools.get(node_id)
if pools is None:
return
pools.discard(pool)
if not pools:
del self.inflight_backup_pools[node_id]
@staticmethod
def _queued_span_key(intent: _UnifiedBackupIntent) -> tuple[NodeId, int]:
snapshot = intent.snapshot
return snapshot.node_id, len(snapshot.key)
def _retain_queued_span(self, intent: _UnifiedBackupIntent) -> None:
span = self._queued_span_key(intent)
refs = self._queued_span_refs.get(span, 0)
if refs == 0:
self.write_backlog_tokens_ += span[1]
self._queued_span_refs[span] = refs + 1
def _release_queued_span(self, intent: _UnifiedBackupIntent) -> None:
span = self._queued_span_key(intent)
refs = self._queued_span_refs[span]
if refs == 1:
del self._queued_span_refs[span]
self.write_backlog_tokens_ -= span[1]
else:
self._queued_span_refs[span] = refs - 1
def begin_insert_walk(self) -> None:
"""Admit each node once per insert walk: the trigger chains every
unbacked ancestor, so a depth-d path is presented d(d+1)/2 times."""
self._walk_admitted = set()
def end_insert_walk(self) -> None:
self._walk_admitted = None
def enqueue_backup_intent(self, node_id: NodeId) -> None:
"""Snapshot a backup intent and commit it to the write queue.
Admission gates: belief skip, parent-cover, backlog cap, oversize.
Rejected intents are counted; the node re-triggers on a later hit."""
"""Queue one intent per pool with something to write; rejected intents
are counted and the node re-triggers on a later hit."""
if not self._cache.enable_storage:
return
if node_id in self.inflight_backup_node_ids:
return
admitted = self._walk_admitted
if admitted is not None:
if node_id in admitted:
return
admitted.add(node_id)
snapshot = self._cache.tree_core.snapshot_buffer_backup(
node_id, self._cache.hicache_storage_pass_prefix_keys
)
if snapshot is None:
return
# Admission cover: beliefs plus content past its D2H launch. The
# launched cover keeps republished content (fill inserts under new
# node ids) from re-writing while the original write drains.
if self._cache.storage_existence_cache.covers_all(
PoolName.KV,
snapshot.hash_values,
extra_cover=self.inflight_backup_hashes,
):
intents = self._write_intents(snapshot)
if not intents:
return
intent_tokens = len(snapshot.hash_values) * self._cache.page_size
if self.write_backlog_tokens_ >= self.write_backlog_cap:
# The cap sits at 2x the intrinsic live-backlog ceiling (see
# init_hicache), so reaching it means leaked accounting or a
# broken stale sweep — a bug, not load.
self._backlog_cap_hits += 1
if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0:
logger.error(
"HiCache write backlog cap hit (occurrence %d): "
"backlog=%d cap=%d queue=%d. Live backlog is bounded "
"by the device pool span, so this indicates a "
"stale-sweep or accounting leak.",
self._backlog_cap_hits,
self.write_backlog_tokens_,
self.write_backlog_cap,
len(self.pending_write_queue),
)
self._log_backup_dropped(intent_tokens)
return
# A span larger than any pool's whole staging capacity can never
# stage; admitting it would wedge the head-of-line queue forever.
state = BufferBackupState(
parent_node_id=snapshot.parent_node_id,
parent_is_root=snapshot.parent_is_root,
parent_last_hash=snapshot.parent_last_hash,
pending = self.inflight_backup_pools.get(node_id, set())
parent_covered = self._backup_parent_covered(
BufferBackupState(
parent_node_id=snapshot.parent_node_id,
parent_is_root=snapshot.parent_is_root,
parent_last_hash=snapshot.parent_last_hash,
),
node_id,
)
if not self._backup_parent_covered(state) or self._backup_oversize(
snapshot.node_id, snapshot.hash_values, intent_tokens
):
self._log_backup_dropped(intent_tokens)
return
intent = _UnifiedBackupIntent(snapshot=snapshot)
self.pending_write_queue.append(intent)
self.inflight_backup_node_ids.add(snapshot.node_id)
self.write_backlog_tokens_ += intent_tokens
def _build_aux_staging_transfers(
self,
node_id: NodeId,
hash_values: list[str],
comp_xfers: Optional[dict[ComponentType, list[PoolTransfer]]] = None,
) -> list[PoolTransfer]:
"""Keys-only aux transfers mirroring what BACKUP_STORAGE would write;
sizes the per-pool oversize gate (beliefs do not consult these)."""
transfers: list[PoolTransfer] = []
if ComponentType.SWA in self._cache.components:
current = (
comp_xfers.get(ComponentType.SWA)
if comp_xfers is not None
else self._cache.tree_core.build_hicache_transfers(
ComponentType.SWA,
node_id,
CacheTransferPhase.BACKUP_HOST,
)
)
for transfer in current or ():
if transfer.device_indices is None:
continue
num_pages = len(transfer.device_indices) // self._cache.page_size
if num_pages > 0:
transfers.append(
PoolTransfer(
name=PoolName.SWA,
keys=hash_values[-num_pages:],
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
page_size = self._cache.page_size
span = (snapshot.node_id, len(snapshot.key))
for pool, keys in intents:
if pool in pending:
continue
intent_tokens = len(keys) * page_size
if (
span not in self._queued_span_refs
and self.write_backlog_tokens_ >= self.write_backlog_cap
):
# The cap sits at 2x the intrinsic live-backlog ceiling (see
# init_hicache), so reaching it means leaked accounting or a
# broken stale sweep — a bug, not load.
self._backlog_cap_hits += 1
if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0:
logger.error(
"HiCache write backlog cap hit (occurrence %d): "
"backlog=%d cap=%d queue=%d. Live backlog is bounded "
"by the device pool span, so this indicates a "
"stale-sweep or accounting leak.",
self._backlog_cap_hits,
self.write_backlog_tokens_,
self.write_backlog_cap,
len(self.pending_write_queue),
)
return transfers
self._log_backup_dropped(intent_tokens)
continue
# A span larger than the pool's whole staging capacity can never
# stage; admitting it would wedge the head-of-line queue forever.
if not parent_covered or self._backup_oversize(*self._sizing(pool, keys)):
self._log_backup_dropped(intent_tokens)
continue
intent = _UnifiedBackupIntent(snapshot=snapshot, pool=pool, keys=list(keys))
self.pending_write_queue.append(intent)
self.inflight_backup_pools.setdefault(node_id, set()).add(pool)
self._retain_queued_span(intent)
def _backup_oversize(
self,
node_id: NodeId,
hash_values: list[str],
intent_tokens: int,
aux_xfers: Optional[list[PoolTransfer]] = None,
) -> bool:
def _backup_oversize(self, kv_tokens: int, aux_xfers: list[PoolTransfer]) -> bool:
"""True if any pool's staging need exceeds that pool's write-usable
capacity (total for KV, total minus the loads-priority margin for aux
pools — matching ``_aux_budget_blocked``'s admission ceiling): such an
intent could never stage and would wedge the FIFO head."""
if aux_xfers is None:
aux_xfers = self._build_aux_staging_transfers(node_id, hash_values)
intent could never stage and would wedge the FIFO head. Aux pools
staged through the anchor's host pool count against the KV total."""
cc = self._cache.cache_controller
shared_requests = self._shared_host_requests(intent_tokens, aux_xfers)
shared_requests = self._shared_host_requests(kv_tokens, aux_xfers)
if shared_requests is not None:
pool_tokens = cc.mem_pool_host.size
max_write_units = pool_tokens - pool_tokens // 10
if self._host_request_units(intent_tokens, aux_xfers) > max_write_units:
if self._host_request_units(kv_tokens, aux_xfers) > max_write_units:
return True
return not self._shared_backup_fits(shared_requests, empty=True)
if intent_tokens > cc.mem_pool_host.size:
return True
for t in aux_xfers or ():
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
anchor_need = kv_tokens
for t in aux_xfers:
entry = cc.mem_pool_host.entry_map.get(t.name)
if entry is not None and (
len(t.keys) * entry.host_pool.page_size
> entry.host_pool.size - self._aux_loads_margin(entry.host_pool)
):
if entry is None:
continue
need = len(t.keys) * entry.host_pool.page_size
if entry.host_pool is anchor_pool:
anchor_need += need
elif need > entry.host_pool.size - self._aux_loads_margin(entry.host_pool):
return True
return False
return anchor_need > cc.mem_pool_host.size
def _aux_loads_margin(self, host_pool) -> int:
"""Aux-pool tokens reserved for loads: at least one trailing window
@@ -623,9 +742,9 @@ class BufferModePipeline:
snapshot = intent.snapshot
state = self._validate_backup_intent(intent)
if state is None:
self.inflight_backup_node_ids.discard(snapshot.node_id)
intent_tokens = len(snapshot.hash_values) * page_size
self.write_backlog_tokens_ -= intent_tokens
self._finish_inflight(snapshot.node_id, intent.pool)
intent_tokens = len(intent.keys) * page_size
self._release_queued_span(intent)
swept_tokens += intent_tokens
continue
survivors.append(intent)
@@ -657,54 +776,57 @@ class BufferModePipeline:
while self.pending_write_queue:
intent = self.pending_write_queue[0]
snapshot = intent.snapshot
state = states[snapshot.node_id]
intent_tokens = len(snapshot.hash_values) * self._cache.page_size
if not self._backup_parent_covered(state):
node_id = snapshot.node_id
state = states[node_id]
intent_tokens = len(intent.keys) * self._cache.page_size
if not self._backup_parent_covered(state, node_id):
# Cascade a dropped parent down the chain rather than creating
# a permanent storage hole.
self.pending_write_queue.popleft()
self.inflight_backup_node_ids.discard(snapshot.node_id)
self.write_backlog_tokens_ -= intent_tokens
self._log_backup_dropped(intent_tokens)
self._drop_queue_head(intent_tokens, dropped=True)
continue
if self.write_staged_tokens_ >= live_cap:
if (
self.write_staged_tokens_ >= live_cap
and self._host_request_units(*self._sizing(intent.pool, intent.keys))
> 0
):
# Wait for current copies to free staging before rebuilding the head.
break
device_value, comp_xfers = self._cache.tree_core.build_backup_spec(
snapshot.node_id
)
sizing_xfers = self._build_aux_staging_transfers(
snapshot.node_id, snapshot.hash_values, comp_xfers
)
if self._backup_oversize(
snapshot.node_id,
snapshot.hash_values,
intent_tokens,
sizing_xfers,
launch = self._launch_transfers(intent)
# The pool may have become covered since admission (an earlier
# launch this round, a sibling's ack): nothing left to write
# retires the intent, not a drop.
if launch is None or all(
self._pool_covered(pool, keys) for pool, keys in launch[1].items()
):
self._drop_queue_head(intent_tokens, dropped=False)
continue
device_value, keys_by_pool, transfers = launch
sizing_xfers = [t for t in transfers if t.indices_from_pool is None]
kv_tokens = len(device_value)
if self._backup_oversize(kv_tokens, sizing_xfers):
# A permanently unstageable head must not block the queue.
self.pending_write_queue.popleft()
self.inflight_backup_node_ids.discard(snapshot.node_id)
self.write_backlog_tokens_ -= intent_tokens
self._log_backup_dropped(intent_tokens)
self._drop_queue_head(intent_tokens, dropped=True)
continue
shared_domain = self._shared_host_domain()
staging_at_limit = (
self.write_staged_tokens_
+ self._host_request_units(intent_tokens, sizing_xfers)
+ self._host_request_units(kv_tokens, sizing_xfers)
> live_cap
if shared_domain is not None
else self.write_staged_tokens_ >= live_cap
and self._host_request_units(kv_tokens, sizing_xfers) > 0
)
if staging_at_limit:
# Yield to live fetch demand; retry next round.
break
if self._aux_budget_blocked(intent, sizing_xfers):
if self._aux_budget_blocked(kv_tokens, sizing_xfers):
# An aux pool lacks staging headroom: yield at the gate
# instead of failing the alloc inside cc.write; acks free
# aux staging, retry next round.
break
if not self._stage_backup_intent(intent, device_value, comp_xfers):
if not self._stage_backup_intent(
intent, device_value, transfers, keys_by_pool
):
# Pool full of in-flight staging and nothing reclaimable
# (the tree never holds host values in buffer mode):
# defer, head-of-line; pending acks will free slots.
@@ -715,11 +837,106 @@ class BufferModePipeline:
# Do not leave prepared copies deferred across scheduler admission.
cc.start_writing()
def _drop_queue_head(self, intent_tokens: int, *, dropped: bool) -> None:
"""Retire the FIFO head without a D2H launch; ``dropped`` feeds the
dropped-tokens metric (a pool with nothing left to write is not a
drop)."""
intent = self.pending_write_queue.popleft()
self._finish_inflight(intent.snapshot.node_id, intent.pool)
self._release_queued_span(intent)
if dropped:
self._log_backup_dropped(intent_tokens)
def _build_backup_transfers(
self, node_id: NodeId, *, kv_only: bool = False
) -> tuple[
torch.Tensor, dict[ComponentType, list[PoolTransfer]], Optional[list[str]]
]:
"""The node's D2H spec. A FULL component's BACKUP_HOST KV transfer
replaces the KV rows (its keys name their pages); its other transfers
ride as aux pools. A KV intent skips the other components' transfers."""
tree_core = self._cache.tree_core
if kv_only:
device_value = tree_core.get_component_device_value(
node_id, ComponentType.FULL
)
comp_xfers = {}
else:
device_value, comp_xfers = tree_core.build_backup_spec(node_id)
full_xfers = tree_core.build_hicache_transfers(
ComponentType.FULL, node_id, CacheTransferPhase.BACKUP_HOST
)
kv_keys = None
aux: list[PoolTransfer] = []
for transfer in full_xfers or ():
if transfer.name == PoolName.KV:
device_value = (
transfer.device_indices
if transfer.device_indices is not None
else device_value[:0]
)
kv_keys = list(transfer.keys or ())
else:
aux.append(transfer)
if aux:
comp_xfers[ComponentType.FULL] = aux
return device_value, comp_xfers, kv_keys
def _launch_transfers(
self, intent: _UnifiedBackupIntent
) -> Optional[tuple[torch.Tensor, dict[PoolName, list[str]], list[PoolTransfer]]]:
"""(KV rows, keys per written pool, transfers to stage) of one intent,
off the node's fresh spec; None when nothing is left to write."""
snapshot = intent.snapshot
hash_values = snapshot.hash_values
device_value, comp_xfers, kv_keys = self._build_backup_transfers(
snapshot.node_id, kv_only=intent.pool == PoolName.KV
)
transfers: list[PoolTransfer] = []
if intent.pool == PoolName.KV:
keys = hash_values if kv_keys is None else kv_keys
if not keys or len(device_value) == 0:
return None
comp_xfers = {}
else:
device_value = device_value[:0]
keys = []
for ct, xfers in comp_xfers.items():
kept = []
for transfer in xfers:
if transfer.name != intent.pool or transfer.indices_from_pool:
continue
t_keys = self._transfer_keys(transfer, hash_values)
if not t_keys:
continue
if transfer.keys is None:
transfer.keys = t_keys
transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES
kept.append(transfer)
keys.extend(t_keys)
if kept:
transfers.extend(kept)
comp_xfers[ct] = kept
if not transfers:
return None
comp_xfers = {ct: xfers for ct, xfers in comp_xfers.items() if xfers}
keys_by_pool = {intent.pool: list(keys)}
# Sidecars ride their source pool's slots and D2H operation; they
# allocate no staging of their own.
for sidecar in self._cache._build_backup_sidecar(device_value, comp_xfers):
if sidecar.indices_from_pool != intent.pool:
continue
sidecar.keys = list(keys)
keys_by_pool[sidecar.name] = list(keys)
transfers.append(sidecar)
return device_value, keys_by_pool, transfers
def _stage_backup_intent(
self,
intent: _UnifiedBackupIntent,
device_value: torch.Tensor,
comp_xfers: dict[ComponentType, list[PoolTransfer]],
transfers: list[PoolTransfer],
keys_by_pool: dict[PoolName, list[str]],
) -> bool:
"""Allocate host staging and pin one admitted intent's source.
@@ -730,64 +947,54 @@ class BufferModePipeline:
cache = self._cache
cc = cache.cache_controller
snapshot = intent.snapshot
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
# Sidecars reuse the source pool's transient host/device indices. This
# includes both KV-derived pools and SWA-derived DSV4 state pools. They
# allocate no additional staging, but must ride the same D2H operation
# so their bytes are present when the storage write starts.
aux_xfers.extend(cache._build_backup_sidecar(device_value, comp_xfers))
# One ack id per write: a node may have several pools in flight.
self._backup_ack_seq += 1
ack_id = -(1 << 40) - self._backup_ack_seq
host_indices = cc.write(
device_value,
node_id=snapshot.node_id,
extra_pools=aux_xfers or None,
node_id=ack_id,
extra_pools=transfers or None,
flush=False,
)
if host_indices is None:
self._backup_ack_seq -= 1
return False
_track_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
beliefs = cache.storage_existence_cache
for pool, keys in keys_by_pool.items():
beliefs.pool(pool).track_inflight(keys)
# NOTE: no commit_backup — the node must never appear
# host-resident in buffer mode; staging slots live in the entry.
lock_params = cache.inc_lock_ref(snapshot.node_id).to_dec_params()
occupied_units = self.host_allocation_units(host_indices, aux_xfers)
self.ongoing_write_through[snapshot.node_id] = _UnifiedBufferBackupEntry(
occupied_units = self.host_allocation_units(host_indices, transfers)
self.ongoing_write_through[ack_id] = _UnifiedBufferBackupEntry(
intent=intent,
host_indices=host_indices,
aux_xfers=aux_xfers,
aux_xfers=transfers,
lock_params=lock_params,
occupied_units=occupied_units,
keys_by_pool=keys_by_pool,
)
self.write_staged_tokens_ += occupied_units
self.write_backlog_tokens_ -= len(snapshot.hash_values) * cache.page_size
self._release_queued_span(intent)
return True
def _aux_budget_blocked(
self,
intent: _UnifiedBackupIntent,
aux: Optional[list[PoolTransfer]] = None,
) -> bool:
def _aux_budget_blocked(self, kv_tokens: int, aux: list[PoolTransfer]) -> bool:
"""True when an aux pool cannot stage this intent right now (free
minus the loads-priority margin falls short of the need): defer at
the gate instead of failing the alloc inside cc.write and blocking
pure-KV intents behind an unallocatable head. The margin enforces
loads-have-priority on aux pools the way live_cap does on the KV
pool; avail already reflects prefetch-held slots, so no occupancy
subtraction here."""
snapshot = intent.snapshot
if aux is None:
aux = self._build_aux_staging_transfers(
snapshot.node_id, snapshot.hash_values
)
subtraction here. Aux pools staged through the anchor's host pool
are gated by live_cap and the allocation itself."""
cc = self._cache.cache_controller
shared_requests = self._shared_host_requests(
len(snapshot.hash_values) * self._cache.page_size, aux
)
shared_requests = self._shared_host_requests(kv_tokens, aux)
if shared_requests is not None:
return not self._shared_backup_fits(shared_requests)
if not aux:
return False
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
for t in aux:
entry = cc.mem_pool_host.entry_map.get(t.name)
if entry is None:
if entry is None or entry.host_pool is anchor_pool:
continue
need = len(t.keys) * entry.host_pool.page_size
headroom = entry.host_pool.available_size() - self._aux_loads_margin(
@@ -797,23 +1004,6 @@ class BufferModePipeline:
return True
return False
def _aux_window_keys(
self, hash_values: list[str], transfer: PoolTransfer
) -> Optional[list[str]]:
"""Trailing KV page hashes keying an aux transfer's staged window
(one key per aux-pool page)."""
if transfer.host_indices is None or transfer.host_indices.numel() == 0:
return None
if transfer.indices_from_pool is not None:
return None # sidecar rides another pool's slots; nothing to key
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
return None
num_keys = len(transfer.host_indices) // entry.host_pool.page_size
if num_keys == 0 or num_keys > len(hash_values):
return None
return hash_values[-num_keys:]
def finish_backup_ack(self, ack_id: int) -> None:
"""D2H confirmed: drop the device lock and enqueue the storage write
(which reads from the staging copy, so device eviction may proceed)."""
@@ -822,73 +1012,65 @@ class BufferModePipeline:
snapshot = intent.snapshot
self._cache.dec_lock_ref(snapshot.node_id, entry.lock_params)
# Every independently staged aux pool writes a trailing snapshot keyed
# by the last KV page hashes it covers. A derived sidecar writes the
# exact key span of its source pool while reusing that source's slots.
storage_xfers: list[PoolTransfer] = []
storage_sources = {
PoolName.KV: PoolTransfer(
name=PoolName.KV,
host_indices=entry.host_indices,
keys=snapshot.hash_values,
)
}
for staged in entry.aux_xfers:
if staged.indices_from_pool is not None:
continue
keys = self._aux_window_keys(snapshot.hash_values, staged)
if keys is None:
continue
transfer = PoolTransfer(
name=staged.name,
host_indices=staged.host_indices,
keys=keys,
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
storage_xfers.append(transfer)
storage_sources.setdefault(staged.name, transfer)
for staged in entry.aux_xfers:
if staged.indices_from_pool is None:
continue
source = storage_sources.get(staged.indices_from_pool)
if source is None:
raise AssertionError(
"Buffer-mode storage sidecar source missing: "
f"{staged.name} from {staged.indices_from_pool}."
)
storage_xfers.append(
PoolTransfer(
name=staged.name,
keys=source.keys,
hit_policy=staged.hit_policy,
indices_from_pool=staged.indices_from_pool,
)
)
storage_xfers = self._storage_transfers(entry.aux_xfers)
# Another pool's write carries no KV pages; its keys ride the transfer.
hash_value = (
list(entry.keys_by_pool[PoolName.KV]) if intent.pool == PoolName.KV else []
)
operation_id = self._cache.cache_controller.write_storage(
entry.host_indices,
snapshot.key.token_ids,
snapshot.hash_values,
hash_value,
snapshot.prefix_keys,
extra_pools=storage_xfers or None,
)
self.ongoing_backup[operation_id] = entry
@staticmethod
def _storage_transfers(staged: list[PoolTransfer]) -> list[PoolTransfer]:
"""One storage transfer per pool: a pool staged in several transfers
becomes one contiguous key span over their slots; a derived sidecar
writes its source's key span over the source's slots."""
merged: dict[PoolName, PoolTransfer] = {}
for transfer in staged:
if transfer.indices_from_pool is not None:
merged[transfer.name] = PoolTransfer(
name=transfer.name,
keys=list(transfer.keys),
hit_policy=transfer.hit_policy,
indices_from_pool=transfer.indices_from_pool,
)
continue
earlier = merged.get(transfer.name)
if earlier is None:
merged[transfer.name] = PoolTransfer(
name=transfer.name,
host_indices=transfer.host_indices,
keys=list(transfer.keys),
hit_policy=transfer.hit_policy,
)
continue
earlier.keys.extend(transfer.keys)
earlier.host_indices = torch.cat(
[earlier.host_indices, transfer.host_indices]
)
return list(merged.values())
def finish_storage_write_ack(self, operation_id: int) -> None:
"""Storage write acked (rank-synced drain): free the entry's staging
outright. Existence entries are added unconditionally
(completed_tokens can diverge across ranks under backend failure) to
keep admission decisions TP-deterministic. No-op for operations this
pipeline does not own (e.g. acks for already-reset state)."""
"""Storage ack (rank-synced drain): free the staging and feed each
written pool's beliefs unconditionally, keeping admission
TP-deterministic under per-rank backend failures."""
entry = self.ongoing_backup.pop(operation_id, None)
if entry is None:
return
intent = entry.intent
snapshot = intent.snapshot
self._cache.storage_existence_cache.add(PoolName.KV, snapshot.hash_values)
beliefs = self._cache.storage_existence_cache
for pool, keys in entry.keys_by_pool.items():
pool_beliefs = beliefs.pool(pool)
pool_beliefs.add(keys)
pool_beliefs.untrack_inflight(keys)
self._free_staging_now(entry.host_indices, entry.aux_xfers)
self.write_staged_tokens_ -= entry.occupied_units
self.inflight_backup_node_ids.discard(snapshot.node_id)
_untrack_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
self._finish_inflight(entry.intent.snapshot.node_id, entry.intent.pool)
def _free_staging_now(
self, host_indices: torch.Tensor, aux_xfers: list[PoolTransfer]
@@ -1163,10 +1345,16 @@ class BufferModePipeline:
staged_pages = num_tokens // cache.page_size
staged_hashes = hash_value[:staged_pages]
staged_kv = host_indices[:num_tokens]
# Feed existence beliefs from the storage-fetched pages: the fetch
# itself is the evidence, so feeding is sound even if this staged
# prefetch is later dropped unconsumed.
cache.storage_existence_cache.add(PoolName.KV, list(staged_hashes))
# The fetch is evidence for every pool it delivered (sound even if the
# staged prefetch is later dropped): KV-derived sidecars came with the
# staged pages, other pools with the keys their transfer carries.
beliefs = cache.storage_existence_cache
beliefs.pool(PoolName.KV).add(staged_hashes)
for transfer in aux_xfers:
if transfer.indices_from_pool == PoolName.KV:
beliefs.pool(transfer.name).add(staged_hashes)
elif transfer.keys:
beliefs.pool(transfer.name).add(transfer.keys)
self.staged_prefetches[request] = _StagedPrefetch(
request=request,
key_tokens=array(
@@ -1,17 +1,18 @@
"""Local existence cache for HiCache buffer_only mode.
"""Local existence beliefs for HiCache buffer_only mode.
In buffer mode host memory holds no persistent copy, so without a local
existence signal every re-insert of a hot prefix re-stages and re-writes to
L3 storage. This cache is that signal: a bounded LRU of (pool, page-hash)
entries *believed* present in storage.
L3 storage. The beliefs are that signal: per storage pool, a bounded LRU of
page hashes *believed* present in storage, kept and healed independently so
a node is written pool by pool (only the pools not believed stored).
Semantics are advisory, not authoritative:
- A hit skips the redundant D2H + storage write.
- A hit skips the pool's redundant D2H staging + storage write.
- A stale positive (backend evicted the data) costs skipped write-backs until
a prefetch hit-query shortfall invalidates the entries; the next insert
then writes the data back. Never a correctness issue — at worst one cold
recompute, the same as any cache miss.
a prefetch hit-query shortfall invalidates the pool's entries; the next
insert then writes that pool back. Never a correctness issue — at worst
one cold recompute, the same as any cache miss.
- A miss (entry LRU-evicted or never seen) just costs one redundant write
(idempotent: storage keys are content-addressed).
@@ -21,6 +22,10 @@ recompute (same tokens => same chain). Page hashes are chained and the write
path is prefix-contiguous (parent-cover gate), so a node's own page set is
the only thing a caller needs to check.
A pool also tracks its in-flight content (D2H launch to storage ack), which
admission treats as covered; its ``PoolBeliefPolicy`` says how its keys sit
on the chain (a component may supply one via ``buffer_belief_policy``).
TP determinism: replicas stay identical because every mutation happens on the
scheduler thread at lockstep points with cross-rank-reduced inputs
(storage-ack drain, prefetch-hit drain, fill commit). Do not touch it from
@@ -30,82 +35,165 @@ anywhere else.
from __future__ import annotations
from collections import OrderedDict
from collections.abc import Container, Iterable, Sequence
from collections.abc import Iterable, Sequence
# ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers roughly
# 32M tokens at page size 64 across all pools.
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy
# Per pool: ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers
# roughly 32M tokens at page size 64.
HICACHE_EXISTENCE_CACHE_MAX_ENTRIES = 512 * 1024
class StorageExistenceCache:
class PoolExistenceBeliefs:
"""One pool's LRU of page hashes believed stored, plus its refcounted
in-flight content (D2H launch to storage ack)."""
def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES):
self.max_entries = max_entries
self._entries: OrderedDict[tuple[str, str], None] = OrderedDict()
self._entries: OrderedDict[str, None] = OrderedDict()
self._inflight: dict[str, int] = {}
def __len__(self) -> int:
return len(self._entries)
def add(self, pool: str, hashes: Iterable[str]) -> None:
def add(self, hashes: Iterable[str]) -> None:
entries = self._entries
move_to_end = entries.move_to_end
for h in hashes:
key = (pool, h)
entries[key] = None
move_to_end(key)
entries[h] = None
move_to_end(h)
pop_oldest = entries.popitem
for _ in range(len(entries) - self.max_entries):
pop_oldest(last=False)
def contains(self, pool: str, page_hash: str) -> bool:
def contains(self, page_hash: str) -> bool:
entries = self._entries
key = (pool, page_hash)
if key not in entries:
if page_hash not in entries:
return False
entries.move_to_end(key)
entries.move_to_end(page_hash)
return True
def contains_all(self, pool: str, hashes: Iterable[str]) -> bool:
def contains_all(self, hashes: Iterable[str]) -> bool:
entries = self._entries
move_to_end = entries.move_to_end
for h in hashes:
key = (pool, h)
try:
move_to_end(key)
move_to_end(h)
except KeyError:
return False
return True
def covers_all(
self,
pool: str,
hashes: Iterable[str],
extra_cover: Container[str] = frozenset(),
) -> bool:
"""True when every page is believed stored or sits in
``extra_cover`` (e.g. content past its D2H launch, which always
reaches its storage-ack). LRU-touches the believed entries."""
def covered(self, hashes: Iterable[str]) -> bool:
"""Every page believed stored or in flight; LRU-touches believed entries."""
inflight = self._inflight
if not inflight:
return self.contains_all(hashes)
entries = self._entries
move_to_end = entries.move_to_end
if extra_cover:
for h in hashes:
key = (pool, h)
if key in entries:
move_to_end(key)
elif h not in extra_cover:
return False
return True
return self.contains_all(pool, hashes)
for h in hashes:
if h in entries:
move_to_end(h)
elif h not in inflight:
return False
return True
def invalidate_beyond(
self, pool: str, hashes: Sequence[str], keep_pages: int
) -> None:
"""Ground-truth heal from a prefetch hit query: discard beliefs
beyond the leading ``keep_pages`` of a hash chain (the folded
usable cut). The next insert re-writes the discarded span, closing
stale positives and aux holes at the cut."""
def track_inflight(self, hashes: Iterable[str]) -> None:
"""One ref per page hash at its D2H launch (several launched writes
can carry the same content)."""
refs = self._inflight
for h in hashes:
refs[h] = refs.get(h, 0) + 1
def untrack_inflight(self, hashes: Iterable[str]) -> None:
"""Drop one ref per page hash at the storage ack (or a failed launch)."""
refs = self._inflight
for h in hashes:
n = refs.get(h, 0) - 1
if n <= 0:
refs.pop(h, None)
else:
refs[h] = n
def invalidate_beyond(self, hashes: Sequence[str], keep_pages: int) -> None:
"""Discard the beliefs beyond the leading ``keep_pages`` of ``hashes``;
the next insert re-writes that span."""
pop = self._entries.pop
for h in hashes[keep_pages:]:
pop((pool, h), None)
pop(h, None)
def clear(self) -> None:
self._entries.clear()
self._inflight.clear()
def clear_inflight(self) -> None:
self._inflight.clear()
class StorageExistenceCache:
"""The beliefs of every storage pool, created on first use."""
def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES):
self.max_entries = max_entries
self._pools: dict[str, PoolExistenceBeliefs] = {}
def pool(self, name: str) -> PoolExistenceBeliefs:
beliefs = self._pools.get(name)
if beliefs is None:
beliefs = self._pools[name] = PoolExistenceBeliefs(self.max_entries)
return beliefs
def clear(self) -> None:
for beliefs in self._pools.values():
beliefs.clear()
def clear_inflight(self) -> None:
for beliefs in self._pools.values():
beliefs.clear_inflight()
class PoolBeliefPolicy:
"""How a pool's keys sit on a node's page-hash chain, hence how a storage
verdict heals its beliefs: a rank-agreed hit-query boundary (``evidence``
= the query reported this pool) or a transfer's delivered page count."""
def heal_from_query(
self,
beliefs: PoolExistenceBeliefs,
chain: Sequence[str],
keys: Sequence[str],
hit_pages: int,
*,
evidence: bool,
) -> None:
raise NotImplementedError
def heal_from_transfer(
self,
beliefs: PoolExistenceBeliefs,
keys: Sequence[str],
delivered_pages: int,
) -> None:
beliefs.invalidate_beyond(keys, delivered_pages)
class ChainPagesPolicy(PoolBeliefPolicy):
"""One key per chain page (ALL_PAGES). The agreed boundary is the MIN of
per-rank prefix hits, so every rank verified the chain up to it."""
def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence):
beliefs.invalidate_beyond(chain, hit_pages)
if evidence:
beliefs.add(chain[:hit_pages])
class TrailingWindowPolicy(PoolBeliefPolicy):
"""Keys are a trailing window of the chain (TRAILING_PAGES). A rank only
verified the window ending at its own boundary, so the query only
invalidates; acks and fills teach presence."""
def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence):
beliefs.invalidate_beyond(chain, hit_pages)
def default_belief_policy(hit_policy: PoolHitPolicy) -> PoolBeliefPolicy:
"""The built-in policy for a pool no component claims."""
if hit_policy == PoolHitPolicy.TRAILING_PAGES:
return TrailingWindowPolicy()
return ChainPagesPolicy()
@@ -51,6 +51,10 @@ from sglang.srt.utils import broadcast_pyobj
logger = logging.getLogger(__name__)
# Pool boundaries are nonnegative; an omitted rank must dominate MIN so
# a peer cannot establish presence of that rank's storage shard.
_NO_POOL_VERDICT = -1
@dataclass(frozen=True)
class PPPrefetchPoolSpec:
@@ -165,6 +169,10 @@ class PrefetchOperation(StorageOperation):
pool_transfers=pool_transfers,
)
self.pool_transfers_done = not bool(pool_transfers)
# The hit query's verdict per pool, as the chain prefix length (in
# pages) up to which the pool is present; rank-reduced with the
# folded hit count. Empty until the query ran on this rank.
self.query_pool_hit_pages: dict[PoolName, int] = {}
self.buffer_host_occupied_units: Optional[int] = None
# The Python transfer worker leaves the unfinished tail to the ACK drain;
# a controller that releases it itself must set this False.
@@ -1348,9 +1356,11 @@ class HybridCacheController(BaseHiCacheController):
if operation.assume_stored:
# A prior hit on a suffix of this span proved it stored, and writes
# are prefix-covered, so re-querying only adds a round trip.
# are prefix-covered, so re-querying only adds a round trip. Nothing
# is verified now, so no pool gets a verdict to learn from.
kv_hit_pages = len(hash_value)
operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages)
operation.query_pool_hit_pages = {}
return hash_value, kv_hit_pages * self.page_size
extra_info = HiCacheStorageExtraInfo(
@@ -1369,12 +1379,45 @@ class HybridCacheController(BaseHiCacheController):
kv_hit_pages = hit_result.kv_hit_pages
operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages)
# Each pool's own boundary (the KV pool's where the backend reports
# it, else the folded cut); the beliefs heal pool by pool. A pool the
# backend did not report carries no verdict.
pool_hits = hit_result.extra_pool_hit_pages
operation.query_pool_hit_pages = {
PoolName.KV: pool_hits.get(PoolName.KV, kv_hit_pages)
}
for transfer in operation.pool_transfers or ():
if transfer.name in pool_hits:
operation.query_pool_hit_pages[transfer.name] = pool_hits[transfer.name]
return (
hash_value[:kv_hit_pages],
kv_hit_pages * self.page_size,
)
def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int:
"""One MIN collective for the folded hit count and every pool's own
boundary, so per-pool beliefs stay rank-identical. A pool omitted
by any rank heals from the folded cut without learning presence."""
pools = list(PoolName)
local = operation.query_pool_hit_pages
packed = torch.tensor(
[storage_hit_count] + [local.get(pool, _NO_POOL_VERDICT) for pool in pools],
dtype=torch.int,
)
self._all_reduce(
packed,
torch.distributed.ReduceOp.MIN,
self.prefetch_hits_sync_groups,
)
reduced = packed.tolist()
operation.query_pool_hit_pages = {
pool: pages
for pool, pages in zip(pools, reduced[1:], strict=True)
if pages != _NO_POOL_VERDICT
}
return int(reduced[0])
def _move_pool_indices(
self, host_pool, host_indices, device_indices, *, write_back_jit: bool
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -17,7 +17,11 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult
from sglang.srt.mem_cache.hicache_storage import (
PoolName,
PoolTransfer,
PoolTransferResult,
)
from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401
BASE_COMPONENT_TYPE,
ComponentType,
@@ -25,7 +29,11 @@ from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
PoolBeliefPolicy,
)
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.pool_host.group import HostPoolGroup, PoolEntry
from sglang.srt.mem_cache.unified_cache.cache_action import (
CacheAction,
ComponentAction,
@@ -699,6 +707,40 @@ class TreeComponent(ABC):
"""Allocate prefetch staging sized by prepare_prefetch, once the hit is known."""
return None
# ---- Buffer-mode backup hooks (hicache_host_memory_mode=buffer_only) ----
# Buffer mode writes a node pool by pool: keys per pool come from
# ``buffer_backup_keys`` or the BACKUP_HOST transfers, beliefs heal through
# ``buffer_belief_policy``; the pipeline pins the node from D2H launch to
# ack, so a component dropping rows asks ``buffer_backup_pending`` first.
def buffer_mode_host_pool_entries(
self, host_pool_group: HostPoolGroup
) -> list[PoolEntry]:
"""Extra host pool entries this component stages through (an entry
may alias the anchor's host pool)."""
return []
def buffer_backup_keys(
self, node: UnifiedTreeNode, hash_values: list[str]
) -> dict[PoolName, list[str]]:
"""Per-pool storage keys a backup of ``node`` writes for this component
(empty list: withhold the pool; empty dict: derive them from its
BACKUP_HOST transfers). A component naming any pool names all of them."""
return {}
def buffer_backup_parent_covered(
self, parent_node_id: NodeId, node_id: NodeId
) -> bool:
"""Whether this component covers the parent of ``node_id`` without
the parent's KV pages in storage (no prefix hole for a reader)."""
return False
def buffer_belief_policy(self, pool: PoolName) -> Optional[PoolBeliefPolicy]:
"""The policy healing ``pool``'s existence beliefs, for a pool this
component owns; None picks the built-in one for the transfer's hit
policy."""
return None
def build_hicache_transfers(
self,
node: UnifiedTreeNode,
@@ -1055,6 +1055,26 @@ class SWAComponent(TreeComponent):
host_indices = self._swa_kv_pool_host.alloc(num_tokens)
return host_indices
def buffer_backup_keys(
self, node: UnifiedTreeNode, hash_values: list[str]
) -> dict[PoolName, list[str]]:
# Buffer mode stages the node's own SWA rows, one key per page from
# the chain's tail, so admission needs no transfer (and no device op).
if not self.tree_core.is_host_memory_buffer_only:
return {}
cd = node.component_data[self.component_type]
if not self.tree_core.has_swa_host_pool or cd.value is None:
return {PoolName.SWA: []}
rows = (
node.component_data[BASE_COMPONENT_TYPE].value
if self._unified_allocator() is not None
else cd.value
)
num_pages = len(rows) // self._swa_kv_pool_host.page_size
if num_pages == 0 or num_pages > len(hash_values):
return {PoolName.SWA: []}
return {PoolName.SWA: list(hash_values[-num_pages:])}
def build_hicache_transfers(
self,
node: UnifiedTreeNode,
@@ -2381,6 +2381,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
"""Read a node's device->host backup spec (device value + component transfers) now."""
return self._build_backup_spec(self.node_by_id(node_id))
def buffer_backup_pool_keys(
self, node_id: NodeId, hash_values: list[str]
) -> dict[ComponentType, dict[PoolName, list[str]]]:
node = self.node_by_id(node_id)
named: dict[ComponentType, dict[PoolName, list[str]]] = {}
for comp in self.components:
pool_keys = comp.buffer_backup_keys(node, hash_values)
if pool_keys:
named[comp.component_type] = pool_keys
return named
def build_backup_kv_action(self, node_id: NodeId) -> BackupKV:
return self._build_backup_kv_action(self.node_by_id(node_id))
def _build_backup_spec(self, node: UnifiedTreeNode):
"""Gather missing Full and component transfers for Host backup."""
device_value = node.component_data[BASE_COMPONENT_TYPE].value
@@ -127,7 +127,11 @@ if TYPE_CHECKING:
MatchResult,
)
from sglang.srt.mem_cache.events import KVCacheEventRecorder
from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult
from sglang.srt.mem_cache.hicache_storage import (
PoolName,
PoolTransfer,
PoolTransferResult,
)
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.cache_action import (
BackupKV,
@@ -549,6 +553,20 @@ class UnifiedTreeCoreInterface(ABC):
"""Read a node's device->host backup spec (device value + transfers) now."""
...
def buffer_backup_pool_keys(
self, node_id: NodeId, hash_values: list[str]
) -> dict[ComponentType, dict[PoolName, list[str]]]:
"""Per component, the storage keys per pool a buffer-mode backup of
the node writes (``TreeComponent.buffer_backup_keys``); Rust cores name none."""
return {}
def build_backup_kv_action(self, node_id: NodeId) -> BackupKV:
"""The write-through backup action for a node and its unstored
ancestors; only component-requested buffer backups need it."""
raise NotImplementedError(
f"{type(self).__name__} does not expose build_backup_kv_action"
)
@abstractmethod
def build_storage_backup_spec(
self, node_id: NodeId, pass_prefix_keys: bool
@@ -36,10 +36,17 @@ from sglang.srt.mem_cache.buffer_mode.pipeline import (
validate_buffer_only_stack,
)
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
PoolBeliefPolicy,
StorageExistenceCache,
default_belief_policy,
)
from sglang.srt.mem_cache.common import RetractionBackup
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer, SidecarPoolSpec
from sglang.srt.mem_cache.hicache_storage import (
PoolHitPolicy,
PoolName,
PoolTransfer,
SidecarPoolSpec,
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
PPPrefetchDecision,
@@ -391,6 +398,8 @@ class UnifiedRadixCache(BasePrefixCache):
self.ongoing_backup: dict[int, tuple[NodeId, DecLockRefParams]] = {}
if self.buffer_pipeline is not None:
self.buffer_pipeline.reset()
# The dropped writes' content no longer reaches a storage ack.
self.storage_existence_cache.clear_inflight()
if self.cache_controller is not None:
self.cache_controller.reset()
@@ -487,10 +496,9 @@ class UnifiedRadixCache(BasePrefixCache):
if swa is not None and self.tree_core.has_swa_host_pool
else 0
),
# Leak backstop only: live queued tokens are intrinsically
# bounded by the FULL device pool (one intent per node, stale
# intents swept per tick), so a cap that binds on live
# content would drop-newest and punch storage holes.
# Leak backstop only: the queue is charged once per unique node
# span across pools; a cap binding on live content would
# drop-newest and punch storage holes.
write_backlog_cap=2 * self.token_to_kv_pool_allocator.size_full,
)
# State initialization
@@ -498,6 +506,13 @@ class UnifiedRadixCache(BasePrefixCache):
self.cache_controller.host_write_staged_tokens_fn = lambda: (
self.buffer_pipeline.write_staged_tokens_
)
# Pools a component's backups stage through besides the stock
# ones (an entry may alias the anchor's host pool).
for component in self.components.values():
for entry in component.buffer_mode_host_pool_entries(
self.host_pool_group
):
self.cache_controller.register_host_pool_entry(entry)
self.write_through_threshold = (
1 if get_memory().hicache_write_policy == "write_through" else 2
)
@@ -596,6 +611,8 @@ class UnifiedRadixCache(BasePrefixCache):
return InsertResult(prefix_len=0)
# Fail fast on re-entrancy without touching the in-flight walk.
assert not self.tree_core.has_ongoing_insert(), "re-entrant insert"
if self.buffer_pipeline is not None:
self.buffer_pipeline.begin_insert_walk()
# Pump the resumable insert, applying each step's actions at its barrier.
try:
step = self.tree_core.begin_insert(params)
@@ -609,6 +626,8 @@ class UnifiedRadixCache(BasePrefixCache):
finally:
# Drain still-pending actions so frees reach the allocator on abort.
self._apply_cache_actions(self.tree_core.end_insert())
if self.buffer_pipeline is not None:
self.buffer_pipeline.end_insert_walk()
@rank_consensus(same_params=True, same_results=True)
def evict(self, params: EvictParams) -> EvictResult:
@@ -1577,6 +1596,22 @@ class UnifiedRadixCache(BasePrefixCache):
self.writing_check(write_back=True)
return True
def buffer_backup_pending(self, node_id: NodeId) -> bool:
"""Whether a buffer-mode write of ``node_id`` is queued, staged or
awaiting its storage ack (see ``BufferModePipeline.backup_pending``)."""
return self.buffer_pipeline is not None and self.buffer_pipeline.backup_pending(
node_id
)
def request_buffer_backup(self, node_id: NodeId) -> None:
"""Queue a buffer-mode backup of ``node_id`` (and its unstored
ancestors) outside the insert trigger; no-op without buffer storage."""
if self.buffer_pipeline is None or not self.enable_storage:
return
self._execute_and_commit_kv_backup(
self.tree_core.build_backup_kv_action(node_id)
)
def _execute_and_commit_kv_backup(
self, action: BackupKV, write_back: bool = False
) -> int:
@@ -2408,9 +2443,9 @@ class UnifiedRadixCache(BasePrefixCache):
for transfer, count in zip(pool_transfers, pool_hit_pages)
)
if pool_transfers and not all_succeeded:
# Drop the KV beliefs from the first page any pool failed to serve;
# the next insert then re-writes that span through one FULL check,
# restoring the missing aux pages.
# Drop each pool's beliefs beyond what it served (KV and its
# sidecars beyond the completed span); the next insert re-writes
# only the missing pools.
invalidation_hashes = (
operation.sidecar_hash_values
if operation.sidecar_hash_values is not None
@@ -2418,18 +2453,29 @@ class UnifiedRadixCache(BasePrefixCache):
)
trimmed_pages = len(invalidation_hashes) - len(hash_value)
keep_pages = trimmed_pages + completed_tokens // self.page_size
beliefs = self.storage_existence_cache
if keep_pages < len(invalidation_hashes):
self._belief_policy(
PoolName.KV, PoolHitPolicy.ALL_PAGES
).heal_from_transfer(
beliefs.pool(PoolName.KV), invalidation_hashes, keep_pages
)
for transfer in operation.pool_transfers or ():
if transfer.indices_from_pool == PoolName.KV:
self._belief_policy(
transfer.name, transfer.hit_policy
).heal_from_transfer(
beliefs.pool(transfer.name),
invalidation_hashes,
keep_pages,
)
for transfer, count in zip(pool_transfers, pool_hit_pages):
if transfer.keys is None:
keep_pages = 0
elif count < len(transfer.keys):
# Aux transfers key the chain's trailing pages.
keep_pages = min(
keep_pages,
max(0, len(invalidation_hashes) - len(transfer.keys)),
if transfer.keys is not None and count < len(transfer.keys):
self._belief_policy(
transfer.name, transfer.hit_policy
).heal_from_transfer(
beliefs.pool(transfer.name), transfer.keys, count
)
self.storage_existence_cache.invalidate_beyond(
PoolName.KV, invalidation_hashes, keep_pages=keep_pages
)
self._finish_storage_prefetch(
request, fulfilled_tokens=0, reason="storage_transfer"
)
@@ -2444,11 +2490,15 @@ class UnifiedRadixCache(BasePrefixCache):
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
logger.warning(
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
"kv_beliefs_kept_pages=%d",
"kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s",
request.rid,
completed_tokens,
expected_tokens,
keep_pages,
{
getattr(transfer.name, "value", transfer.name): count
for transfer, count in zip(pool_transfers, pool_hit_pages)
},
)
return False
return True
@@ -2637,18 +2687,47 @@ class UnifiedRadixCache(BasePrefixCache):
),
)
def _belief_policy(
self, pool: PoolName, hit_policy: PoolHitPolicy
) -> PoolBeliefPolicy:
"""The policy healing ``pool``'s existence beliefs: the owning
component's, else the built-in one for the transfer's hit policy."""
for component in self.components.values():
policy = component.buffer_belief_policy(pool)
if policy is not None:
return policy
return default_belief_policy(hit_policy)
def _invalidate_absent_from_hit_query(self, operation) -> None:
"""Drop KV beliefs beyond the folded usable cut (rank-synced): the
next insert then re-writes the node (all pools), healing stale
positives and aux holes at the cut through one FULL check."""
"""Heal each pool's beliefs beyond its rank-agreed boundary (a
chain-page pool also learns presence up to it); a pool without a
verdict heals from the folded cut."""
if self.host_memory_mode != "buffer_only":
return
chain = operation.all_hash_values
if chain is None:
return
self.storage_existence_cache.invalidate_beyond(
PoolName.KV, chain, keep_pages=operation.storage_hit_count // self.page_size
beliefs = self.storage_existence_cache
verdicts = operation.query_pool_hit_pages
folded_pages = operation.storage_hit_count // self.page_size
def verdict(pool: PoolName) -> tuple[int, bool]:
pages = verdicts.get(pool)
return (folded_pages, False) if pages is None else (pages, True)
kv_pages, kv_evidence = verdict(PoolName.KV)
self._belief_policy(PoolName.KV, PoolHitPolicy.ALL_PAGES).heal_from_query(
beliefs.pool(PoolName.KV), chain, chain, kv_pages, evidence=kv_evidence
)
for transfer in operation.pool_transfers or ():
hit_pages, evidence = verdict(transfer.name)
self._belief_policy(transfer.name, transfer.hit_policy).heal_from_query(
beliefs.pool(transfer.name),
chain,
list(transfer.keys or ()),
hit_pages,
evidence=evidence,
)
def _account_prefetch_outcome(self, operation, revoked: bool) -> None:
"""Feed the cumulative prefetch-outcome counters at the (rank-synced)
@@ -2,31 +2,47 @@
import unittest
from array import array
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import msgspec
import torch
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
from sglang.srt.mem_cache.buffer_mode.pipeline import (
BufferModePipeline,
_UnifiedBackupIntent,
validate_buffer_only_stack,
)
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
StorageExistenceCache,
)
from sglang.srt.mem_cache.hicache_storage import (
HiCacheFile,
HiCacheStorageConfig,
PoolHitPolicy,
PoolName,
PoolTransfer,
SidecarPoolSpec,
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
PrefetchOperation,
)
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.components.base import (
ComponentType,
)
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
BufferBackupSnapshot,
BufferBackupState,
)
from sglang.srt.mem_cache.unified_radix_cache import _OngoingPrefetch
from sglang.srt.mem_cache.unified_radix_cache import (
UnifiedRadixCache,
_OngoingPrefetch,
)
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
@@ -44,6 +60,171 @@ class TestBufferModeSidecar(unittest.TestCase):
),
)
def test_file_query_missing_rank_preserves_swa_repair(self):
snapshot = self._backup_snapshot()
token_ids = list(snapshot.key.token_ids)
chain = get_storage_hash_str(token_ids, page_size=2)
snapshot = msgspec.structs.replace(snapshot, hash_values=chain)
swa = PoolTransfer(
name=PoolName.SWA,
keys=chain[-1:],
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
for peer_swa_pages in (0, 1):
with (
TemporaryDirectory() as directory,
patch.dict(
"os.environ", {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": directory}
),
):
ranks, packed = [], []
def capture(values, reduce_op, groups):
self.assertEqual(reduce_op, torch.distributed.ReduceOp.MIN)
packed.append(values.clone())
for rank, swa_pages in enumerate((2, peer_swa_pages)):
backend = HiCacheFile(
HiCacheStorageConfig(
tp_rank=rank,
tp_size=2,
pp_rank=0,
pp_size=1,
attn_cp_rank=0,
attn_cp_size=1,
is_mla_model=False,
enable_storage_metrics=False,
is_page_first_layout=False,
model_name="swa-repair",
extra_config={"enable_metadata_cache": False},
)
)
for pool, keys in (
(PoolName.KV, chain),
(PoolName.SWA, chain[:swa_pages]),
):
for key in keys:
Path(
directory,
backend._get_component_key(key, pool) + ".bin",
).touch()
operation = PrefetchOperation(
CacheRequestHandle("repair", 0), token_ids, pool_transfers=[swa]
)
controller = SimpleNamespace(
storage_backend=backend,
page_size=2,
prefetch_hits_sync_groups=[],
_all_reduce=capture,
)
_, hit_tokens = HybridCacheController._storage_hit_query(
controller, operation
)
self.assertEqual(operation.all_hash_values, chain)
self.assertEqual(
operation.query_pool_hit_pages,
{
PoolName.KV: 2,
**({PoolName.SWA: swa_pages} if swa_pages else {}),
},
)
HybridCacheController._sync_prefetch_hit_query(
controller, operation, hit_tokens
)
ranks.append((controller, operation, hit_tokens))
# Only transport is simulated: reduce the tensors built by the
# actual controller, then deliver the same MIN to both ranks.
agreed = torch.stack(packed).amin(dim=0)
for rank, (controller, operation, hit_tokens) in enumerate(ranks):
with self.subTest(peer_swa_pages=peer_swa_pages, rank=rank):
controller._all_reduce = lambda values, *_: values.copy_(agreed)
operation.storage_hit_count = (
HybridCacheController._sync_prefetch_hit_query(
controller, operation, hit_tokens
)
)
tree = MagicMock()
tree.build_hicache_transfers.return_value = [swa]
tree.buffer_backup_pool_keys.return_value = {}
pipeline = self._launch_pipeline(tree, [])
cache = pipeline._cache
cache.host_memory_mode, cache.page_size = "buffer_only", 2
cache.components = {
ct: SimpleNamespace(buffer_belief_policy=lambda _: None)
for ct in (ComponentType.FULL, ComponentType.SWA)
}
cache._belief_policy = UnifiedRadixCache._belief_policy.__get__(
cache
)
for pool in (PoolName.KV, PoolName.SWA):
cache.storage_existence_cache.pool(pool).add(chain)
UnifiedRadixCache._invalidate_absent_from_hit_query(
cache, operation
)
intents = pipeline._write_intents(snapshot)
# Retain FULL while making only the missing SWA tail
# eligible, including when one rank omitted its verdict.
self.assertEqual(intents, [(PoolName.SWA, chain[-1:])])
self.assertTrue(
cache.storage_existence_cache.pool(
PoolName.KV
).contains_all(chain)
)
self.assertEqual(
operation.storage_hit_count, peer_swa_pages * 2
)
self.assertEqual(
[
cache.storage_existence_cache.pool(
PoolName.SWA
).contains(h)
for h in chain
],
[i < peer_swa_pages for i in range(2)],
)
# An assumed-stored retry verifies nothing now, so it carries no pool
# verdict and re-learns nothing a shortfall heal dropped.
operation = PrefetchOperation(
CacheRequestHandle("assumed", 0),
token_ids,
pool_transfers=[swa],
assume_stored=True,
)
controller = SimpleNamespace(
storage_backend=None,
page_size=2,
prefetch_hits_sync_groups=[],
_all_reduce=lambda values, *_: None,
)
_, hit_tokens = HybridCacheController._storage_hit_query(controller, operation)
operation.storage_hit_count = HybridCacheController._sync_prefetch_hit_query(
controller, operation, hit_tokens
)
self.assertEqual(
(operation.storage_hit_count, operation.query_pool_hit_pages), (4, {})
)
pipeline = self._launch_pipeline(MagicMock(), [])
cache = pipeline._cache
cache.host_memory_mode, cache.page_size = "buffer_only", 2
cache.components = {
ct: SimpleNamespace(buffer_belief_policy=lambda _: None)
for ct in (ComponentType.FULL, ComponentType.SWA)
}
cache._belief_policy = UnifiedRadixCache._belief_policy.__get__(cache)
cache.storage_existence_cache.pool(PoolName.KV).add(chain)
UnifiedRadixCache._invalidate_absent_from_hit_query(cache, operation)
self.assertTrue(
cache.storage_existence_cache.pool(PoolName.KV).contains_all(chain)
)
self.assertFalse(
any(
cache.storage_existence_cache.pool(PoolName.SWA).contains(h)
for h in chain
)
)
@staticmethod
def _dsv4_specs():
return [
@@ -120,7 +301,39 @@ class TestBufferModeSidecar(unittest.TestCase):
swa_component=self._swa_component(),
)
@staticmethod
def _backup_snapshot():
return BufferBackupSnapshot(
node_id=7,
parent_node_id=0,
parent_is_root=True,
parent_last_hash=None,
hash_values=["page-0", "page-1"],
key=RadixKey(array("q", [1, 2, 3, 4])),
prefix_keys=["page-p"],
)
@staticmethod
def _launch_pipeline(tree, sidecar_pool_specs):
cache = SimpleNamespace(
tree_core=tree,
components={ComponentType.FULL: SimpleNamespace()},
sidecar_pool_specs=sidecar_pool_specs,
storage_existence_cache=StorageExistenceCache(),
)
cache._build_sidecar_transfers = (
UnifiedRadixCache._build_sidecar_transfers.__get__(cache)
)
cache._build_backup_sidecar = UnifiedRadixCache._build_backup_sidecar.__get__(
cache
)
pipeline = BufferModePipeline.__new__(BufferModePipeline)
pipeline._cache = cache
return pipeline
def test_write_stages_and_persists_dsv4_full_and_swa_sidecars(self):
"""Each pool of a node is a write of its own; every sidecar rides its
source pool's write under the source's keys."""
page_size = 2
device_indices = torch.arange(4, dtype=torch.int64)
host_indices = torch.arange(10, 14, dtype=torch.int64)
@@ -139,49 +352,68 @@ class TestBufferModeSidecar(unittest.TestCase):
)
for spec in self._dsv4_specs()
]
kv_sidecars = [s for s in sidecars if s.indices_from_pool == PoolName.KV]
swa_sidecars = [s for s in sidecars if s.indices_from_pool == PoolName.SWA]
controller = MagicMock()
controller.mem_pool_host.anchor_entry.host_pool.shared_allocation_domain = None
controller.mem_pool_host.entry_map = {
PoolName.SWA: SimpleNamespace(
host_pool=SimpleNamespace(page_size=page_size)
host_pool=SimpleNamespace(
page_size=page_size,
size=8,
available_size=lambda: 8,
free=MagicMock(),
)
)
}
controller.mem_pool_host.size = 32
controller.mem_pool_host.anchor_entry.host_pool = SimpleNamespace()
controller.prefetch_tokens_occupied = 0
ack_ids = []
def _write(device_value, node_id, extra_pools, flush):
self.assertEqual(node_id, 7)
self.assertFalse(flush)
self.assertEqual(
[transfer.name for transfer in extra_pools],
[PoolName.SWA, *[transfer.name for transfer in sidecars]],
)
ack_ids.append(node_id)
# HostPoolGroup.resolve_host_transfers gives derived pools their
# source pool's indices without allocating another staging span.
swa.host_indices = swa_host_indices
for sidecar in sidecars:
if sidecar.indices_from_pool == PoolName.KV:
if device_value.numel():
self.assertEqual(extra_pools, kv_sidecars)
for sidecar in extra_pools:
sidecar.host_indices = host_indices
sidecar.device_indices = device_value
else:
sidecar.host_indices = swa_host_indices
sidecar.device_indices = swa_device_indices
return host_indices
return host_indices
self.assertEqual(extra_pools, [swa, *swa_sidecars])
swa.host_indices = swa_host_indices
for sidecar in swa_sidecars:
sidecar.host_indices = swa_host_indices
sidecar.device_indices = swa_device_indices
return device_value
controller.write.side_effect = _write
controller.write_storage.return_value = 99
controller.write_storage.side_effect = [98, 99]
cache = MagicMock()
cache.cache_controller = controller
cache.page_size = page_size
cache.enable_storage = True
cache.enable_storage_metrics = False
cache.hicache_storage_pass_prefix_keys = True
cache.components = {ComponentType.FULL: MagicMock()}
cache.sidecar_pool_specs = self._dsv4_specs()
cache._build_backup_sidecar.return_value = sidecars
cache.storage_existence_cache = StorageExistenceCache()
pipeline = BufferModePipeline.__new__(BufferModePipeline)
pipeline._cache = cache
pipeline.ongoing_write_through = {}
pipeline.ongoing_backup = {}
pipeline.inflight_backup_hashes = {}
pipeline.write_staged_tokens_ = 0
pipeline.write_backlog_tokens_ = len(device_indices)
pipeline._swa_window_pages = 1
pipeline.write_backlog_cap = 8
pipeline.reset()
pipeline._build_backup_transfers = lambda node_id, *, kv_only=False: (
device_indices,
{ComponentType.SWA: [swa]},
None,
)
hashes = ["page-0", "page-1"]
snapshot = BufferBackupSnapshot(
@@ -191,24 +423,51 @@ class TestBufferModeSidecar(unittest.TestCase):
parent_last_hash=None,
hash_values=hashes,
key=RadixKey(array("q", [1, 2, 3, 4])),
prefix_keys=None,
prefix_keys=["page-p"],
)
intent = _UnifiedBackupIntent(snapshot=snapshot)
self.assertTrue(
pipeline._stage_backup_intent(
intent,
device_indices,
comp_xfers={ComponentType.SWA: [swa]},
)
tree = cache.tree_core
tree.snapshot_buffer_backup.return_value = snapshot
tree.validate_buffer_backup.return_value = BufferBackupState(0, True, None)
tree.buffer_backup_pool_keys.return_value = {
ComponentType.SWA: {PoolName.SWA: hashes[-1:]}
}
pipeline.enqueue_backup_intent(7)
self.assertEqual(
[intent.pool for intent in pipeline.pending_write_queue],
[PoolName.KV, PoolName.SWA],
)
self.assertEqual(pipeline.ongoing_write_through[7].aux_xfers, [swa, *sidecars])
pipeline.flush_pending_writes()
self.assertFalse(pipeline.pending_write_queue)
kv_ack, swa_ack = ack_ids
self.assertEqual(pipeline.ongoing_write_through[kv_ack].aux_xfers, kv_sidecars)
self.assertEqual(
pipeline.ongoing_write_through[swa_ack].aux_xfers, [swa, *swa_sidecars]
)
expected_keys = {PoolName.KV: hashes, PoolName.SWA: hashes[-1:]}
for spec in self._dsv4_specs():
expected_keys[spec.pool_name] = expected_keys[spec.indices_from_pool]
for pool, keys in expected_keys.items():
# In flight (covered for admission) but not yet believed stored.
pool_beliefs = cache.storage_existence_cache.pool(pool)
self.assertTrue(pool_beliefs.covered(keys))
self.assertFalse(pool_beliefs.contains_all(keys))
self.assertEqual(pipeline.write_backlog_tokens_, 0)
pipeline.finish_backup_ack(7)
pipeline.finish_backup_ack(kv_ack)
pipeline.finish_backup_ack(swa_ack)
kv_call, swa_call = controller.write_storage.call_args_list
self.assertTrue(torch.equal(kv_call.args[0], host_indices))
self.assertEqual(kv_call.args[2], hashes)
self.assertEqual(kv_call.args[3], ["page-p"])
# No KV page in the SWA write; the window's keys ride the transfer.
self.assertEqual(swa_call.args[0].numel(), 0)
self.assertEqual(swa_call.args[2], [])
self.assertEqual(swa_call.args[3], ["page-p"])
storage_transfers = {
transfer.name: transfer
for transfer in controller.write_storage.call_args.kwargs["extra_pools"]
for call in (kv_call, swa_call)
for transfer in call.kwargs["extra_pools"]
}
self.assertEqual(storage_transfers[PoolName.SWA].keys, [hashes[-1]])
self.assertTrue(
@@ -216,14 +475,16 @@ class TestBufferModeSidecar(unittest.TestCase):
)
for spec in self._dsv4_specs():
storage_sidecar = storage_transfers[spec.pool_name]
expected_keys = (
hashes if spec.indices_from_pool == PoolName.KV else [hashes[-1]]
)
self.assertEqual(storage_sidecar.keys, expected_keys)
self.assertEqual(storage_sidecar.keys, expected_keys[spec.pool_name])
self.assertEqual(storage_sidecar.hit_policy, spec.hit_policy)
self.assertEqual(storage_sidecar.indices_from_pool, spec.indices_from_pool)
self.assertIsNone(storage_sidecar.host_indices)
self.assertIn(99, pipeline.ongoing_backup)
self.assertEqual(set(pipeline.ongoing_backup), {98, 99})
pipeline.finish_storage_write_ack(98)
self.assertTrue(pipeline.backup_pending(7))
pipeline.finish_storage_write_ack(99)
self.assertFalse(pipeline.backup_pending(7))
self.assertEqual(pipeline.write_staged_tokens_, 0)
def test_completed_prefetch_keeps_dsv4_full_and_swa_sidecars_for_h2d(self):
swa = PoolTransfer(
@@ -62,6 +62,7 @@ from sglang.srt.mem_cache.memory_pool import (
MHATokenToKVPool,
ReqToTokenPool,
)
from sglang.srt.mem_cache.pool_host import PoolEntry
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -77,11 +78,13 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
SWARebuild,
)
from sglang.srt.mem_cache.unified_cache.components.base import (
BASE_COMPONENT_TYPE,
CacheTransferPhase,
ComponentType,
EvictLayer,
TreeComponent,
)
from sglang.srt.mem_cache.unified_cache.components.full import FullComponent
from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment
from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
_TREE_CORE_REGISTRY,
@@ -238,6 +241,46 @@ class _FakeFullComponent(TreeComponent):
pass
class _ExtraPoolFullComponent(FullComponent):
"""FULL component that also stores every page under a pool of its own,
staged through the anchor's host pool."""
EXTRA_POOL = PoolName.DRAFT
def buffer_mode_host_pool_entries(self, host_pool_group):
anchor = host_pool_group.anchor_entry
return [
PoolEntry(
name=self.EXTRA_POOL,
host_pool=anchor.host_pool,
device_pool=anchor.device_pool,
layer_mapper=anchor.layer_mapper,
)
]
def buffer_backup_keys(self, node, hash_values):
return {self.EXTRA_POOL: list(hash_values)}
def build_hicache_transfers(self, node, phase, **kwargs):
if phase == CacheTransferPhase.BACKUP_HOST:
# Two transfers of the pool, each keyed by its own pages: the
# storage write must keep the keys per transfer.
value = node.component_data[BASE_COMPONENT_TYPE].value
keys = list(node.hash_value)
split = len(keys) // 2
page_size = self.tree_core.page_size
halves = [
(value[: split * page_size], keys[:split]),
(value[split * page_size :], keys[split:]),
]
return [
PoolTransfer(name=self.EXTRA_POOL, device_indices=rows, keys=part)
for rows, part in halves
if part
]
return super().build_hicache_transfers(node, phase, **kwargs)
def _drop_hicache_atexit_pin(cache):
"""init_hicache registers ``cache.shutdown`` with atexit, which pins the
fixture's device and host pools until process exit; hundreds of HiCache
@@ -468,6 +511,9 @@ def build_fixture(
enable_session_radix_cache: bool = False,
tree_page_size: Optional[int] = None,
mamba_cache_chunk_size: Optional[int] = None,
component_registry_override: Optional[
dict[ComponentType, type[TreeComponent]]
] = None,
tree_core_backend: Optional[str] = None,
):
"""Create (tree, allocator, req_to_token_pool) from a CacheConfig.
@@ -602,6 +648,7 @@ def build_fixture(
enable_session_radix_cache=enable_session_radix_cache,
eviction_policy=cfg.eviction_policy,
is_eagle=cfg.is_eagle,
component_registry_override=component_registry_override,
)
requested_backend = tree_core_backend or _selected_tree_core_test_backend()
selected_backend = resolve_tree_core_backend(requested_backend, cache_init_params)
@@ -1135,13 +1182,18 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase):
# every in-flight backup.
lock_params = cache.inc_lock_ref(leaf_id).to_dec_params()
pipeline = BufferModePipeline.__new__(BufferModePipeline)
keys = list(snapshot.hash_values)
# Entries are keyed by the write's ack id, not by the node.
pipeline.ongoing_write_through = {
leaf_id: _UnifiedBufferBackupEntry(
intent=_UnifiedBackupIntent(snapshot=snapshot),
-1: _UnifiedBufferBackupEntry(
intent=_UnifiedBackupIntent(
snapshot=snapshot, pool=PoolName.KV, keys=keys
),
host_indices=torch.empty(0, dtype=torch.int64),
aux_xfers=[],
lock_params=lock_params,
occupied_units=0,
keys_by_pool={PoolName.KV: keys},
)
}
cache.buffer_pipeline = pipeline
@@ -4031,8 +4083,14 @@ class UnifiedRadixCacheSuite:
pipeline = cache.buffer_pipeline
controller = cache.cache_controller
_write_backup(cache, leaf)
node_ids = [intent.snapshot.node_id for intent in pipeline.pending_write_queue]
# One intent per pool of every node on the path.
intents = list(pipeline.pending_write_queue)
node_ids = sorted({intent.snapshot.node_id for intent in intents})
self.assertGreaterEqual(len(node_ids), 2)
self.assertEqual(
pipeline.inflight_backup_pools[leaf],
{intent.pool for intent in intents if intent.snapshot.node_id == leaf},
)
device_allocators = [allocator]
if self.cfg.has_swa:
@@ -4064,9 +4122,19 @@ class UnifiedRadixCacheSuite:
self.assertFalse(pipeline.pending_write_queue)
self.assertFalse(controller.write_queue)
self.assertEqual(len(controller.ack_write_queue), 1)
# Every intent is a write of its own, acked under its own id.
ack = controller.ack_write_queue[0]
self.assertEqual(ack.node_ids, node_ids)
self.assertEqual(set(pipeline.ongoing_write_through), set(node_ids))
self.assertEqual(len(ack.node_ids), len(intents))
self.assertEqual(list(ack.node_ids), list(pipeline.ongoing_write_through))
self.assertEqual(
sorted(
{
entry.intent.snapshot.node_id
for entry in pipeline.ongoing_write_through.values()
}
),
node_ids,
)
staged_avail = self._host_avail_sizes(cache)
self.assertNotEqual(staged_avail, avail0)
@@ -4076,7 +4144,7 @@ class UnifiedRadixCacheSuite:
self.assertGreater(_device_lock_ref(cache, node_id, ComponentType.FULL), 0)
cache.writing_check(finish_count=1)
self.assertFalse(pipeline.ongoing_write_through)
self.assertEqual(len(pipeline.ongoing_backup), len(node_ids))
self.assertEqual(len(pipeline.ongoing_backup), len(intents))
for node_id in node_ids:
self.assertEqual(_device_lock_ref(cache, node_id, ComponentType.FULL), 0)
# Each storage write still owns its staging until its separate ACK.
@@ -4103,13 +4171,259 @@ class UnifiedRadixCacheSuite:
)
for n in chain:
self.assertTrue(
cache.storage_existence_cache.contains_all(
PoolName.KV, cache.tree_core.get_hash_values(n)
cache.storage_existence_cache.pool(PoolName.KV).contains_all(
cache.tree_core.get_hash_values(n)
)
)
# Re-hit absorbed by the (FULL-focused) belief skip.
for transfer in _aux_storage_key_transfers(cache, n) or ():
self.assertTrue(
cache.storage_existence_cache.pool(transfer.name).contains_all(
transfer.keys
)
)
# Re-hit absorbed when every pool is believed present.
_write_backup(cache, leaf)
self.assertNotIn(leaf, cache.buffer_pipeline.inflight_backup_node_ids)
if self.cfg.has_swa:
# Only the SWA window dropped: the next insert queues the SWA pool
# alone. A private SWA arena's write proceeds even with FULL staging
# at the live cap (shared or anchor-aliased layouts keep the gate).
swa_transfer = _aux_storage_key_transfers(cache, leaf)[0]
cache.storage_existence_cache.pool(PoolName.SWA).invalidate_beyond(
swa_transfer.keys, keep_pages=0
)
host_group = controller.mem_pool_host
private_swa = (
pipeline._shared_host_domain() is None
and host_group.entry_map[PoolName.SWA].host_pool
is not host_group.anchor_entry.host_pool
)
staged_before = pipeline.write_staged_tokens_
if private_swa:
pipeline.write_staged_tokens_ = host_group.size
with (
mock.patch.object(controller, "write", wraps=controller.write) as stage,
mock.patch.object(
controller, "write_storage", wraps=controller.write_storage
) as write_storage,
mock.patch.object(
controller, "page_set_func", wraps=controller.page_set_func
) as write_full,
mock.patch.object(
pipeline, "_write_intents", wraps=pipeline._write_intents
) as admissions,
):
# The fixture parks the stock trigger; arm it here.
cache.write_through_threshold = 1
self._insert(cache, allocator, req_to_token_pool, seq_ab)
self.assertEqual(
[
(i.snapshot.node_id, i.pool)
for i in pipeline.pending_write_queue
],
[(leaf, PoolName.SWA)],
)
# One admission per node per walk.
self.assertEqual(admissions.call_count, len(node_ids))
self.assertTrue(cache.buffer_backup_pending(leaf))
self._pump_hicache_until(
cache,
lambda: (
not pipeline.inflight_backup_node_ids
and not pipeline.ongoing_backup
),
"selective buffer backup did not drain",
)
self.assertFalse(cache.buffer_backup_pending(leaf))
if private_swa:
pipeline.write_staged_tokens_ -= host_group.size - staged_before
self.assertTrue(stage.called)
self.assertTrue(
all(call.args[0].numel() == 0 for call in stage.call_args_list)
)
# SWA writes carry no KV page; the window's keys ride the transfer.
self.assertTrue(write_storage.called)
snapshot = cache.tree_core.snapshot_buffer_backup(
leaf, cache.hicache_storage_pass_prefix_keys
)
for call in write_storage.call_args_list:
host_indices, _, hash_value, prefix_keys = call.args[:4]
self.assertEqual(host_indices.numel(), 0)
self.assertEqual(hash_value, [])
transfers = call.kwargs["extra_pools"]
self.assertEqual(
[t.name for t in transfers if t.indices_from_pool is None],
[PoolName.SWA],
)
self.assertEqual(prefix_keys, snapshot.prefix_keys)
write_full.assert_not_called()
self.assertTrue(
cache.storage_existence_cache.pool(PoolName.SWA).contains_all(
swa_transfer.keys
)
)
self.assertEqual(self._host_avail_sizes(cache), avail0)
cache.sanity_check()
@staticmethod
def _writes_by_pool(write_storage, extra):
"""Key tuples of the KV writes and of the ``extra`` pool's writes
among the storage writes recorded by ``write_storage``."""
kv_writes, extra_writes = set(), set()
for call in write_storage.call_args_list:
host_indices, _, hash_value = call.args[:3]
transfers = call.kwargs.get("extra_pools") or []
names = [transfer.name for transfer in transfers]
if hash_value:
assert extra not in names
kv_writes.add(tuple(hash_value))
elif extra in names:
# The pool's transfers reach storage as one contiguous span.
assert host_indices.numel() == 0 and names == [extra]
extra_writes.add(tuple(transfers[0].keys))
return kv_writes, extra_writes
def test_buffer_only_writes_component_pools_independently(self):
"""A component's own pool (aliased host entry, per-pool keys,
BACKUP_HOST transfers) is written, believed and re-written
independently of the KV pool."""
self._skip_unsupported_hicache_test()
storage_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, storage_dir, ignore_errors=True)
# Component buffer-mode hooks are Python-core only.
cache, allocator, req_to_token_pool = build_fixture(
self.cfg,
component_registry_override={ComponentType.FULL: _ExtraPoolFullComponent},
tree_core_backend="python",
)
self._init_buffer_hicache(cache, storage_dir)
extra = _ExtraPoolFullComponent.EXTRA_POOL
controller = cache.cache_controller
self.assertIn(extra, controller.mem_pool_host.entry_map)
avail0 = self._host_avail_sizes(cache)
seq = self._buffer_swa_seq()
self._insert(cache, allocator, req_to_token_pool, seq)
leaf = cache.match_prefix(
MatchPrefixParams(key=RadixKey(array("q", seq)))
).last_device_node
chain = self._path_chain(cache, leaf)
pipeline = cache.buffer_pipeline
expected_intents = []
expected_backlog = 0
for node_id in chain:
snapshot = cache.tree_core.snapshot_buffer_backup(
node_id, cache.hicache_storage_pass_prefix_keys
)
expected_backlog += len(snapshot.key)
expected_intents.extend(
(node_id, pool) for pool, _ in pipeline._write_intents(snapshot)
)
pipeline.write_backlog_cap = expected_backlog
if self.cfg.has_swa:
self.assertEqual(
{pool for node_id, pool in expected_intents if node_id == leaf},
{PoolName.KV, PoolName.DRAFT, PoolName.SWA},
)
self.assertFalse(cache.buffer_backup_pending(leaf))
with mock.patch.object(
controller, "write_storage", wraps=controller.write_storage
) as write_storage:
_write_backup(cache, leaf)
self.assertEqual(
[
(intent.snapshot.node_id, intent.pool)
for intent in pipeline.pending_write_queue
],
expected_intents,
)
self.assertEqual(pipeline.write_backlog_tokens_, expected_backlog)
self.assertEqual(pipeline._backlog_cap_hits, 0)
self.assertTrue(cache.buffer_backup_pending(leaf))
self._pump_hicache_until(
cache,
lambda: (
not cache.buffer_pipeline.inflight_backup_node_ids
and not cache.buffer_pipeline.ongoing_backup
),
"buffer backup pipeline did not drain",
)
self.assertFalse(cache.buffer_backup_pending(leaf))
# Each pool of each node is a write of its own: the KV chain with its
# pages, the component's pool with no KV rows and its transfers
# keyed by their own pages.
expected = {tuple(cache.tree_core.get_hash_values(n)) for n in chain}
kv_writes, extra_writes = self._writes_by_pool(write_storage, extra)
self.assertEqual(kv_writes, expected)
self.assertEqual(extra_writes, expected)
page_hashes = self._all_page_hashes(cache, leaf)
self.assertEqual(
self._storage_exists_count(
cache, page_hashes, [PoolTransfer(name=extra, keys=page_hashes)]
),
len(page_hashes),
)
for n in chain:
keys = cache.tree_core.get_hash_values(n)
self.assertTrue(
cache.storage_existence_cache.pool(PoolName.KV).contains_all(keys)
)
self.assertTrue(
cache.storage_existence_cache.pool(extra).contains_all(keys)
)
self.assertEqual(self._host_avail_sizes(cache), avail0)
# Only the component's pool dropped from the beliefs: its requested
# backup writes that pool alone, staged through the anchor's host
# pool with no KV rows.
for n in chain:
cache.storage_existence_cache.pool(extra).invalidate_beyond(
cache.tree_core.get_hash_values(n), keep_pages=0
)
with (
mock.patch.object(controller, "write", wraps=controller.write) as stage,
mock.patch.object(
controller, "write_storage", wraps=controller.write_storage
) as write_storage,
mock.patch.object(
controller, "page_set_func", wraps=controller.page_set_func
) as write_full,
):
cache.request_buffer_backup(leaf)
intents = list(pipeline.pending_write_queue)
self.assertEqual(
[(intent.snapshot.node_id, intent.pool) for intent in intents],
[(n, extra) for n in chain],
)
self._pump_hicache_until(
cache,
lambda: (
not pipeline.inflight_backup_node_ids
and not pipeline.ongoing_backup
),
"component-pool backup did not drain",
)
self.assertTrue(stage.called)
self.assertTrue(all(call.args[0].numel() == 0 for call in stage.call_args_list))
self.assertEqual(len(write_storage.call_args_list), len(chain))
kv_writes, extra_writes = self._writes_by_pool(write_storage, extra)
self.assertEqual(kv_writes, set())
self.assertEqual(extra_writes, expected)
write_full.assert_not_called()
for n in chain:
self.assertTrue(
cache.storage_existence_cache.pool(extra).contains_all(
cache.tree_core.get_hash_values(n)
)
)
self.assertEqual(self._host_avail_sizes(cache), avail0)
# Everything believed stored: a requested backup is absorbed.
cache.request_buffer_backup(leaf)
self.assertNotIn(leaf, pipeline.inflight_backup_node_ids)
cache.sanity_check()
def test_buffer_only_read_path_roundtrip(self):
@@ -4228,10 +4542,18 @@ class UnifiedRadixCacheSuite:
for component_type in cons.tree_components:
self.assertIsNone(_host_value(cons, cur, component_type))
self.assertTrue(
cons.storage_existence_cache.contains_all(
PoolName.KV, self._all_page_hashes(cons, leaf)
cons.storage_existence_cache.pool(PoolName.KV).contains_all(
self._all_page_hashes(cons, leaf)
)
)
# The fetch is evidence for every pool it delivered, so the
# consumer's own re-write of the span is skipped pool by pool.
for transfer in _aux_storage_key_transfers(cons, leaf) or ():
self.assertTrue(
cons.storage_existence_cache.pool(transfer.name).contains_all(
transfer.keys
)
)
loaded_k, loaded_v = self._snapshot_full_kv(cons_alloc, mc.device_indices)
self.assertTrue(torch.equal(loaded_k, expected_k))
self.assertTrue(torch.equal(loaded_v, expected_v))