[HiCache] Manage buffer-mode backups per storage pool (#42633)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-06 16:45:05 -07:00
committed by GitHub
co-authored by merrymercy xiezhq-hermann
parent fd62ee84f7
commit 08aad0c196
11 changed files with 1495 additions and 415 deletions
+13 -8
View File
@@ -1295,15 +1295,9 @@ class HiCacheController:
)
hash_value, storage_hit_count = [], 0
storage_hit_count_tensor = torch.tensor(
storage_hit_count, dtype=torch.int
storage_hit_count = self._sync_prefetch_hit_query(
operation, storage_hit_count
)
self._all_reduce(
storage_hit_count_tensor,
torch.distributed.ReduceOp.MIN,
self.prefetch_hits_sync_groups,
)
storage_hit_count = storage_hit_count_tensor.item()
# Record the TP-synced hit count; the scheduler thread decides
# at drain time whether to revoke (below threshold) or allocate.
@@ -1316,6 +1310,17 @@ class HiCacheController:
except Empty:
continue
def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int:
"""Rank-reduce a hit query (MIN) so every rank agrees on the usable
prefix. Runs for every operation, hit or miss."""
hit_count = torch.tensor(storage_hit_count, dtype=torch.int)
self._all_reduce(
hit_count,
torch.distributed.ReduceOp.MIN,
self.prefetch_hits_sync_groups,
)
return int(hit_count.item())
def write_storage(
self,
host_indices: torch.Tensor,
@@ -81,23 +81,28 @@ def _minimum_full_transfer_tokens(
class _UnifiedBackupIntent(msgspec.Struct):
"""Buffer-mode backup intent, unpinned while queued.
"""One pool of one node queued for a buffer-mode write (unpinned while
queued); a derived sidecar rides its source pool's intent.
Snapshots node identity at enqueue time: a split rewrites the node's
key/hash in place while these copies stay intact, so a key-length change
detects a split and a missing FULL device value detects eviction
(``_validate_backup_intent``).
(``_validate_backup_intent``). ``keys`` is the admission view (it sizes
drop metrics); the D2H launch re-reads rows and keys off the node.
"""
snapshot: BufferBackupSnapshot
pool: PoolName
keys: list[str]
class _UnifiedBufferBackupEntry(msgspec.Struct):
"""A buffer-mode backup after its D2H launch: intent + staging slots.
``host_indices`` are FULL-pool staging slots; ``aux_xfers`` carry the
staged aux-pool slots (e.g. the SWA window). All are freed at the
storage-write ack — host memory is never retained as a cache tier.
``host_indices`` are the KV staging slots (empty for another pool's
intent); ``aux_xfers`` the staged transfers and riding sidecars; all are
freed at the storage-write ack (host memory is never a cache tier).
``keys_by_pool`` are the keys the write stores, learned at the ack.
"""
intent: _UnifiedBackupIntent
@@ -105,6 +110,7 @@ class _UnifiedBufferBackupEntry(msgspec.Struct):
aux_xfers: list[PoolTransfer]
lock_params: DecLockRefParams
occupied_units: int
keys_by_pool: dict[PoolName, list[str]]
class _StagedPrefetch(msgspec.Struct):
@@ -147,24 +153,6 @@ class _AnchorLock(msgspec.Struct):
tokens: int
def _track_content_refs(refs: dict[str, int], hash_values: list[str]) -> None:
"""Add one content ref per page hash (at D2H launch). Refcounted,
not a flag: several launched entries can carry the same content
(duplicate staging of republished spans)."""
for h in hash_values:
refs[h] = refs.get(h, 0) + 1
def _untrack_content_refs(refs: dict[str, int], hash_values: list[str]) -> None:
"""Drop one content ref per page hash (at storage-ack)."""
for h in hash_values:
n = refs.get(h, 0) - 1
if n <= 0:
refs.pop(h, None)
else:
refs[h] = n
def validate_buffer_only_stack(
sidecar_pool_specs: list[SidecarPoolSpec],
host_pool_group: HostPoolGroup,
@@ -270,8 +258,8 @@ class BufferModePipeline:
# a host pool (0 = KV-only: no trailing window staged). Static after
# pool assembly.
self._swa_window_pages = swa_window_pages
# Metadata-only pending-write backlog cap; beyond it new intents
# are dropped at admission (re-trigger on a later hit).
# Metadata-only pending-write backlog cap, measured in unique FULL
# snapshot spans; beyond it new spans are dropped at admission.
self.write_backlog_cap = write_backlog_cap
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -318,19 +306,18 @@ class BufferModePipeline:
] = {}
self.staged_prefetches: dict[CacheRequestHandle, _StagedPrefetch] = {}
self.ongoing_buffer_load_back: dict[int, _OngoingBufferLoadBack] = {}
# Backup pipeline: FIFO intents awaiting a D2H slot, node ids
# anywhere in flight (dedupes re-triggers), and a content refcount
# of every page hash between D2H launch and storage-ack — admission
# skips content covered by beliefs + launched writes.
# FIFO intents (one per pool of a node) awaiting a D2H slot, and the
# pools in flight per node (admission to storage ack).
self.pending_write_queue: deque[_UnifiedBackupIntent] = deque()
self.inflight_backup_node_ids: set[int] = set()
self.inflight_backup_hashes: dict[str, int] = {}
# Backups between D2H launch and D2H ack (keyed by node id), then
# between storage-write launch and storage ack (keyed by operation
# id). Mirrors the cache-mode ongoing_write_through/ongoing_backup
# stages, with buffer entries.
self._queued_span_refs: dict[tuple[NodeId, int], int] = {}
self.inflight_backup_pools: dict[NodeId, set[PoolName]] = {}
# Nodes admitted during the current insert walk (None outside one).
self._walk_admitted: Optional[set[NodeId]] = None
# Launched writes: D2H in flight (by ack id), then the storage write
# in flight (by operation id).
self.ongoing_write_through: dict[int, _UnifiedBufferBackupEntry] = {}
self.ongoing_backup: dict[int, _UnifiedBufferBackupEntry] = {}
self._backup_ack_seq = 0
self.write_staged_tokens_ = 0
self.write_backlog_tokens_ = 0
self._backlog_cap_hits = 0
@@ -391,7 +378,9 @@ class BufferModePipeline:
cc = self._cache.cache_controller
anchor = cc.mem_pool_host.anchor_entry.host_pool
if self._shared_host_domain() is None:
return kv_tokens
# Aux transfers staged through the anchor's own host pool (an
# entry aliasing it) occupy anchor tokens as well.
return kv_tokens + self._anchor_pool_aux_tokens(aux_xfers or [])
num_bytes = sum(
tokens * pool.size_per_token
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
@@ -420,6 +409,18 @@ class BufferModePipeline:
)
return requests
@property
def inflight_backup_node_ids(self):
"""Nodes with a write of any pool queued, staged or awaiting its
storage ack."""
return self.inflight_backup_pools.keys()
def backup_pending(self, node_id: NodeId) -> bool:
"""A write of ``node_id`` is queued, staged or awaiting its storage
ack. A component that drops rows a pending write still reads asks
here first; the pipeline pins the node only from D2H launch to ack."""
return node_id in self.inflight_backup_pools
def is_idle(self) -> bool:
"""No queued, staged, or in-flight buffer-mode work or anchor pins."""
return not (
@@ -444,19 +445,37 @@ class BufferModePipeline:
# ---- backup pipeline (device -> staging -> storage) ----
def _backup_parent_covered(self, state: BufferBackupState) -> bool:
def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int:
"""Staging tokens aux transfers hold in the anchor's host pool."""
group = self._cache.cache_controller.mem_pool_host
anchor_pool = group.anchor_entry.host_pool
tokens = 0
for t in aux_xfers:
if t.indices_from_pool is not None:
continue
entry = group.entry_map.get(t.name)
if entry is not None and entry.host_pool is anchor_pool:
tokens += self._transfer_tokens(t)
return tokens
def _backup_parent_covered(self, state: BufferBackupState, node_id: NodeId) -> bool:
"""Only admit a node whose parent is stored/in-flight: writing above
a dropped parent creates a permanent longest-prefix hole."""
if (
state.parent_is_root
or state.parent_node_id in self.inflight_backup_node_ids
a dropped parent creates a permanent longest-prefix hole. A component
may vouch for the parent through a pool of its own."""
if state.parent_is_root or PoolName.KV in self.inflight_backup_pools.get(
state.parent_node_id, ()
):
return True
return (
if (
state.parent_last_hash is not None
and self._cache.storage_existence_cache.contains(
PoolName.KV, state.parent_last_hash
and self._cache.storage_existence_cache.pool(PoolName.KV).contains(
state.parent_last_hash
)
):
return True
return any(
component.buffer_backup_parent_covered(state.parent_node_id, node_id)
for component in self._cache.components.values()
)
def _log_backup_dropped(self, num_tokens: int) -> None:
@@ -464,129 +483,229 @@ class BufferModePipeline:
if cache.enable_storage_metrics and cache.storage_metrics_collector is not None:
cache.storage_metrics_collector.log_backup_dropped_tokens(num_tokens)
def _pool_covered(self, pool: PoolName, keys: list[str]) -> bool:
"""Beliefs plus content past a D2H launch (republished content must
not re-write while the original write drains)."""
return self._cache.storage_existence_cache.pool(pool).covered(keys)
def _transfer_keys(
self, transfer: PoolTransfer, hash_values: list[str]
) -> Optional[list[str]]:
"""The builder's keys, else the chain's trailing pages, one per staged
page (the SWA convention); None when the transfer stages nothing."""
if transfer.keys is not None:
return list(transfer.keys)
if transfer.indices_from_pool is not None or transfer.device_indices is None:
return None
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
return None
num_pages = len(transfer.device_indices) // entry.host_pool.page_size
if num_pages == 0 or num_pages > len(hash_values):
return None
return list(hash_values[-num_pages:])
def _backup_pool_keys(
self, snapshot: BufferBackupSnapshot
) -> dict[PoolName, list[str]]:
"""Admission view of the keys a backup writes per pool: the KV chain,
each aux component's BACKUP_HOST transfers, component overrides on
top, every sidecar under its source's keys."""
hash_values = snapshot.hash_values
tree_core = self._cache.tree_core
pool_keys: dict[PoolName, list[str]] = {PoolName.KV: list(hash_values)}
overrides = tree_core.buffer_backup_pool_keys(snapshot.node_id, hash_values)
for ct in self._cache.components:
# A component that names its pools' keys spares the transfer build.
if ct == ComponentType.FULL or ct in overrides:
continue
transfers = tree_core.build_hicache_transfers(
ct, snapshot.node_id, CacheTransferPhase.BACKUP_HOST
)
for transfer in transfers or ():
keys = self._transfer_keys(transfer, hash_values)
if keys:
pool_keys.setdefault(transfer.name, []).extend(keys)
for named in overrides.values():
for pool, keys in named.items():
if keys:
pool_keys[pool] = list(keys)
else:
pool_keys.pop(pool, None)
for spec in self._cache.sidecar_pool_specs:
source = pool_keys.get(spec.indices_from_pool)
if source:
pool_keys[spec.pool_name] = list(source)
return pool_keys
@staticmethod
def _pool_order(pool: PoolName) -> tuple[bool, str]:
# KV first, then a fixed order: every rank must admit identically.
return (pool != PoolName.KV, pool.value)
def _write_intents(
self, snapshot: BufferBackupSnapshot
) -> list[tuple[PoolName, list[str]]]:
"""(pool, keys) of the pools not believed stored or in flight; a
sidecar riding a covered pool drives that pool too."""
pool_keys = self._backup_pool_keys(snapshot)
sidecars = {
spec.pool_name: spec.indices_from_pool
for spec in self._cache.sidecar_pool_specs
}
intents: list[tuple[PoolName, list[str]]] = []
for pool in sorted(pool_keys, key=self._pool_order):
keys = pool_keys[pool]
if pool in sidecars or not keys:
continue
riders = [
name
for name, source in sidecars.items()
if source == pool and pool_keys.get(name)
]
if all(
self._pool_covered(name, pool_keys[name]) for name in (pool, *riders)
):
continue
intents.append((pool, keys))
return intents
def _sizing(
self, pool: PoolName, keys: list[str]
) -> tuple[int, list[PoolTransfer]]:
"""(KV tokens, keys-only aux transfers) an intent stages, for the
oversize and budget gates."""
if pool == PoolName.KV:
return len(keys) * self._cache.page_size, []
return 0, [PoolTransfer(name=pool, keys=list(keys))]
def _finish_inflight(self, node_id: NodeId, pool: PoolName) -> None:
pools = self.inflight_backup_pools.get(node_id)
if pools is None:
return
pools.discard(pool)
if not pools:
del self.inflight_backup_pools[node_id]
@staticmethod
def _queued_span_key(intent: _UnifiedBackupIntent) -> tuple[NodeId, int]:
snapshot = intent.snapshot
return snapshot.node_id, len(snapshot.key)
def _retain_queued_span(self, intent: _UnifiedBackupIntent) -> None:
span = self._queued_span_key(intent)
refs = self._queued_span_refs.get(span, 0)
if refs == 0:
self.write_backlog_tokens_ += span[1]
self._queued_span_refs[span] = refs + 1
def _release_queued_span(self, intent: _UnifiedBackupIntent) -> None:
span = self._queued_span_key(intent)
refs = self._queued_span_refs[span]
if refs == 1:
del self._queued_span_refs[span]
self.write_backlog_tokens_ -= span[1]
else:
self._queued_span_refs[span] = refs - 1
def begin_insert_walk(self) -> None:
"""Admit each node once per insert walk: the trigger chains every
unbacked ancestor, so a depth-d path is presented d(d+1)/2 times."""
self._walk_admitted = set()
def end_insert_walk(self) -> None:
self._walk_admitted = None
def enqueue_backup_intent(self, node_id: NodeId) -> None:
"""Snapshot a backup intent and commit it to the write queue.
Admission gates: belief skip, parent-cover, backlog cap, oversize.
Rejected intents are counted; the node re-triggers on a later hit."""
"""Queue one intent per pool with something to write; rejected intents
are counted and the node re-triggers on a later hit."""
if not self._cache.enable_storage:
return
if node_id in self.inflight_backup_node_ids:
return
admitted = self._walk_admitted
if admitted is not None:
if node_id in admitted:
return
admitted.add(node_id)
snapshot = self._cache.tree_core.snapshot_buffer_backup(
node_id, self._cache.hicache_storage_pass_prefix_keys
)
if snapshot is None:
return
# Admission cover: beliefs plus content past its D2H launch. The
# launched cover keeps republished content (fill inserts under new
# node ids) from re-writing while the original write drains.
if self._cache.storage_existence_cache.covers_all(
PoolName.KV,
snapshot.hash_values,
extra_cover=self.inflight_backup_hashes,
):
intents = self._write_intents(snapshot)
if not intents:
return
intent_tokens = len(snapshot.hash_values) * self._cache.page_size
if self.write_backlog_tokens_ >= self.write_backlog_cap:
# The cap sits at 2x the intrinsic live-backlog ceiling (see
# init_hicache), so reaching it means leaked accounting or a
# broken stale sweep — a bug, not load.
self._backlog_cap_hits += 1
if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0:
logger.error(
"HiCache write backlog cap hit (occurrence %d): "
"backlog=%d cap=%d queue=%d. Live backlog is bounded "
"by the device pool span, so this indicates a "
"stale-sweep or accounting leak.",
self._backlog_cap_hits,
self.write_backlog_tokens_,
self.write_backlog_cap,
len(self.pending_write_queue),
)
self._log_backup_dropped(intent_tokens)
return
# A span larger than any pool's whole staging capacity can never
# stage; admitting it would wedge the head-of-line queue forever.
state = BufferBackupState(
parent_node_id=snapshot.parent_node_id,
parent_is_root=snapshot.parent_is_root,
parent_last_hash=snapshot.parent_last_hash,
pending = self.inflight_backup_pools.get(node_id, set())
parent_covered = self._backup_parent_covered(
BufferBackupState(
parent_node_id=snapshot.parent_node_id,
parent_is_root=snapshot.parent_is_root,
parent_last_hash=snapshot.parent_last_hash,
),
node_id,
)
if not self._backup_parent_covered(state) or self._backup_oversize(
snapshot.node_id, snapshot.hash_values, intent_tokens
):
self._log_backup_dropped(intent_tokens)
return
intent = _UnifiedBackupIntent(snapshot=snapshot)
self.pending_write_queue.append(intent)
self.inflight_backup_node_ids.add(snapshot.node_id)
self.write_backlog_tokens_ += intent_tokens
def _build_aux_staging_transfers(
self,
node_id: NodeId,
hash_values: list[str],
comp_xfers: Optional[dict[ComponentType, list[PoolTransfer]]] = None,
) -> list[PoolTransfer]:
"""Keys-only aux transfers mirroring what BACKUP_STORAGE would write;
sizes the per-pool oversize gate (beliefs do not consult these)."""
transfers: list[PoolTransfer] = []
if ComponentType.SWA in self._cache.components:
current = (
comp_xfers.get(ComponentType.SWA)
if comp_xfers is not None
else self._cache.tree_core.build_hicache_transfers(
ComponentType.SWA,
node_id,
CacheTransferPhase.BACKUP_HOST,
)
)
for transfer in current or ():
if transfer.device_indices is None:
continue
num_pages = len(transfer.device_indices) // self._cache.page_size
if num_pages > 0:
transfers.append(
PoolTransfer(
name=PoolName.SWA,
keys=hash_values[-num_pages:],
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
page_size = self._cache.page_size
span = (snapshot.node_id, len(snapshot.key))
for pool, keys in intents:
if pool in pending:
continue
intent_tokens = len(keys) * page_size
if (
span not in self._queued_span_refs
and self.write_backlog_tokens_ >= self.write_backlog_cap
):
# The cap sits at 2x the intrinsic live-backlog ceiling (see
# init_hicache), so reaching it means leaked accounting or a
# broken stale sweep — a bug, not load.
self._backlog_cap_hits += 1
if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0:
logger.error(
"HiCache write backlog cap hit (occurrence %d): "
"backlog=%d cap=%d queue=%d. Live backlog is bounded "
"by the device pool span, so this indicates a "
"stale-sweep or accounting leak.",
self._backlog_cap_hits,
self.write_backlog_tokens_,
self.write_backlog_cap,
len(self.pending_write_queue),
)
return transfers
self._log_backup_dropped(intent_tokens)
continue
# A span larger than the pool's whole staging capacity can never
# stage; admitting it would wedge the head-of-line queue forever.
if not parent_covered or self._backup_oversize(*self._sizing(pool, keys)):
self._log_backup_dropped(intent_tokens)
continue
intent = _UnifiedBackupIntent(snapshot=snapshot, pool=pool, keys=list(keys))
self.pending_write_queue.append(intent)
self.inflight_backup_pools.setdefault(node_id, set()).add(pool)
self._retain_queued_span(intent)
def _backup_oversize(
self,
node_id: NodeId,
hash_values: list[str],
intent_tokens: int,
aux_xfers: Optional[list[PoolTransfer]] = None,
) -> bool:
def _backup_oversize(self, kv_tokens: int, aux_xfers: list[PoolTransfer]) -> bool:
"""True if any pool's staging need exceeds that pool's write-usable
capacity (total for KV, total minus the loads-priority margin for aux
pools — matching ``_aux_budget_blocked``'s admission ceiling): such an
intent could never stage and would wedge the FIFO head."""
if aux_xfers is None:
aux_xfers = self._build_aux_staging_transfers(node_id, hash_values)
intent could never stage and would wedge the FIFO head. Aux pools
staged through the anchor's host pool count against the KV total."""
cc = self._cache.cache_controller
shared_requests = self._shared_host_requests(intent_tokens, aux_xfers)
shared_requests = self._shared_host_requests(kv_tokens, aux_xfers)
if shared_requests is not None:
pool_tokens = cc.mem_pool_host.size
max_write_units = pool_tokens - pool_tokens // 10
if self._host_request_units(intent_tokens, aux_xfers) > max_write_units:
if self._host_request_units(kv_tokens, aux_xfers) > max_write_units:
return True
return not self._shared_backup_fits(shared_requests, empty=True)
if intent_tokens > cc.mem_pool_host.size:
return True
for t in aux_xfers or ():
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
anchor_need = kv_tokens
for t in aux_xfers:
entry = cc.mem_pool_host.entry_map.get(t.name)
if entry is not None and (
len(t.keys) * entry.host_pool.page_size
> entry.host_pool.size - self._aux_loads_margin(entry.host_pool)
):
if entry is None:
continue
need = len(t.keys) * entry.host_pool.page_size
if entry.host_pool is anchor_pool:
anchor_need += need
elif need > entry.host_pool.size - self._aux_loads_margin(entry.host_pool):
return True
return False
return anchor_need > cc.mem_pool_host.size
def _aux_loads_margin(self, host_pool) -> int:
"""Aux-pool tokens reserved for loads: at least one trailing window
@@ -623,9 +742,9 @@ class BufferModePipeline:
snapshot = intent.snapshot
state = self._validate_backup_intent(intent)
if state is None:
self.inflight_backup_node_ids.discard(snapshot.node_id)
intent_tokens = len(snapshot.hash_values) * page_size
self.write_backlog_tokens_ -= intent_tokens
self._finish_inflight(snapshot.node_id, intent.pool)
intent_tokens = len(intent.keys) * page_size
self._release_queued_span(intent)
swept_tokens += intent_tokens
continue
survivors.append(intent)
@@ -657,54 +776,57 @@ class BufferModePipeline:
while self.pending_write_queue:
intent = self.pending_write_queue[0]
snapshot = intent.snapshot
state = states[snapshot.node_id]
intent_tokens = len(snapshot.hash_values) * self._cache.page_size
if not self._backup_parent_covered(state):
node_id = snapshot.node_id
state = states[node_id]
intent_tokens = len(intent.keys) * self._cache.page_size
if not self._backup_parent_covered(state, node_id):
# Cascade a dropped parent down the chain rather than creating
# a permanent storage hole.
self.pending_write_queue.popleft()
self.inflight_backup_node_ids.discard(snapshot.node_id)
self.write_backlog_tokens_ -= intent_tokens
self._log_backup_dropped(intent_tokens)
self._drop_queue_head(intent_tokens, dropped=True)
continue
if self.write_staged_tokens_ >= live_cap:
if (
self.write_staged_tokens_ >= live_cap
and self._host_request_units(*self._sizing(intent.pool, intent.keys))
> 0
):
# Wait for current copies to free staging before rebuilding the head.
break
device_value, comp_xfers = self._cache.tree_core.build_backup_spec(
snapshot.node_id
)
sizing_xfers = self._build_aux_staging_transfers(
snapshot.node_id, snapshot.hash_values, comp_xfers
)
if self._backup_oversize(
snapshot.node_id,
snapshot.hash_values,
intent_tokens,
sizing_xfers,
launch = self._launch_transfers(intent)
# The pool may have become covered since admission (an earlier
# launch this round, a sibling's ack): nothing left to write
# retires the intent, not a drop.
if launch is None or all(
self._pool_covered(pool, keys) for pool, keys in launch[1].items()
):
self._drop_queue_head(intent_tokens, dropped=False)
continue
device_value, keys_by_pool, transfers = launch
sizing_xfers = [t for t in transfers if t.indices_from_pool is None]
kv_tokens = len(device_value)
if self._backup_oversize(kv_tokens, sizing_xfers):
# A permanently unstageable head must not block the queue.
self.pending_write_queue.popleft()
self.inflight_backup_node_ids.discard(snapshot.node_id)
self.write_backlog_tokens_ -= intent_tokens
self._log_backup_dropped(intent_tokens)
self._drop_queue_head(intent_tokens, dropped=True)
continue
shared_domain = self._shared_host_domain()
staging_at_limit = (
self.write_staged_tokens_
+ self._host_request_units(intent_tokens, sizing_xfers)
+ self._host_request_units(kv_tokens, sizing_xfers)
> live_cap
if shared_domain is not None
else self.write_staged_tokens_ >= live_cap
and self._host_request_units(kv_tokens, sizing_xfers) > 0
)
if staging_at_limit:
# Yield to live fetch demand; retry next round.
break
if self._aux_budget_blocked(intent, sizing_xfers):
if self._aux_budget_blocked(kv_tokens, sizing_xfers):
# An aux pool lacks staging headroom: yield at the gate
# instead of failing the alloc inside cc.write; acks free
# aux staging, retry next round.
break
if not self._stage_backup_intent(intent, device_value, comp_xfers):
if not self._stage_backup_intent(
intent, device_value, transfers, keys_by_pool
):
# Pool full of in-flight staging and nothing reclaimable
# (the tree never holds host values in buffer mode):
# defer, head-of-line; pending acks will free slots.
@@ -715,11 +837,106 @@ class BufferModePipeline:
# Do not leave prepared copies deferred across scheduler admission.
cc.start_writing()
def _drop_queue_head(self, intent_tokens: int, *, dropped: bool) -> None:
"""Retire the FIFO head without a D2H launch; ``dropped`` feeds the
dropped-tokens metric (a pool with nothing left to write is not a
drop)."""
intent = self.pending_write_queue.popleft()
self._finish_inflight(intent.snapshot.node_id, intent.pool)
self._release_queued_span(intent)
if dropped:
self._log_backup_dropped(intent_tokens)
def _build_backup_transfers(
self, node_id: NodeId, *, kv_only: bool = False
) -> tuple[
torch.Tensor, dict[ComponentType, list[PoolTransfer]], Optional[list[str]]
]:
"""The node's D2H spec. A FULL component's BACKUP_HOST KV transfer
replaces the KV rows (its keys name their pages); its other transfers
ride as aux pools. A KV intent skips the other components' transfers."""
tree_core = self._cache.tree_core
if kv_only:
device_value = tree_core.get_component_device_value(
node_id, ComponentType.FULL
)
comp_xfers = {}
else:
device_value, comp_xfers = tree_core.build_backup_spec(node_id)
full_xfers = tree_core.build_hicache_transfers(
ComponentType.FULL, node_id, CacheTransferPhase.BACKUP_HOST
)
kv_keys = None
aux: list[PoolTransfer] = []
for transfer in full_xfers or ():
if transfer.name == PoolName.KV:
device_value = (
transfer.device_indices
if transfer.device_indices is not None
else device_value[:0]
)
kv_keys = list(transfer.keys or ())
else:
aux.append(transfer)
if aux:
comp_xfers[ComponentType.FULL] = aux
return device_value, comp_xfers, kv_keys
def _launch_transfers(
self, intent: _UnifiedBackupIntent
) -> Optional[tuple[torch.Tensor, dict[PoolName, list[str]], list[PoolTransfer]]]:
"""(KV rows, keys per written pool, transfers to stage) of one intent,
off the node's fresh spec; None when nothing is left to write."""
snapshot = intent.snapshot
hash_values = snapshot.hash_values
device_value, comp_xfers, kv_keys = self._build_backup_transfers(
snapshot.node_id, kv_only=intent.pool == PoolName.KV
)
transfers: list[PoolTransfer] = []
if intent.pool == PoolName.KV:
keys = hash_values if kv_keys is None else kv_keys
if not keys or len(device_value) == 0:
return None
comp_xfers = {}
else:
device_value = device_value[:0]
keys = []
for ct, xfers in comp_xfers.items():
kept = []
for transfer in xfers:
if transfer.name != intent.pool or transfer.indices_from_pool:
continue
t_keys = self._transfer_keys(transfer, hash_values)
if not t_keys:
continue
if transfer.keys is None:
transfer.keys = t_keys
transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES
kept.append(transfer)
keys.extend(t_keys)
if kept:
transfers.extend(kept)
comp_xfers[ct] = kept
if not transfers:
return None
comp_xfers = {ct: xfers for ct, xfers in comp_xfers.items() if xfers}
keys_by_pool = {intent.pool: list(keys)}
# Sidecars ride their source pool's slots and D2H operation; they
# allocate no staging of their own.
for sidecar in self._cache._build_backup_sidecar(device_value, comp_xfers):
if sidecar.indices_from_pool != intent.pool:
continue
sidecar.keys = list(keys)
keys_by_pool[sidecar.name] = list(keys)
transfers.append(sidecar)
return device_value, keys_by_pool, transfers
def _stage_backup_intent(
self,
intent: _UnifiedBackupIntent,
device_value: torch.Tensor,
comp_xfers: dict[ComponentType, list[PoolTransfer]],
transfers: list[PoolTransfer],
keys_by_pool: dict[PoolName, list[str]],
) -> bool:
"""Allocate host staging and pin one admitted intent's source.
@@ -730,64 +947,54 @@ class BufferModePipeline:
cache = self._cache
cc = cache.cache_controller
snapshot = intent.snapshot
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
# Sidecars reuse the source pool's transient host/device indices. This
# includes both KV-derived pools and SWA-derived DSV4 state pools. They
# allocate no additional staging, but must ride the same D2H operation
# so their bytes are present when the storage write starts.
aux_xfers.extend(cache._build_backup_sidecar(device_value, comp_xfers))
# One ack id per write: a node may have several pools in flight.
self._backup_ack_seq += 1
ack_id = -(1 << 40) - self._backup_ack_seq
host_indices = cc.write(
device_value,
node_id=snapshot.node_id,
extra_pools=aux_xfers or None,
node_id=ack_id,
extra_pools=transfers or None,
flush=False,
)
if host_indices is None:
self._backup_ack_seq -= 1
return False
_track_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
beliefs = cache.storage_existence_cache
for pool, keys in keys_by_pool.items():
beliefs.pool(pool).track_inflight(keys)
# NOTE: no commit_backup — the node must never appear
# host-resident in buffer mode; staging slots live in the entry.
lock_params = cache.inc_lock_ref(snapshot.node_id).to_dec_params()
occupied_units = self.host_allocation_units(host_indices, aux_xfers)
self.ongoing_write_through[snapshot.node_id] = _UnifiedBufferBackupEntry(
occupied_units = self.host_allocation_units(host_indices, transfers)
self.ongoing_write_through[ack_id] = _UnifiedBufferBackupEntry(
intent=intent,
host_indices=host_indices,
aux_xfers=aux_xfers,
aux_xfers=transfers,
lock_params=lock_params,
occupied_units=occupied_units,
keys_by_pool=keys_by_pool,
)
self.write_staged_tokens_ += occupied_units
self.write_backlog_tokens_ -= len(snapshot.hash_values) * cache.page_size
self._release_queued_span(intent)
return True
def _aux_budget_blocked(
self,
intent: _UnifiedBackupIntent,
aux: Optional[list[PoolTransfer]] = None,
) -> bool:
def _aux_budget_blocked(self, kv_tokens: int, aux: list[PoolTransfer]) -> bool:
"""True when an aux pool cannot stage this intent right now (free
minus the loads-priority margin falls short of the need): defer at
the gate instead of failing the alloc inside cc.write and blocking
pure-KV intents behind an unallocatable head. The margin enforces
loads-have-priority on aux pools the way live_cap does on the KV
pool; avail already reflects prefetch-held slots, so no occupancy
subtraction here."""
snapshot = intent.snapshot
if aux is None:
aux = self._build_aux_staging_transfers(
snapshot.node_id, snapshot.hash_values
)
subtraction here. Aux pools staged through the anchor's host pool
are gated by live_cap and the allocation itself."""
cc = self._cache.cache_controller
shared_requests = self._shared_host_requests(
len(snapshot.hash_values) * self._cache.page_size, aux
)
shared_requests = self._shared_host_requests(kv_tokens, aux)
if shared_requests is not None:
return not self._shared_backup_fits(shared_requests)
if not aux:
return False
anchor_pool = cc.mem_pool_host.anchor_entry.host_pool
for t in aux:
entry = cc.mem_pool_host.entry_map.get(t.name)
if entry is None:
if entry is None or entry.host_pool is anchor_pool:
continue
need = len(t.keys) * entry.host_pool.page_size
headroom = entry.host_pool.available_size() - self._aux_loads_margin(
@@ -797,23 +1004,6 @@ class BufferModePipeline:
return True
return False
def _aux_window_keys(
self, hash_values: list[str], transfer: PoolTransfer
) -> Optional[list[str]]:
"""Trailing KV page hashes keying an aux transfer's staged window
(one key per aux-pool page)."""
if transfer.host_indices is None or transfer.host_indices.numel() == 0:
return None
if transfer.indices_from_pool is not None:
return None # sidecar rides another pool's slots; nothing to key
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
return None
num_keys = len(transfer.host_indices) // entry.host_pool.page_size
if num_keys == 0 or num_keys > len(hash_values):
return None
return hash_values[-num_keys:]
def finish_backup_ack(self, ack_id: int) -> None:
"""D2H confirmed: drop the device lock and enqueue the storage write
(which reads from the staging copy, so device eviction may proceed)."""
@@ -822,73 +1012,65 @@ class BufferModePipeline:
snapshot = intent.snapshot
self._cache.dec_lock_ref(snapshot.node_id, entry.lock_params)
# Every independently staged aux pool writes a trailing snapshot keyed
# by the last KV page hashes it covers. A derived sidecar writes the
# exact key span of its source pool while reusing that source's slots.
storage_xfers: list[PoolTransfer] = []
storage_sources = {
PoolName.KV: PoolTransfer(
name=PoolName.KV,
host_indices=entry.host_indices,
keys=snapshot.hash_values,
)
}
for staged in entry.aux_xfers:
if staged.indices_from_pool is not None:
continue
keys = self._aux_window_keys(snapshot.hash_values, staged)
if keys is None:
continue
transfer = PoolTransfer(
name=staged.name,
host_indices=staged.host_indices,
keys=keys,
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
storage_xfers.append(transfer)
storage_sources.setdefault(staged.name, transfer)
for staged in entry.aux_xfers:
if staged.indices_from_pool is None:
continue
source = storage_sources.get(staged.indices_from_pool)
if source is None:
raise AssertionError(
"Buffer-mode storage sidecar source missing: "
f"{staged.name} from {staged.indices_from_pool}."
)
storage_xfers.append(
PoolTransfer(
name=staged.name,
keys=source.keys,
hit_policy=staged.hit_policy,
indices_from_pool=staged.indices_from_pool,
)
)
storage_xfers = self._storage_transfers(entry.aux_xfers)
# Another pool's write carries no KV pages; its keys ride the transfer.
hash_value = (
list(entry.keys_by_pool[PoolName.KV]) if intent.pool == PoolName.KV else []
)
operation_id = self._cache.cache_controller.write_storage(
entry.host_indices,
snapshot.key.token_ids,
snapshot.hash_values,
hash_value,
snapshot.prefix_keys,
extra_pools=storage_xfers or None,
)
self.ongoing_backup[operation_id] = entry
@staticmethod
def _storage_transfers(staged: list[PoolTransfer]) -> list[PoolTransfer]:
"""One storage transfer per pool: a pool staged in several transfers
becomes one contiguous key span over their slots; a derived sidecar
writes its source's key span over the source's slots."""
merged: dict[PoolName, PoolTransfer] = {}
for transfer in staged:
if transfer.indices_from_pool is not None:
merged[transfer.name] = PoolTransfer(
name=transfer.name,
keys=list(transfer.keys),
hit_policy=transfer.hit_policy,
indices_from_pool=transfer.indices_from_pool,
)
continue
earlier = merged.get(transfer.name)
if earlier is None:
merged[transfer.name] = PoolTransfer(
name=transfer.name,
host_indices=transfer.host_indices,
keys=list(transfer.keys),
hit_policy=transfer.hit_policy,
)
continue
earlier.keys.extend(transfer.keys)
earlier.host_indices = torch.cat(
[earlier.host_indices, transfer.host_indices]
)
return list(merged.values())
def finish_storage_write_ack(self, operation_id: int) -> None:
"""Storage write acked (rank-synced drain): free the entry's staging
outright. Existence entries are added unconditionally
(completed_tokens can diverge across ranks under backend failure) to
keep admission decisions TP-deterministic. No-op for operations this
pipeline does not own (e.g. acks for already-reset state)."""
"""Storage ack (rank-synced drain): free the staging and feed each
written pool's beliefs unconditionally, keeping admission
TP-deterministic under per-rank backend failures."""
entry = self.ongoing_backup.pop(operation_id, None)
if entry is None:
return
intent = entry.intent
snapshot = intent.snapshot
self._cache.storage_existence_cache.add(PoolName.KV, snapshot.hash_values)
beliefs = self._cache.storage_existence_cache
for pool, keys in entry.keys_by_pool.items():
pool_beliefs = beliefs.pool(pool)
pool_beliefs.add(keys)
pool_beliefs.untrack_inflight(keys)
self._free_staging_now(entry.host_indices, entry.aux_xfers)
self.write_staged_tokens_ -= entry.occupied_units
self.inflight_backup_node_ids.discard(snapshot.node_id)
_untrack_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
self._finish_inflight(entry.intent.snapshot.node_id, entry.intent.pool)
def _free_staging_now(
self, host_indices: torch.Tensor, aux_xfers: list[PoolTransfer]
@@ -1163,10 +1345,16 @@ class BufferModePipeline:
staged_pages = num_tokens // cache.page_size
staged_hashes = hash_value[:staged_pages]
staged_kv = host_indices[:num_tokens]
# Feed existence beliefs from the storage-fetched pages: the fetch
# itself is the evidence, so feeding is sound even if this staged
# prefetch is later dropped unconsumed.
cache.storage_existence_cache.add(PoolName.KV, list(staged_hashes))
# The fetch is evidence for every pool it delivered (sound even if the
# staged prefetch is later dropped): KV-derived sidecars came with the
# staged pages, other pools with the keys their transfer carries.
beliefs = cache.storage_existence_cache
beliefs.pool(PoolName.KV).add(staged_hashes)
for transfer in aux_xfers:
if transfer.indices_from_pool == PoolName.KV:
beliefs.pool(transfer.name).add(staged_hashes)
elif transfer.keys:
beliefs.pool(transfer.name).add(transfer.keys)
self.staged_prefetches[request] = _StagedPrefetch(
request=request,
key_tokens=array(
@@ -1,17 +1,18 @@
"""Local existence cache for HiCache buffer_only mode.
"""Local existence beliefs for HiCache buffer_only mode.
In buffer mode host memory holds no persistent copy, so without a local
existence signal every re-insert of a hot prefix re-stages and re-writes to
L3 storage. This cache is that signal: a bounded LRU of (pool, page-hash)
entries *believed* present in storage.
L3 storage. The beliefs are that signal: per storage pool, a bounded LRU of
page hashes *believed* present in storage, kept and healed independently so
a node is written pool by pool (only the pools not believed stored).
Semantics are advisory, not authoritative:
- A hit skips the redundant D2H + storage write.
- A hit skips the pool's redundant D2H staging + storage write.
- A stale positive (backend evicted the data) costs skipped write-backs until
a prefetch hit-query shortfall invalidates the entries; the next insert
then writes the data back. Never a correctness issue — at worst one cold
recompute, the same as any cache miss.
a prefetch hit-query shortfall invalidates the pool's entries; the next
insert then writes that pool back. Never a correctness issue — at worst
one cold recompute, the same as any cache miss.
- A miss (entry LRU-evicted or never seen) just costs one redundant write
(idempotent: storage keys are content-addressed).
@@ -21,6 +22,10 @@ recompute (same tokens => same chain). Page hashes are chained and the write
path is prefix-contiguous (parent-cover gate), so a node's own page set is
the only thing a caller needs to check.
A pool also tracks its in-flight content (D2H launch to storage ack), which
admission treats as covered; its ``PoolBeliefPolicy`` says how its keys sit
on the chain (a component may supply one via ``buffer_belief_policy``).
TP determinism: replicas stay identical because every mutation happens on the
scheduler thread at lockstep points with cross-rank-reduced inputs
(storage-ack drain, prefetch-hit drain, fill commit). Do not touch it from
@@ -30,82 +35,165 @@ anywhere else.
from __future__ import annotations
from collections import OrderedDict
from collections.abc import Container, Iterable, Sequence
from collections.abc import Iterable, Sequence
# ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers roughly
# 32M tokens at page size 64 across all pools.
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy
# Per pool: ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers
# roughly 32M tokens at page size 64.
HICACHE_EXISTENCE_CACHE_MAX_ENTRIES = 512 * 1024
class StorageExistenceCache:
class PoolExistenceBeliefs:
"""One pool's LRU of page hashes believed stored, plus its refcounted
in-flight content (D2H launch to storage ack)."""
def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES):
self.max_entries = max_entries
self._entries: OrderedDict[tuple[str, str], None] = OrderedDict()
self._entries: OrderedDict[str, None] = OrderedDict()
self._inflight: dict[str, int] = {}
def __len__(self) -> int:
return len(self._entries)
def add(self, pool: str, hashes: Iterable[str]) -> None:
def add(self, hashes: Iterable[str]) -> None:
entries = self._entries
move_to_end = entries.move_to_end
for h in hashes:
key = (pool, h)
entries[key] = None
move_to_end(key)
entries[h] = None
move_to_end(h)
pop_oldest = entries.popitem
for _ in range(len(entries) - self.max_entries):
pop_oldest(last=False)
def contains(self, pool: str, page_hash: str) -> bool:
def contains(self, page_hash: str) -> bool:
entries = self._entries
key = (pool, page_hash)
if key not in entries:
if page_hash not in entries:
return False
entries.move_to_end(key)
entries.move_to_end(page_hash)
return True
def contains_all(self, pool: str, hashes: Iterable[str]) -> bool:
def contains_all(self, hashes: Iterable[str]) -> bool:
entries = self._entries
move_to_end = entries.move_to_end
for h in hashes:
key = (pool, h)
try:
move_to_end(key)
move_to_end(h)
except KeyError:
return False
return True
def covers_all(
self,
pool: str,
hashes: Iterable[str],
extra_cover: Container[str] = frozenset(),
) -> bool:
"""True when every page is believed stored or sits in
``extra_cover`` (e.g. content past its D2H launch, which always
reaches its storage-ack). LRU-touches the believed entries."""
def covered(self, hashes: Iterable[str]) -> bool:
"""Every page believed stored or in flight; LRU-touches believed entries."""
inflight = self._inflight
if not inflight:
return self.contains_all(hashes)
entries = self._entries
move_to_end = entries.move_to_end
if extra_cover:
for h in hashes:
key = (pool, h)
if key in entries:
move_to_end(key)
elif h not in extra_cover:
return False
return True
return self.contains_all(pool, hashes)
for h in hashes:
if h in entries:
move_to_end(h)
elif h not in inflight:
return False
return True
def invalidate_beyond(
self, pool: str, hashes: Sequence[str], keep_pages: int
) -> None:
"""Ground-truth heal from a prefetch hit query: discard beliefs
beyond the leading ``keep_pages`` of a hash chain (the folded
usable cut). The next insert re-writes the discarded span, closing
stale positives and aux holes at the cut."""
def track_inflight(self, hashes: Iterable[str]) -> None:
"""One ref per page hash at its D2H launch (several launched writes
can carry the same content)."""
refs = self._inflight
for h in hashes:
refs[h] = refs.get(h, 0) + 1
def untrack_inflight(self, hashes: Iterable[str]) -> None:
"""Drop one ref per page hash at the storage ack (or a failed launch)."""
refs = self._inflight
for h in hashes:
n = refs.get(h, 0) - 1
if n <= 0:
refs.pop(h, None)
else:
refs[h] = n
def invalidate_beyond(self, hashes: Sequence[str], keep_pages: int) -> None:
"""Discard the beliefs beyond the leading ``keep_pages`` of ``hashes``;
the next insert re-writes that span."""
pop = self._entries.pop
for h in hashes[keep_pages:]:
pop((pool, h), None)
pop(h, None)
def clear(self) -> None:
self._entries.clear()
self._inflight.clear()
def clear_inflight(self) -> None:
self._inflight.clear()
class StorageExistenceCache:
"""The beliefs of every storage pool, created on first use."""
def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES):
self.max_entries = max_entries
self._pools: dict[str, PoolExistenceBeliefs] = {}
def pool(self, name: str) -> PoolExistenceBeliefs:
beliefs = self._pools.get(name)
if beliefs is None:
beliefs = self._pools[name] = PoolExistenceBeliefs(self.max_entries)
return beliefs
def clear(self) -> None:
for beliefs in self._pools.values():
beliefs.clear()
def clear_inflight(self) -> None:
for beliefs in self._pools.values():
beliefs.clear_inflight()
class PoolBeliefPolicy:
"""How a pool's keys sit on a node's page-hash chain, hence how a storage
verdict heals its beliefs: a rank-agreed hit-query boundary (``evidence``
= the query reported this pool) or a transfer's delivered page count."""
def heal_from_query(
self,
beliefs: PoolExistenceBeliefs,
chain: Sequence[str],
keys: Sequence[str],
hit_pages: int,
*,
evidence: bool,
) -> None:
raise NotImplementedError
def heal_from_transfer(
self,
beliefs: PoolExistenceBeliefs,
keys: Sequence[str],
delivered_pages: int,
) -> None:
beliefs.invalidate_beyond(keys, delivered_pages)
class ChainPagesPolicy(PoolBeliefPolicy):
"""One key per chain page (ALL_PAGES). The agreed boundary is the MIN of
per-rank prefix hits, so every rank verified the chain up to it."""
def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence):
beliefs.invalidate_beyond(chain, hit_pages)
if evidence:
beliefs.add(chain[:hit_pages])
class TrailingWindowPolicy(PoolBeliefPolicy):
"""Keys are a trailing window of the chain (TRAILING_PAGES). A rank only
verified the window ending at its own boundary, so the query only
invalidates; acks and fills teach presence."""
def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence):
beliefs.invalidate_beyond(chain, hit_pages)
def default_belief_policy(hit_policy: PoolHitPolicy) -> PoolBeliefPolicy:
"""The built-in policy for a pool no component claims."""
if hit_policy == PoolHitPolicy.TRAILING_PAGES:
return TrailingWindowPolicy()
return ChainPagesPolicy()
@@ -51,6 +51,10 @@ from sglang.srt.utils import broadcast_pyobj
logger = logging.getLogger(__name__)
# Pool boundaries are nonnegative; an omitted rank must dominate MIN so
# a peer cannot establish presence of that rank's storage shard.
_NO_POOL_VERDICT = -1
@dataclass(frozen=True)
class PPPrefetchPoolSpec:
@@ -165,6 +169,10 @@ class PrefetchOperation(StorageOperation):
pool_transfers=pool_transfers,
)
self.pool_transfers_done = not bool(pool_transfers)
# The hit query's verdict per pool, as the chain prefix length (in
# pages) up to which the pool is present; rank-reduced with the
# folded hit count. Empty until the query ran on this rank.
self.query_pool_hit_pages: dict[PoolName, int] = {}
self.buffer_host_occupied_units: Optional[int] = None
# The Python transfer worker leaves the unfinished tail to the ACK drain;
# a controller that releases it itself must set this False.
@@ -1348,9 +1356,11 @@ class HybridCacheController(BaseHiCacheController):
if operation.assume_stored:
# A prior hit on a suffix of this span proved it stored, and writes
# are prefix-covered, so re-querying only adds a round trip.
# are prefix-covered, so re-querying only adds a round trip. Nothing
# is verified now, so no pool gets a verdict to learn from.
kv_hit_pages = len(hash_value)
operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages)
operation.query_pool_hit_pages = {}
return hash_value, kv_hit_pages * self.page_size
extra_info = HiCacheStorageExtraInfo(
@@ -1369,12 +1379,45 @@ class HybridCacheController(BaseHiCacheController):
kv_hit_pages = hit_result.kv_hit_pages
operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages)
# Each pool's own boundary (the KV pool's where the backend reports
# it, else the folded cut); the beliefs heal pool by pool. A pool the
# backend did not report carries no verdict.
pool_hits = hit_result.extra_pool_hit_pages
operation.query_pool_hit_pages = {
PoolName.KV: pool_hits.get(PoolName.KV, kv_hit_pages)
}
for transfer in operation.pool_transfers or ():
if transfer.name in pool_hits:
operation.query_pool_hit_pages[transfer.name] = pool_hits[transfer.name]
return (
hash_value[:kv_hit_pages],
kv_hit_pages * self.page_size,
)
def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int:
"""One MIN collective for the folded hit count and every pool's own
boundary, so per-pool beliefs stay rank-identical. A pool omitted
by any rank heals from the folded cut without learning presence."""
pools = list(PoolName)
local = operation.query_pool_hit_pages
packed = torch.tensor(
[storage_hit_count] + [local.get(pool, _NO_POOL_VERDICT) for pool in pools],
dtype=torch.int,
)
self._all_reduce(
packed,
torch.distributed.ReduceOp.MIN,
self.prefetch_hits_sync_groups,
)
reduced = packed.tolist()
operation.query_pool_hit_pages = {
pool: pages
for pool, pages in zip(pools, reduced[1:], strict=True)
if pages != _NO_POOL_VERDICT
}
return int(reduced[0])
def _move_pool_indices(
self, host_pool, host_indices, device_indices, *, write_back_jit: bool
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -17,7 +17,11 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult
from sglang.srt.mem_cache.hicache_storage import (
PoolName,
PoolTransfer,
PoolTransferResult,
)
from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401
BASE_COMPONENT_TYPE,
ComponentType,
@@ -25,7 +29,11 @@ from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
PoolBeliefPolicy,
)
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.pool_host.group import HostPoolGroup, PoolEntry
from sglang.srt.mem_cache.unified_cache.cache_action import (
CacheAction,
ComponentAction,
@@ -699,6 +707,40 @@ class TreeComponent(ABC):
"""Allocate prefetch staging sized by prepare_prefetch, once the hit is known."""
return None
# ---- Buffer-mode backup hooks (hicache_host_memory_mode=buffer_only) ----
# Buffer mode writes a node pool by pool: keys per pool come from
# ``buffer_backup_keys`` or the BACKUP_HOST transfers, beliefs heal through
# ``buffer_belief_policy``; the pipeline pins the node from D2H launch to
# ack, so a component dropping rows asks ``buffer_backup_pending`` first.
def buffer_mode_host_pool_entries(
self, host_pool_group: HostPoolGroup
) -> list[PoolEntry]:
"""Extra host pool entries this component stages through (an entry
may alias the anchor's host pool)."""
return []
def buffer_backup_keys(
self, node: UnifiedTreeNode, hash_values: list[str]
) -> dict[PoolName, list[str]]:
"""Per-pool storage keys a backup of ``node`` writes for this component
(empty list: withhold the pool; empty dict: derive them from its
BACKUP_HOST transfers). A component naming any pool names all of them."""
return {}
def buffer_backup_parent_covered(
self, parent_node_id: NodeId, node_id: NodeId
) -> bool:
"""Whether this component covers the parent of ``node_id`` without
the parent's KV pages in storage (no prefix hole for a reader)."""
return False
def buffer_belief_policy(self, pool: PoolName) -> Optional[PoolBeliefPolicy]:
"""The policy healing ``pool``'s existence beliefs, for a pool this
component owns; None picks the built-in one for the transfer's hit
policy."""
return None
def build_hicache_transfers(
self,
node: UnifiedTreeNode,
@@ -1055,6 +1055,26 @@ class SWAComponent(TreeComponent):
host_indices = self._swa_kv_pool_host.alloc(num_tokens)
return host_indices
def buffer_backup_keys(
self, node: UnifiedTreeNode, hash_values: list[str]
) -> dict[PoolName, list[str]]:
# Buffer mode stages the node's own SWA rows, one key per page from
# the chain's tail, so admission needs no transfer (and no device op).
if not self.tree_core.is_host_memory_buffer_only:
return {}
cd = node.component_data[self.component_type]
if not self.tree_core.has_swa_host_pool or cd.value is None:
return {PoolName.SWA: []}
rows = (
node.component_data[BASE_COMPONENT_TYPE].value
if self._unified_allocator() is not None
else cd.value
)
num_pages = len(rows) // self._swa_kv_pool_host.page_size
if num_pages == 0 or num_pages > len(hash_values):
return {PoolName.SWA: []}
return {PoolName.SWA: list(hash_values[-num_pages:])}
def build_hicache_transfers(
self,
node: UnifiedTreeNode,
@@ -2381,6 +2381,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
"""Read a node's device->host backup spec (device value + component transfers) now."""
return self._build_backup_spec(self.node_by_id(node_id))
def buffer_backup_pool_keys(
self, node_id: NodeId, hash_values: list[str]
) -> dict[ComponentType, dict[PoolName, list[str]]]:
node = self.node_by_id(node_id)
named: dict[ComponentType, dict[PoolName, list[str]]] = {}
for comp in self.components:
pool_keys = comp.buffer_backup_keys(node, hash_values)
if pool_keys:
named[comp.component_type] = pool_keys
return named
def build_backup_kv_action(self, node_id: NodeId) -> BackupKV:
return self._build_backup_kv_action(self.node_by_id(node_id))
def _build_backup_spec(self, node: UnifiedTreeNode):
"""Gather missing Full and component transfers for Host backup."""
device_value = node.component_data[BASE_COMPONENT_TYPE].value
@@ -127,7 +127,11 @@ if TYPE_CHECKING:
MatchResult,
)
from sglang.srt.mem_cache.events import KVCacheEventRecorder
from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult
from sglang.srt.mem_cache.hicache_storage import (
PoolName,
PoolTransfer,
PoolTransferResult,
)
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.cache_action import (
BackupKV,
@@ -549,6 +553,20 @@ class UnifiedTreeCoreInterface(ABC):
"""Read a node's device->host backup spec (device value + transfers) now."""
...
def buffer_backup_pool_keys(
self, node_id: NodeId, hash_values: list[str]
) -> dict[ComponentType, dict[PoolName, list[str]]]:
"""Per component, the storage keys per pool a buffer-mode backup of
the node writes (``TreeComponent.buffer_backup_keys``); Rust cores name none."""
return {}
def build_backup_kv_action(self, node_id: NodeId) -> BackupKV:
"""The write-through backup action for a node and its unstored
ancestors; only component-requested buffer backups need it."""
raise NotImplementedError(
f"{type(self).__name__} does not expose build_backup_kv_action"
)
@abstractmethod
def build_storage_backup_spec(
self, node_id: NodeId, pass_prefix_keys: bool
@@ -36,10 +36,17 @@ from sglang.srt.mem_cache.buffer_mode.pipeline import (
validate_buffer_only_stack,
)
from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import (
PoolBeliefPolicy,
StorageExistenceCache,
default_belief_policy,
)
from sglang.srt.mem_cache.common import RetractionBackup
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer, SidecarPoolSpec
from sglang.srt.mem_cache.hicache_storage import (
PoolHitPolicy,
PoolName,
PoolTransfer,
SidecarPoolSpec,
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
PPPrefetchDecision,
@@ -391,6 +398,8 @@ class UnifiedRadixCache(BasePrefixCache):
self.ongoing_backup: dict[int, tuple[NodeId, DecLockRefParams]] = {}
if self.buffer_pipeline is not None:
self.buffer_pipeline.reset()
# The dropped writes' content no longer reaches a storage ack.
self.storage_existence_cache.clear_inflight()
if self.cache_controller is not None:
self.cache_controller.reset()
@@ -487,10 +496,9 @@ class UnifiedRadixCache(BasePrefixCache):
if swa is not None and self.tree_core.has_swa_host_pool
else 0
),
# Leak backstop only: live queued tokens are intrinsically
# bounded by the FULL device pool (one intent per node, stale
# intents swept per tick), so a cap that binds on live
# content would drop-newest and punch storage holes.
# Leak backstop only: the queue is charged once per unique node
# span across pools; a cap binding on live content would
# drop-newest and punch storage holes.
write_backlog_cap=2 * self.token_to_kv_pool_allocator.size_full,
)
# State initialization
@@ -498,6 +506,13 @@ class UnifiedRadixCache(BasePrefixCache):
self.cache_controller.host_write_staged_tokens_fn = lambda: (
self.buffer_pipeline.write_staged_tokens_
)
# Pools a component's backups stage through besides the stock
# ones (an entry may alias the anchor's host pool).
for component in self.components.values():
for entry in component.buffer_mode_host_pool_entries(
self.host_pool_group
):
self.cache_controller.register_host_pool_entry(entry)
self.write_through_threshold = (
1 if get_memory().hicache_write_policy == "write_through" else 2
)
@@ -596,6 +611,8 @@ class UnifiedRadixCache(BasePrefixCache):
return InsertResult(prefix_len=0)
# Fail fast on re-entrancy without touching the in-flight walk.
assert not self.tree_core.has_ongoing_insert(), "re-entrant insert"
if self.buffer_pipeline is not None:
self.buffer_pipeline.begin_insert_walk()
# Pump the resumable insert, applying each step's actions at its barrier.
try:
step = self.tree_core.begin_insert(params)
@@ -609,6 +626,8 @@ class UnifiedRadixCache(BasePrefixCache):
finally:
# Drain still-pending actions so frees reach the allocator on abort.
self._apply_cache_actions(self.tree_core.end_insert())
if self.buffer_pipeline is not None:
self.buffer_pipeline.end_insert_walk()
@rank_consensus(same_params=True, same_results=True)
def evict(self, params: EvictParams) -> EvictResult:
@@ -1577,6 +1596,22 @@ class UnifiedRadixCache(BasePrefixCache):
self.writing_check(write_back=True)
return True
def buffer_backup_pending(self, node_id: NodeId) -> bool:
"""Whether a buffer-mode write of ``node_id`` is queued, staged or
awaiting its storage ack (see ``BufferModePipeline.backup_pending``)."""
return self.buffer_pipeline is not None and self.buffer_pipeline.backup_pending(
node_id
)
def request_buffer_backup(self, node_id: NodeId) -> None:
"""Queue a buffer-mode backup of ``node_id`` (and its unstored
ancestors) outside the insert trigger; no-op without buffer storage."""
if self.buffer_pipeline is None or not self.enable_storage:
return
self._execute_and_commit_kv_backup(
self.tree_core.build_backup_kv_action(node_id)
)
def _execute_and_commit_kv_backup(
self, action: BackupKV, write_back: bool = False
) -> int:
@@ -2408,9 +2443,9 @@ class UnifiedRadixCache(BasePrefixCache):
for transfer, count in zip(pool_transfers, pool_hit_pages)
)
if pool_transfers and not all_succeeded:
# Drop the KV beliefs from the first page any pool failed to serve;
# the next insert then re-writes that span through one FULL check,
# restoring the missing aux pages.
# Drop each pool's beliefs beyond what it served (KV and its
# sidecars beyond the completed span); the next insert re-writes
# only the missing pools.
invalidation_hashes = (
operation.sidecar_hash_values
if operation.sidecar_hash_values is not None
@@ -2418,18 +2453,29 @@ class UnifiedRadixCache(BasePrefixCache):
)
trimmed_pages = len(invalidation_hashes) - len(hash_value)
keep_pages = trimmed_pages + completed_tokens // self.page_size
beliefs = self.storage_existence_cache
if keep_pages < len(invalidation_hashes):
self._belief_policy(
PoolName.KV, PoolHitPolicy.ALL_PAGES
).heal_from_transfer(
beliefs.pool(PoolName.KV), invalidation_hashes, keep_pages
)
for transfer in operation.pool_transfers or ():
if transfer.indices_from_pool == PoolName.KV:
self._belief_policy(
transfer.name, transfer.hit_policy
).heal_from_transfer(
beliefs.pool(transfer.name),
invalidation_hashes,
keep_pages,
)
for transfer, count in zip(pool_transfers, pool_hit_pages):
if transfer.keys is None:
keep_pages = 0
elif count < len(transfer.keys):
# Aux transfers key the chain's trailing pages.
keep_pages = min(
keep_pages,
max(0, len(invalidation_hashes) - len(transfer.keys)),
if transfer.keys is not None and count < len(transfer.keys):
self._belief_policy(
transfer.name, transfer.hit_policy
).heal_from_transfer(
beliefs.pool(transfer.name), transfer.keys, count
)
self.storage_existence_cache.invalidate_beyond(
PoolName.KV, invalidation_hashes, keep_pages=keep_pages
)
self._finish_storage_prefetch(
request, fulfilled_tokens=0, reason="storage_transfer"
)
@@ -2444,11 +2490,15 @@ class UnifiedRadixCache(BasePrefixCache):
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
logger.warning(
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
"kv_beliefs_kept_pages=%d",
"kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s",
request.rid,
completed_tokens,
expected_tokens,
keep_pages,
{
getattr(transfer.name, "value", transfer.name): count
for transfer, count in zip(pool_transfers, pool_hit_pages)
},
)
return False
return True
@@ -2637,18 +2687,47 @@ class UnifiedRadixCache(BasePrefixCache):
),
)
def _belief_policy(
self, pool: PoolName, hit_policy: PoolHitPolicy
) -> PoolBeliefPolicy:
"""The policy healing ``pool``'s existence beliefs: the owning
component's, else the built-in one for the transfer's hit policy."""
for component in self.components.values():
policy = component.buffer_belief_policy(pool)
if policy is not None:
return policy
return default_belief_policy(hit_policy)
def _invalidate_absent_from_hit_query(self, operation) -> None:
"""Drop KV beliefs beyond the folded usable cut (rank-synced): the
next insert then re-writes the node (all pools), healing stale
positives and aux holes at the cut through one FULL check."""
"""Heal each pool's beliefs beyond its rank-agreed boundary (a
chain-page pool also learns presence up to it); a pool without a
verdict heals from the folded cut."""
if self.host_memory_mode != "buffer_only":
return
chain = operation.all_hash_values
if chain is None:
return
self.storage_existence_cache.invalidate_beyond(
PoolName.KV, chain, keep_pages=operation.storage_hit_count // self.page_size
beliefs = self.storage_existence_cache
verdicts = operation.query_pool_hit_pages
folded_pages = operation.storage_hit_count // self.page_size
def verdict(pool: PoolName) -> tuple[int, bool]:
pages = verdicts.get(pool)
return (folded_pages, False) if pages is None else (pages, True)
kv_pages, kv_evidence = verdict(PoolName.KV)
self._belief_policy(PoolName.KV, PoolHitPolicy.ALL_PAGES).heal_from_query(
beliefs.pool(PoolName.KV), chain, chain, kv_pages, evidence=kv_evidence
)
for transfer in operation.pool_transfers or ():
hit_pages, evidence = verdict(transfer.name)
self._belief_policy(transfer.name, transfer.hit_policy).heal_from_query(
beliefs.pool(transfer.name),
chain,
list(transfer.keys or ()),
hit_pages,
evidence=evidence,
)
def _account_prefetch_outcome(self, operation, revoked: bool) -> None:
"""Feed the cumulative prefetch-outcome counters at the (rank-synced)