diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index a05c7623dd..27b775d1f5 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -1295,15 +1295,9 @@ class HiCacheController: ) hash_value, storage_hit_count = [], 0 - storage_hit_count_tensor = torch.tensor( - storage_hit_count, dtype=torch.int + storage_hit_count = self._sync_prefetch_hit_query( + operation, storage_hit_count ) - self._all_reduce( - storage_hit_count_tensor, - torch.distributed.ReduceOp.MIN, - self.prefetch_hits_sync_groups, - ) - storage_hit_count = storage_hit_count_tensor.item() # Record the TP-synced hit count; the scheduler thread decides # at drain time whether to revoke (below threshold) or allocate. @@ -1316,6 +1310,17 @@ class HiCacheController: except Empty: continue + def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int: + """Rank-reduce a hit query (MIN) so every rank agrees on the usable + prefix. Runs for every operation, hit or miss.""" + hit_count = torch.tensor(storage_hit_count, dtype=torch.int) + self._all_reduce( + hit_count, + torch.distributed.ReduceOp.MIN, + self.prefetch_hits_sync_groups, + ) + return int(hit_count.item()) + def write_storage( self, host_indices: torch.Tensor, diff --git a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py index f8870919a4..45c1e358cf 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py +++ b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py @@ -81,23 +81,28 @@ def _minimum_full_transfer_tokens( class _UnifiedBackupIntent(msgspec.Struct): - """Buffer-mode backup intent, unpinned while queued. + """One pool of one node queued for a buffer-mode write (unpinned while + queued); a derived sidecar rides its source pool's intent. Snapshots node identity at enqueue time: a split rewrites the node's key/hash in place while these copies stay intact, so a key-length change detects a split and a missing FULL device value detects eviction - (``_validate_backup_intent``). + (``_validate_backup_intent``). ``keys`` is the admission view (it sizes + drop metrics); the D2H launch re-reads rows and keys off the node. """ snapshot: BufferBackupSnapshot + pool: PoolName + keys: list[str] class _UnifiedBufferBackupEntry(msgspec.Struct): """A buffer-mode backup after its D2H launch: intent + staging slots. - ``host_indices`` are FULL-pool staging slots; ``aux_xfers`` carry the - staged aux-pool slots (e.g. the SWA window). All are freed at the - storage-write ack — host memory is never retained as a cache tier. + ``host_indices`` are the KV staging slots (empty for another pool's + intent); ``aux_xfers`` the staged transfers and riding sidecars; all are + freed at the storage-write ack (host memory is never a cache tier). + ``keys_by_pool`` are the keys the write stores, learned at the ack. """ intent: _UnifiedBackupIntent @@ -105,6 +110,7 @@ class _UnifiedBufferBackupEntry(msgspec.Struct): aux_xfers: list[PoolTransfer] lock_params: DecLockRefParams occupied_units: int + keys_by_pool: dict[PoolName, list[str]] class _StagedPrefetch(msgspec.Struct): @@ -147,24 +153,6 @@ class _AnchorLock(msgspec.Struct): tokens: int -def _track_content_refs(refs: dict[str, int], hash_values: list[str]) -> None: - """Add one content ref per page hash (at D2H launch). Refcounted, - not a flag: several launched entries can carry the same content - (duplicate staging of republished spans).""" - for h in hash_values: - refs[h] = refs.get(h, 0) + 1 - - -def _untrack_content_refs(refs: dict[str, int], hash_values: list[str]) -> None: - """Drop one content ref per page hash (at storage-ack).""" - for h in hash_values: - n = refs.get(h, 0) - 1 - if n <= 0: - refs.pop(h, None) - else: - refs[h] = n - - def validate_buffer_only_stack( sidecar_pool_specs: list[SidecarPoolSpec], host_pool_group: HostPoolGroup, @@ -270,8 +258,8 @@ class BufferModePipeline: # a host pool (0 = KV-only: no trailing window staged). Static after # pool assembly. self._swa_window_pages = swa_window_pages - # Metadata-only pending-write backlog cap; beyond it new intents - # are dropped at admission (re-trigger on a later hit). + # Metadata-only pending-write backlog cap, measured in unique FULL + # snapshot spans; beyond it new spans are dropped at admission. self.write_backlog_cap = write_backlog_cap from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -318,19 +306,18 @@ class BufferModePipeline: ] = {} self.staged_prefetches: dict[CacheRequestHandle, _StagedPrefetch] = {} self.ongoing_buffer_load_back: dict[int, _OngoingBufferLoadBack] = {} - # Backup pipeline: FIFO intents awaiting a D2H slot, node ids - # anywhere in flight (dedupes re-triggers), and a content refcount - # of every page hash between D2H launch and storage-ack — admission - # skips content covered by beliefs + launched writes. + # FIFO intents (one per pool of a node) awaiting a D2H slot, and the + # pools in flight per node (admission to storage ack). self.pending_write_queue: deque[_UnifiedBackupIntent] = deque() - self.inflight_backup_node_ids: set[int] = set() - self.inflight_backup_hashes: dict[str, int] = {} - # Backups between D2H launch and D2H ack (keyed by node id), then - # between storage-write launch and storage ack (keyed by operation - # id). Mirrors the cache-mode ongoing_write_through/ongoing_backup - # stages, with buffer entries. + self._queued_span_refs: dict[tuple[NodeId, int], int] = {} + self.inflight_backup_pools: dict[NodeId, set[PoolName]] = {} + # Nodes admitted during the current insert walk (None outside one). + self._walk_admitted: Optional[set[NodeId]] = None + # Launched writes: D2H in flight (by ack id), then the storage write + # in flight (by operation id). self.ongoing_write_through: dict[int, _UnifiedBufferBackupEntry] = {} self.ongoing_backup: dict[int, _UnifiedBufferBackupEntry] = {} + self._backup_ack_seq = 0 self.write_staged_tokens_ = 0 self.write_backlog_tokens_ = 0 self._backlog_cap_hits = 0 @@ -391,7 +378,9 @@ class BufferModePipeline: cc = self._cache.cache_controller anchor = cc.mem_pool_host.anchor_entry.host_pool if self._shared_host_domain() is None: - return kv_tokens + # Aux transfers staged through the anchor's own host pool (an + # entry aliasing it) occupy anchor tokens as well. + return kv_tokens + self._anchor_pool_aux_tokens(aux_xfers or []) num_bytes = sum( tokens * pool.size_per_token for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers) @@ -420,6 +409,18 @@ class BufferModePipeline: ) return requests + @property + def inflight_backup_node_ids(self): + """Nodes with a write of any pool queued, staged or awaiting its + storage ack.""" + return self.inflight_backup_pools.keys() + + def backup_pending(self, node_id: NodeId) -> bool: + """A write of ``node_id`` is queued, staged or awaiting its storage + ack. A component that drops rows a pending write still reads asks + here first; the pipeline pins the node only from D2H launch to ack.""" + return node_id in self.inflight_backup_pools + def is_idle(self) -> bool: """No queued, staged, or in-flight buffer-mode work or anchor pins.""" return not ( @@ -444,19 +445,37 @@ class BufferModePipeline: # ---- backup pipeline (device -> staging -> storage) ---- - def _backup_parent_covered(self, state: BufferBackupState) -> bool: + def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int: + """Staging tokens aux transfers hold in the anchor's host pool.""" + group = self._cache.cache_controller.mem_pool_host + anchor_pool = group.anchor_entry.host_pool + tokens = 0 + for t in aux_xfers: + if t.indices_from_pool is not None: + continue + entry = group.entry_map.get(t.name) + if entry is not None and entry.host_pool is anchor_pool: + tokens += self._transfer_tokens(t) + return tokens + + def _backup_parent_covered(self, state: BufferBackupState, node_id: NodeId) -> bool: """Only admit a node whose parent is stored/in-flight: writing above - a dropped parent creates a permanent longest-prefix hole.""" - if ( - state.parent_is_root - or state.parent_node_id in self.inflight_backup_node_ids + a dropped parent creates a permanent longest-prefix hole. A component + may vouch for the parent through a pool of its own.""" + if state.parent_is_root or PoolName.KV in self.inflight_backup_pools.get( + state.parent_node_id, () ): return True - return ( + if ( state.parent_last_hash is not None - and self._cache.storage_existence_cache.contains( - PoolName.KV, state.parent_last_hash + and self._cache.storage_existence_cache.pool(PoolName.KV).contains( + state.parent_last_hash ) + ): + return True + return any( + component.buffer_backup_parent_covered(state.parent_node_id, node_id) + for component in self._cache.components.values() ) def _log_backup_dropped(self, num_tokens: int) -> None: @@ -464,129 +483,229 @@ class BufferModePipeline: if cache.enable_storage_metrics and cache.storage_metrics_collector is not None: cache.storage_metrics_collector.log_backup_dropped_tokens(num_tokens) + def _pool_covered(self, pool: PoolName, keys: list[str]) -> bool: + """Beliefs plus content past a D2H launch (republished content must + not re-write while the original write drains).""" + return self._cache.storage_existence_cache.pool(pool).covered(keys) + + def _transfer_keys( + self, transfer: PoolTransfer, hash_values: list[str] + ) -> Optional[list[str]]: + """The builder's keys, else the chain's trailing pages, one per staged + page (the SWA convention); None when the transfer stages nothing.""" + if transfer.keys is not None: + return list(transfer.keys) + if transfer.indices_from_pool is not None or transfer.device_indices is None: + return None + entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name) + if entry is None: + return None + num_pages = len(transfer.device_indices) // entry.host_pool.page_size + if num_pages == 0 or num_pages > len(hash_values): + return None + return list(hash_values[-num_pages:]) + + def _backup_pool_keys( + self, snapshot: BufferBackupSnapshot + ) -> dict[PoolName, list[str]]: + """Admission view of the keys a backup writes per pool: the KV chain, + each aux component's BACKUP_HOST transfers, component overrides on + top, every sidecar under its source's keys.""" + hash_values = snapshot.hash_values + tree_core = self._cache.tree_core + pool_keys: dict[PoolName, list[str]] = {PoolName.KV: list(hash_values)} + overrides = tree_core.buffer_backup_pool_keys(snapshot.node_id, hash_values) + for ct in self._cache.components: + # A component that names its pools' keys spares the transfer build. + if ct == ComponentType.FULL or ct in overrides: + continue + transfers = tree_core.build_hicache_transfers( + ct, snapshot.node_id, CacheTransferPhase.BACKUP_HOST + ) + for transfer in transfers or (): + keys = self._transfer_keys(transfer, hash_values) + if keys: + pool_keys.setdefault(transfer.name, []).extend(keys) + for named in overrides.values(): + for pool, keys in named.items(): + if keys: + pool_keys[pool] = list(keys) + else: + pool_keys.pop(pool, None) + for spec in self._cache.sidecar_pool_specs: + source = pool_keys.get(spec.indices_from_pool) + if source: + pool_keys[spec.pool_name] = list(source) + return pool_keys + + @staticmethod + def _pool_order(pool: PoolName) -> tuple[bool, str]: + # KV first, then a fixed order: every rank must admit identically. + return (pool != PoolName.KV, pool.value) + + def _write_intents( + self, snapshot: BufferBackupSnapshot + ) -> list[tuple[PoolName, list[str]]]: + """(pool, keys) of the pools not believed stored or in flight; a + sidecar riding a covered pool drives that pool too.""" + pool_keys = self._backup_pool_keys(snapshot) + sidecars = { + spec.pool_name: spec.indices_from_pool + for spec in self._cache.sidecar_pool_specs + } + intents: list[tuple[PoolName, list[str]]] = [] + for pool in sorted(pool_keys, key=self._pool_order): + keys = pool_keys[pool] + if pool in sidecars or not keys: + continue + riders = [ + name + for name, source in sidecars.items() + if source == pool and pool_keys.get(name) + ] + if all( + self._pool_covered(name, pool_keys[name]) for name in (pool, *riders) + ): + continue + intents.append((pool, keys)) + return intents + + def _sizing( + self, pool: PoolName, keys: list[str] + ) -> tuple[int, list[PoolTransfer]]: + """(KV tokens, keys-only aux transfers) an intent stages, for the + oversize and budget gates.""" + if pool == PoolName.KV: + return len(keys) * self._cache.page_size, [] + return 0, [PoolTransfer(name=pool, keys=list(keys))] + + def _finish_inflight(self, node_id: NodeId, pool: PoolName) -> None: + pools = self.inflight_backup_pools.get(node_id) + if pools is None: + return + pools.discard(pool) + if not pools: + del self.inflight_backup_pools[node_id] + + @staticmethod + def _queued_span_key(intent: _UnifiedBackupIntent) -> tuple[NodeId, int]: + snapshot = intent.snapshot + return snapshot.node_id, len(snapshot.key) + + def _retain_queued_span(self, intent: _UnifiedBackupIntent) -> None: + span = self._queued_span_key(intent) + refs = self._queued_span_refs.get(span, 0) + if refs == 0: + self.write_backlog_tokens_ += span[1] + self._queued_span_refs[span] = refs + 1 + + def _release_queued_span(self, intent: _UnifiedBackupIntent) -> None: + span = self._queued_span_key(intent) + refs = self._queued_span_refs[span] + if refs == 1: + del self._queued_span_refs[span] + self.write_backlog_tokens_ -= span[1] + else: + self._queued_span_refs[span] = refs - 1 + + def begin_insert_walk(self) -> None: + """Admit each node once per insert walk: the trigger chains every + unbacked ancestor, so a depth-d path is presented d(d+1)/2 times.""" + self._walk_admitted = set() + + def end_insert_walk(self) -> None: + self._walk_admitted = None + def enqueue_backup_intent(self, node_id: NodeId) -> None: - """Snapshot a backup intent and commit it to the write queue. - Admission gates: belief skip, parent-cover, backlog cap, oversize. - Rejected intents are counted; the node re-triggers on a later hit.""" + """Queue one intent per pool with something to write; rejected intents + are counted and the node re-triggers on a later hit.""" if not self._cache.enable_storage: return - if node_id in self.inflight_backup_node_ids: - return + admitted = self._walk_admitted + if admitted is not None: + if node_id in admitted: + return + admitted.add(node_id) snapshot = self._cache.tree_core.snapshot_buffer_backup( node_id, self._cache.hicache_storage_pass_prefix_keys ) if snapshot is None: return - # Admission cover: beliefs plus content past its D2H launch. The - # launched cover keeps republished content (fill inserts under new - # node ids) from re-writing while the original write drains. - if self._cache.storage_existence_cache.covers_all( - PoolName.KV, - snapshot.hash_values, - extra_cover=self.inflight_backup_hashes, - ): + intents = self._write_intents(snapshot) + if not intents: return - intent_tokens = len(snapshot.hash_values) * self._cache.page_size - if self.write_backlog_tokens_ >= self.write_backlog_cap: - # The cap sits at 2x the intrinsic live-backlog ceiling (see - # init_hicache), so reaching it means leaked accounting or a - # broken stale sweep — a bug, not load. - self._backlog_cap_hits += 1 - if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0: - logger.error( - "HiCache write backlog cap hit (occurrence %d): " - "backlog=%d cap=%d queue=%d. Live backlog is bounded " - "by the device pool span, so this indicates a " - "stale-sweep or accounting leak.", - self._backlog_cap_hits, - self.write_backlog_tokens_, - self.write_backlog_cap, - len(self.pending_write_queue), - ) - self._log_backup_dropped(intent_tokens) - return - # A span larger than any pool's whole staging capacity can never - # stage; admitting it would wedge the head-of-line queue forever. - state = BufferBackupState( - parent_node_id=snapshot.parent_node_id, - parent_is_root=snapshot.parent_is_root, - parent_last_hash=snapshot.parent_last_hash, + pending = self.inflight_backup_pools.get(node_id, set()) + parent_covered = self._backup_parent_covered( + BufferBackupState( + parent_node_id=snapshot.parent_node_id, + parent_is_root=snapshot.parent_is_root, + parent_last_hash=snapshot.parent_last_hash, + ), + node_id, ) - if not self._backup_parent_covered(state) or self._backup_oversize( - snapshot.node_id, snapshot.hash_values, intent_tokens - ): - self._log_backup_dropped(intent_tokens) - return - - intent = _UnifiedBackupIntent(snapshot=snapshot) - self.pending_write_queue.append(intent) - self.inflight_backup_node_ids.add(snapshot.node_id) - self.write_backlog_tokens_ += intent_tokens - - def _build_aux_staging_transfers( - self, - node_id: NodeId, - hash_values: list[str], - comp_xfers: Optional[dict[ComponentType, list[PoolTransfer]]] = None, - ) -> list[PoolTransfer]: - """Keys-only aux transfers mirroring what BACKUP_STORAGE would write; - sizes the per-pool oversize gate (beliefs do not consult these).""" - transfers: list[PoolTransfer] = [] - if ComponentType.SWA in self._cache.components: - current = ( - comp_xfers.get(ComponentType.SWA) - if comp_xfers is not None - else self._cache.tree_core.build_hicache_transfers( - ComponentType.SWA, - node_id, - CacheTransferPhase.BACKUP_HOST, - ) - ) - for transfer in current or (): - if transfer.device_indices is None: - continue - num_pages = len(transfer.device_indices) // self._cache.page_size - if num_pages > 0: - transfers.append( - PoolTransfer( - name=PoolName.SWA, - keys=hash_values[-num_pages:], - hit_policy=PoolHitPolicy.TRAILING_PAGES, - ) + page_size = self._cache.page_size + span = (snapshot.node_id, len(snapshot.key)) + for pool, keys in intents: + if pool in pending: + continue + intent_tokens = len(keys) * page_size + if ( + span not in self._queued_span_refs + and self.write_backlog_tokens_ >= self.write_backlog_cap + ): + # The cap sits at 2x the intrinsic live-backlog ceiling (see + # init_hicache), so reaching it means leaked accounting or a + # broken stale sweep — a bug, not load. + self._backlog_cap_hits += 1 + if self._backlog_cap_hits <= 3 or self._backlog_cap_hits % 1000 == 0: + logger.error( + "HiCache write backlog cap hit (occurrence %d): " + "backlog=%d cap=%d queue=%d. Live backlog is bounded " + "by the device pool span, so this indicates a " + "stale-sweep or accounting leak.", + self._backlog_cap_hits, + self.write_backlog_tokens_, + self.write_backlog_cap, + len(self.pending_write_queue), ) - return transfers + self._log_backup_dropped(intent_tokens) + continue + # A span larger than the pool's whole staging capacity can never + # stage; admitting it would wedge the head-of-line queue forever. + if not parent_covered or self._backup_oversize(*self._sizing(pool, keys)): + self._log_backup_dropped(intent_tokens) + continue + intent = _UnifiedBackupIntent(snapshot=snapshot, pool=pool, keys=list(keys)) + self.pending_write_queue.append(intent) + self.inflight_backup_pools.setdefault(node_id, set()).add(pool) + self._retain_queued_span(intent) - def _backup_oversize( - self, - node_id: NodeId, - hash_values: list[str], - intent_tokens: int, - aux_xfers: Optional[list[PoolTransfer]] = None, - ) -> bool: + def _backup_oversize(self, kv_tokens: int, aux_xfers: list[PoolTransfer]) -> bool: """True if any pool's staging need exceeds that pool's write-usable capacity (total for KV, total minus the loads-priority margin for aux pools — matching ``_aux_budget_blocked``'s admission ceiling): such an - intent could never stage and would wedge the FIFO head.""" - if aux_xfers is None: - aux_xfers = self._build_aux_staging_transfers(node_id, hash_values) + intent could never stage and would wedge the FIFO head. Aux pools + staged through the anchor's host pool count against the KV total.""" cc = self._cache.cache_controller - shared_requests = self._shared_host_requests(intent_tokens, aux_xfers) + shared_requests = self._shared_host_requests(kv_tokens, aux_xfers) if shared_requests is not None: pool_tokens = cc.mem_pool_host.size max_write_units = pool_tokens - pool_tokens // 10 - if self._host_request_units(intent_tokens, aux_xfers) > max_write_units: + if self._host_request_units(kv_tokens, aux_xfers) > max_write_units: return True return not self._shared_backup_fits(shared_requests, empty=True) - if intent_tokens > cc.mem_pool_host.size: - return True - for t in aux_xfers or (): + anchor_pool = cc.mem_pool_host.anchor_entry.host_pool + anchor_need = kv_tokens + for t in aux_xfers: entry = cc.mem_pool_host.entry_map.get(t.name) - if entry is not None and ( - len(t.keys) * entry.host_pool.page_size - > entry.host_pool.size - self._aux_loads_margin(entry.host_pool) - ): + if entry is None: + continue + need = len(t.keys) * entry.host_pool.page_size + if entry.host_pool is anchor_pool: + anchor_need += need + elif need > entry.host_pool.size - self._aux_loads_margin(entry.host_pool): return True - return False + return anchor_need > cc.mem_pool_host.size def _aux_loads_margin(self, host_pool) -> int: """Aux-pool tokens reserved for loads: at least one trailing window @@ -623,9 +742,9 @@ class BufferModePipeline: snapshot = intent.snapshot state = self._validate_backup_intent(intent) if state is None: - self.inflight_backup_node_ids.discard(snapshot.node_id) - intent_tokens = len(snapshot.hash_values) * page_size - self.write_backlog_tokens_ -= intent_tokens + self._finish_inflight(snapshot.node_id, intent.pool) + intent_tokens = len(intent.keys) * page_size + self._release_queued_span(intent) swept_tokens += intent_tokens continue survivors.append(intent) @@ -657,54 +776,57 @@ class BufferModePipeline: while self.pending_write_queue: intent = self.pending_write_queue[0] snapshot = intent.snapshot - state = states[snapshot.node_id] - intent_tokens = len(snapshot.hash_values) * self._cache.page_size - if not self._backup_parent_covered(state): + node_id = snapshot.node_id + state = states[node_id] + intent_tokens = len(intent.keys) * self._cache.page_size + if not self._backup_parent_covered(state, node_id): # Cascade a dropped parent down the chain rather than creating # a permanent storage hole. - self.pending_write_queue.popleft() - self.inflight_backup_node_ids.discard(snapshot.node_id) - self.write_backlog_tokens_ -= intent_tokens - self._log_backup_dropped(intent_tokens) + self._drop_queue_head(intent_tokens, dropped=True) continue - if self.write_staged_tokens_ >= live_cap: + if ( + self.write_staged_tokens_ >= live_cap + and self._host_request_units(*self._sizing(intent.pool, intent.keys)) + > 0 + ): # Wait for current copies to free staging before rebuilding the head. break - device_value, comp_xfers = self._cache.tree_core.build_backup_spec( - snapshot.node_id - ) - sizing_xfers = self._build_aux_staging_transfers( - snapshot.node_id, snapshot.hash_values, comp_xfers - ) - if self._backup_oversize( - snapshot.node_id, - snapshot.hash_values, - intent_tokens, - sizing_xfers, + launch = self._launch_transfers(intent) + # The pool may have become covered since admission (an earlier + # launch this round, a sibling's ack): nothing left to write + # retires the intent, not a drop. + if launch is None or all( + self._pool_covered(pool, keys) for pool, keys in launch[1].items() ): + self._drop_queue_head(intent_tokens, dropped=False) + continue + device_value, keys_by_pool, transfers = launch + sizing_xfers = [t for t in transfers if t.indices_from_pool is None] + kv_tokens = len(device_value) + if self._backup_oversize(kv_tokens, sizing_xfers): # A permanently unstageable head must not block the queue. - self.pending_write_queue.popleft() - self.inflight_backup_node_ids.discard(snapshot.node_id) - self.write_backlog_tokens_ -= intent_tokens - self._log_backup_dropped(intent_tokens) + self._drop_queue_head(intent_tokens, dropped=True) continue shared_domain = self._shared_host_domain() staging_at_limit = ( self.write_staged_tokens_ - + self._host_request_units(intent_tokens, sizing_xfers) + + self._host_request_units(kv_tokens, sizing_xfers) > live_cap if shared_domain is not None else self.write_staged_tokens_ >= live_cap + and self._host_request_units(kv_tokens, sizing_xfers) > 0 ) if staging_at_limit: # Yield to live fetch demand; retry next round. break - if self._aux_budget_blocked(intent, sizing_xfers): + if self._aux_budget_blocked(kv_tokens, sizing_xfers): # An aux pool lacks staging headroom: yield at the gate # instead of failing the alloc inside cc.write; acks free # aux staging, retry next round. break - if not self._stage_backup_intent(intent, device_value, comp_xfers): + if not self._stage_backup_intent( + intent, device_value, transfers, keys_by_pool + ): # Pool full of in-flight staging and nothing reclaimable # (the tree never holds host values in buffer mode): # defer, head-of-line; pending acks will free slots. @@ -715,11 +837,106 @@ class BufferModePipeline: # Do not leave prepared copies deferred across scheduler admission. cc.start_writing() + def _drop_queue_head(self, intent_tokens: int, *, dropped: bool) -> None: + """Retire the FIFO head without a D2H launch; ``dropped`` feeds the + dropped-tokens metric (a pool with nothing left to write is not a + drop).""" + intent = self.pending_write_queue.popleft() + self._finish_inflight(intent.snapshot.node_id, intent.pool) + self._release_queued_span(intent) + if dropped: + self._log_backup_dropped(intent_tokens) + + def _build_backup_transfers( + self, node_id: NodeId, *, kv_only: bool = False + ) -> tuple[ + torch.Tensor, dict[ComponentType, list[PoolTransfer]], Optional[list[str]] + ]: + """The node's D2H spec. A FULL component's BACKUP_HOST KV transfer + replaces the KV rows (its keys name their pages); its other transfers + ride as aux pools. A KV intent skips the other components' transfers.""" + tree_core = self._cache.tree_core + if kv_only: + device_value = tree_core.get_component_device_value( + node_id, ComponentType.FULL + ) + comp_xfers = {} + else: + device_value, comp_xfers = tree_core.build_backup_spec(node_id) + full_xfers = tree_core.build_hicache_transfers( + ComponentType.FULL, node_id, CacheTransferPhase.BACKUP_HOST + ) + kv_keys = None + aux: list[PoolTransfer] = [] + for transfer in full_xfers or (): + if transfer.name == PoolName.KV: + device_value = ( + transfer.device_indices + if transfer.device_indices is not None + else device_value[:0] + ) + kv_keys = list(transfer.keys or ()) + else: + aux.append(transfer) + if aux: + comp_xfers[ComponentType.FULL] = aux + return device_value, comp_xfers, kv_keys + + def _launch_transfers( + self, intent: _UnifiedBackupIntent + ) -> Optional[tuple[torch.Tensor, dict[PoolName, list[str]], list[PoolTransfer]]]: + """(KV rows, keys per written pool, transfers to stage) of one intent, + off the node's fresh spec; None when nothing is left to write.""" + snapshot = intent.snapshot + hash_values = snapshot.hash_values + device_value, comp_xfers, kv_keys = self._build_backup_transfers( + snapshot.node_id, kv_only=intent.pool == PoolName.KV + ) + transfers: list[PoolTransfer] = [] + if intent.pool == PoolName.KV: + keys = hash_values if kv_keys is None else kv_keys + if not keys or len(device_value) == 0: + return None + comp_xfers = {} + else: + device_value = device_value[:0] + keys = [] + for ct, xfers in comp_xfers.items(): + kept = [] + for transfer in xfers: + if transfer.name != intent.pool or transfer.indices_from_pool: + continue + t_keys = self._transfer_keys(transfer, hash_values) + if not t_keys: + continue + if transfer.keys is None: + transfer.keys = t_keys + transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES + kept.append(transfer) + keys.extend(t_keys) + if kept: + transfers.extend(kept) + comp_xfers[ct] = kept + if not transfers: + return None + comp_xfers = {ct: xfers for ct, xfers in comp_xfers.items() if xfers} + keys_by_pool = {intent.pool: list(keys)} + # Sidecars ride their source pool's slots and D2H operation; they + # allocate no staging of their own. + for sidecar in self._cache._build_backup_sidecar(device_value, comp_xfers): + if sidecar.indices_from_pool != intent.pool: + continue + sidecar.keys = list(keys) + keys_by_pool[sidecar.name] = list(keys) + transfers.append(sidecar) + return device_value, keys_by_pool, transfers + def _stage_backup_intent( self, intent: _UnifiedBackupIntent, device_value: torch.Tensor, - comp_xfers: dict[ComponentType, list[PoolTransfer]], + transfers: list[PoolTransfer], + keys_by_pool: dict[PoolName, list[str]], ) -> bool: """Allocate host staging and pin one admitted intent's source. @@ -730,64 +947,54 @@ class BufferModePipeline: cache = self._cache cc = cache.cache_controller snapshot = intent.snapshot - aux_xfers = [x for xfers in comp_xfers.values() for x in xfers] - # Sidecars reuse the source pool's transient host/device indices. This - # includes both KV-derived pools and SWA-derived DSV4 state pools. They - # allocate no additional staging, but must ride the same D2H operation - # so their bytes are present when the storage write starts. - aux_xfers.extend(cache._build_backup_sidecar(device_value, comp_xfers)) + # One ack id per write: a node may have several pools in flight. + self._backup_ack_seq += 1 + ack_id = -(1 << 40) - self._backup_ack_seq host_indices = cc.write( device_value, - node_id=snapshot.node_id, - extra_pools=aux_xfers or None, + node_id=ack_id, + extra_pools=transfers or None, flush=False, ) if host_indices is None: + self._backup_ack_seq -= 1 return False - _track_content_refs(self.inflight_backup_hashes, snapshot.hash_values) + beliefs = cache.storage_existence_cache + for pool, keys in keys_by_pool.items(): + beliefs.pool(pool).track_inflight(keys) # NOTE: no commit_backup — the node must never appear # host-resident in buffer mode; staging slots live in the entry. lock_params = cache.inc_lock_ref(snapshot.node_id).to_dec_params() - occupied_units = self.host_allocation_units(host_indices, aux_xfers) - self.ongoing_write_through[snapshot.node_id] = _UnifiedBufferBackupEntry( + occupied_units = self.host_allocation_units(host_indices, transfers) + self.ongoing_write_through[ack_id] = _UnifiedBufferBackupEntry( intent=intent, host_indices=host_indices, - aux_xfers=aux_xfers, + aux_xfers=transfers, lock_params=lock_params, occupied_units=occupied_units, + keys_by_pool=keys_by_pool, ) self.write_staged_tokens_ += occupied_units - self.write_backlog_tokens_ -= len(snapshot.hash_values) * cache.page_size + self._release_queued_span(intent) return True - def _aux_budget_blocked( - self, - intent: _UnifiedBackupIntent, - aux: Optional[list[PoolTransfer]] = None, - ) -> bool: + def _aux_budget_blocked(self, kv_tokens: int, aux: list[PoolTransfer]) -> bool: """True when an aux pool cannot stage this intent right now (free minus the loads-priority margin falls short of the need): defer at the gate instead of failing the alloc inside cc.write and blocking pure-KV intents behind an unallocatable head. The margin enforces loads-have-priority on aux pools the way live_cap does on the KV pool; avail already reflects prefetch-held slots, so no occupancy - subtraction here.""" - snapshot = intent.snapshot - if aux is None: - aux = self._build_aux_staging_transfers( - snapshot.node_id, snapshot.hash_values - ) + subtraction here. Aux pools staged through the anchor's host pool + are gated by live_cap and the allocation itself.""" cc = self._cache.cache_controller - shared_requests = self._shared_host_requests( - len(snapshot.hash_values) * self._cache.page_size, aux - ) + shared_requests = self._shared_host_requests(kv_tokens, aux) if shared_requests is not None: return not self._shared_backup_fits(shared_requests) - if not aux: - return False + anchor_pool = cc.mem_pool_host.anchor_entry.host_pool for t in aux: entry = cc.mem_pool_host.entry_map.get(t.name) - if entry is None: + if entry is None or entry.host_pool is anchor_pool: continue need = len(t.keys) * entry.host_pool.page_size headroom = entry.host_pool.available_size() - self._aux_loads_margin( @@ -797,23 +1004,6 @@ class BufferModePipeline: return True return False - def _aux_window_keys( - self, hash_values: list[str], transfer: PoolTransfer - ) -> Optional[list[str]]: - """Trailing KV page hashes keying an aux transfer's staged window - (one key per aux-pool page).""" - if transfer.host_indices is None or transfer.host_indices.numel() == 0: - return None - if transfer.indices_from_pool is not None: - return None # sidecar rides another pool's slots; nothing to key - entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name) - if entry is None: - return None - num_keys = len(transfer.host_indices) // entry.host_pool.page_size - if num_keys == 0 or num_keys > len(hash_values): - return None - return hash_values[-num_keys:] - def finish_backup_ack(self, ack_id: int) -> None: """D2H confirmed: drop the device lock and enqueue the storage write (which reads from the staging copy, so device eviction may proceed).""" @@ -822,73 +1012,65 @@ class BufferModePipeline: snapshot = intent.snapshot self._cache.dec_lock_ref(snapshot.node_id, entry.lock_params) - # Every independently staged aux pool writes a trailing snapshot keyed - # by the last KV page hashes it covers. A derived sidecar writes the - # exact key span of its source pool while reusing that source's slots. - storage_xfers: list[PoolTransfer] = [] - storage_sources = { - PoolName.KV: PoolTransfer( - name=PoolName.KV, - host_indices=entry.host_indices, - keys=snapshot.hash_values, - ) - } - for staged in entry.aux_xfers: - if staged.indices_from_pool is not None: - continue - keys = self._aux_window_keys(snapshot.hash_values, staged) - if keys is None: - continue - transfer = PoolTransfer( - name=staged.name, - host_indices=staged.host_indices, - keys=keys, - hit_policy=PoolHitPolicy.TRAILING_PAGES, - ) - storage_xfers.append(transfer) - storage_sources.setdefault(staged.name, transfer) - for staged in entry.aux_xfers: - if staged.indices_from_pool is None: - continue - source = storage_sources.get(staged.indices_from_pool) - if source is None: - raise AssertionError( - "Buffer-mode storage sidecar source missing: " - f"{staged.name} from {staged.indices_from_pool}." - ) - storage_xfers.append( - PoolTransfer( - name=staged.name, - keys=source.keys, - hit_policy=staged.hit_policy, - indices_from_pool=staged.indices_from_pool, - ) - ) + storage_xfers = self._storage_transfers(entry.aux_xfers) + # Another pool's write carries no KV pages; its keys ride the transfer. + hash_value = ( + list(entry.keys_by_pool[PoolName.KV]) if intent.pool == PoolName.KV else [] + ) operation_id = self._cache.cache_controller.write_storage( entry.host_indices, snapshot.key.token_ids, - snapshot.hash_values, + hash_value, snapshot.prefix_keys, extra_pools=storage_xfers or None, ) self.ongoing_backup[operation_id] = entry + @staticmethod + def _storage_transfers(staged: list[PoolTransfer]) -> list[PoolTransfer]: + """One storage transfer per pool: a pool staged in several transfers + becomes one contiguous key span over their slots; a derived sidecar + writes its source's key span over the source's slots.""" + merged: dict[PoolName, PoolTransfer] = {} + for transfer in staged: + if transfer.indices_from_pool is not None: + merged[transfer.name] = PoolTransfer( + name=transfer.name, + keys=list(transfer.keys), + hit_policy=transfer.hit_policy, + indices_from_pool=transfer.indices_from_pool, + ) + continue + earlier = merged.get(transfer.name) + if earlier is None: + merged[transfer.name] = PoolTransfer( + name=transfer.name, + host_indices=transfer.host_indices, + keys=list(transfer.keys), + hit_policy=transfer.hit_policy, + ) + continue + earlier.keys.extend(transfer.keys) + earlier.host_indices = torch.cat( + [earlier.host_indices, transfer.host_indices] + ) + return list(merged.values()) + def finish_storage_write_ack(self, operation_id: int) -> None: - """Storage write acked (rank-synced drain): free the entry's staging - outright. Existence entries are added unconditionally - (completed_tokens can diverge across ranks under backend failure) to - keep admission decisions TP-deterministic. No-op for operations this - pipeline does not own (e.g. acks for already-reset state).""" + """Storage ack (rank-synced drain): free the staging and feed each + written pool's beliefs unconditionally, keeping admission + TP-deterministic under per-rank backend failures.""" entry = self.ongoing_backup.pop(operation_id, None) if entry is None: return - intent = entry.intent - snapshot = intent.snapshot - self._cache.storage_existence_cache.add(PoolName.KV, snapshot.hash_values) + beliefs = self._cache.storage_existence_cache + for pool, keys in entry.keys_by_pool.items(): + pool_beliefs = beliefs.pool(pool) + pool_beliefs.add(keys) + pool_beliefs.untrack_inflight(keys) self._free_staging_now(entry.host_indices, entry.aux_xfers) self.write_staged_tokens_ -= entry.occupied_units - self.inflight_backup_node_ids.discard(snapshot.node_id) - _untrack_content_refs(self.inflight_backup_hashes, snapshot.hash_values) + self._finish_inflight(entry.intent.snapshot.node_id, entry.intent.pool) def _free_staging_now( self, host_indices: torch.Tensor, aux_xfers: list[PoolTransfer] @@ -1163,10 +1345,16 @@ class BufferModePipeline: staged_pages = num_tokens // cache.page_size staged_hashes = hash_value[:staged_pages] staged_kv = host_indices[:num_tokens] - # Feed existence beliefs from the storage-fetched pages: the fetch - # itself is the evidence, so feeding is sound even if this staged - # prefetch is later dropped unconsumed. - cache.storage_existence_cache.add(PoolName.KV, list(staged_hashes)) + # The fetch is evidence for every pool it delivered (sound even if the + # staged prefetch is later dropped): KV-derived sidecars came with the + # staged pages, other pools with the keys their transfer carries. + beliefs = cache.storage_existence_cache + beliefs.pool(PoolName.KV).add(staged_hashes) + for transfer in aux_xfers: + if transfer.indices_from_pool == PoolName.KV: + beliefs.pool(transfer.name).add(staged_hashes) + elif transfer.keys: + beliefs.pool(transfer.name).add(transfer.keys) self.staged_prefetches[request] = _StagedPrefetch( request=request, key_tokens=array( diff --git a/python/sglang/srt/mem_cache/buffer_mode/storage_existence_cache.py b/python/sglang/srt/mem_cache/buffer_mode/storage_existence_cache.py index b9ef1e9c7d..cc883203cf 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/storage_existence_cache.py +++ b/python/sglang/srt/mem_cache/buffer_mode/storage_existence_cache.py @@ -1,17 +1,18 @@ -"""Local existence cache for HiCache buffer_only mode. +"""Local existence beliefs for HiCache buffer_only mode. In buffer mode host memory holds no persistent copy, so without a local existence signal every re-insert of a hot prefix re-stages and re-writes to -L3 storage. This cache is that signal: a bounded LRU of (pool, page-hash) -entries *believed* present in storage. +L3 storage. The beliefs are that signal: per storage pool, a bounded LRU of +page hashes *believed* present in storage, kept and healed independently so +a node is written pool by pool (only the pools not believed stored). Semantics are advisory, not authoritative: -- A hit skips the redundant D2H + storage write. +- A hit skips the pool's redundant D2H staging + storage write. - A stale positive (backend evicted the data) costs skipped write-backs until - a prefetch hit-query shortfall invalidates the entries; the next insert - then writes the data back. Never a correctness issue — at worst one cold - recompute, the same as any cache miss. + a prefetch hit-query shortfall invalidates the pool's entries; the next + insert then writes that pool back. Never a correctness issue — at worst + one cold recompute, the same as any cache miss. - A miss (entry LRU-evicted or never seen) just costs one redundant write (idempotent: storage keys are content-addressed). @@ -21,6 +22,10 @@ recompute (same tokens => same chain). Page hashes are chained and the write path is prefix-contiguous (parent-cover gate), so a node's own page set is the only thing a caller needs to check. +A pool also tracks its in-flight content (D2H launch to storage ack), which +admission treats as covered; its ``PoolBeliefPolicy`` says how its keys sit +on the chain (a component may supply one via ``buffer_belief_policy``). + TP determinism: replicas stay identical because every mutation happens on the scheduler thread at lockstep points with cross-rank-reduced inputs (storage-ack drain, prefetch-hit drain, fill commit). Do not touch it from @@ -30,82 +35,165 @@ anywhere else. from __future__ import annotations from collections import OrderedDict -from collections.abc import Container, Iterable, Sequence +from collections.abc import Iterable, Sequence -# ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers roughly -# 32M tokens at page size 64 across all pools. +from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy + +# Per pool: ~524K entries; at ~150-250 B/entry this is <= ~125 MiB and covers +# roughly 32M tokens at page size 64. HICACHE_EXISTENCE_CACHE_MAX_ENTRIES = 512 * 1024 -class StorageExistenceCache: +class PoolExistenceBeliefs: + """One pool's LRU of page hashes believed stored, plus its refcounted + in-flight content (D2H launch to storage ack).""" + def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES): self.max_entries = max_entries - self._entries: OrderedDict[tuple[str, str], None] = OrderedDict() + self._entries: OrderedDict[str, None] = OrderedDict() + self._inflight: dict[str, int] = {} - def __len__(self) -> int: - return len(self._entries) - - def add(self, pool: str, hashes: Iterable[str]) -> None: + def add(self, hashes: Iterable[str]) -> None: entries = self._entries move_to_end = entries.move_to_end for h in hashes: - key = (pool, h) - entries[key] = None - move_to_end(key) + entries[h] = None + move_to_end(h) pop_oldest = entries.popitem for _ in range(len(entries) - self.max_entries): pop_oldest(last=False) - def contains(self, pool: str, page_hash: str) -> bool: + def contains(self, page_hash: str) -> bool: entries = self._entries - key = (pool, page_hash) - if key not in entries: + if page_hash not in entries: return False - entries.move_to_end(key) + entries.move_to_end(page_hash) return True - def contains_all(self, pool: str, hashes: Iterable[str]) -> bool: + def contains_all(self, hashes: Iterable[str]) -> bool: entries = self._entries move_to_end = entries.move_to_end for h in hashes: - key = (pool, h) try: - move_to_end(key) + move_to_end(h) except KeyError: return False return True - def covers_all( - self, - pool: str, - hashes: Iterable[str], - extra_cover: Container[str] = frozenset(), - ) -> bool: - """True when every page is believed stored or sits in - ``extra_cover`` (e.g. content past its D2H launch, which always - reaches its storage-ack). LRU-touches the believed entries.""" + def covered(self, hashes: Iterable[str]) -> bool: + """Every page believed stored or in flight; LRU-touches believed entries.""" + inflight = self._inflight + if not inflight: + return self.contains_all(hashes) entries = self._entries move_to_end = entries.move_to_end - if extra_cover: - for h in hashes: - key = (pool, h) - if key in entries: - move_to_end(key) - elif h not in extra_cover: - return False - return True - return self.contains_all(pool, hashes) + for h in hashes: + if h in entries: + move_to_end(h) + elif h not in inflight: + return False + return True - def invalidate_beyond( - self, pool: str, hashes: Sequence[str], keep_pages: int - ) -> None: - """Ground-truth heal from a prefetch hit query: discard beliefs - beyond the leading ``keep_pages`` of a hash chain (the folded - usable cut). The next insert re-writes the discarded span, closing - stale positives and aux holes at the cut.""" + def track_inflight(self, hashes: Iterable[str]) -> None: + """One ref per page hash at its D2H launch (several launched writes + can carry the same content).""" + refs = self._inflight + for h in hashes: + refs[h] = refs.get(h, 0) + 1 + + def untrack_inflight(self, hashes: Iterable[str]) -> None: + """Drop one ref per page hash at the storage ack (or a failed launch).""" + refs = self._inflight + for h in hashes: + n = refs.get(h, 0) - 1 + if n <= 0: + refs.pop(h, None) + else: + refs[h] = n + + def invalidate_beyond(self, hashes: Sequence[str], keep_pages: int) -> None: + """Discard the beliefs beyond the leading ``keep_pages`` of ``hashes``; + the next insert re-writes that span.""" pop = self._entries.pop for h in hashes[keep_pages:]: - pop((pool, h), None) + pop(h, None) def clear(self) -> None: self._entries.clear() + self._inflight.clear() + + def clear_inflight(self) -> None: + self._inflight.clear() + + +class StorageExistenceCache: + """The beliefs of every storage pool, created on first use.""" + + def __init__(self, max_entries: int = HICACHE_EXISTENCE_CACHE_MAX_ENTRIES): + self.max_entries = max_entries + self._pools: dict[str, PoolExistenceBeliefs] = {} + + def pool(self, name: str) -> PoolExistenceBeliefs: + beliefs = self._pools.get(name) + if beliefs is None: + beliefs = self._pools[name] = PoolExistenceBeliefs(self.max_entries) + return beliefs + + def clear(self) -> None: + for beliefs in self._pools.values(): + beliefs.clear() + + def clear_inflight(self) -> None: + for beliefs in self._pools.values(): + beliefs.clear_inflight() + + +class PoolBeliefPolicy: + """How a pool's keys sit on a node's page-hash chain, hence how a storage + verdict heals its beliefs: a rank-agreed hit-query boundary (``evidence`` + = the query reported this pool) or a transfer's delivered page count.""" + + def heal_from_query( + self, + beliefs: PoolExistenceBeliefs, + chain: Sequence[str], + keys: Sequence[str], + hit_pages: int, + *, + evidence: bool, + ) -> None: + raise NotImplementedError + + def heal_from_transfer( + self, + beliefs: PoolExistenceBeliefs, + keys: Sequence[str], + delivered_pages: int, + ) -> None: + beliefs.invalidate_beyond(keys, delivered_pages) + + +class ChainPagesPolicy(PoolBeliefPolicy): + """One key per chain page (ALL_PAGES). The agreed boundary is the MIN of + per-rank prefix hits, so every rank verified the chain up to it.""" + + def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence): + beliefs.invalidate_beyond(chain, hit_pages) + if evidence: + beliefs.add(chain[:hit_pages]) + + +class TrailingWindowPolicy(PoolBeliefPolicy): + """Keys are a trailing window of the chain (TRAILING_PAGES). A rank only + verified the window ending at its own boundary, so the query only + invalidates; acks and fills teach presence.""" + + def heal_from_query(self, beliefs, chain, keys, hit_pages, *, evidence): + beliefs.invalidate_beyond(chain, hit_pages) + + +def default_belief_policy(hit_policy: PoolHitPolicy) -> PoolBeliefPolicy: + """The built-in policy for a pool no component claims.""" + if hit_policy == PoolHitPolicy.TRAILING_PAGES: + return TrailingWindowPolicy() + return ChainPagesPolicy() diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index ec4d7390de..6698a40bce 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -51,6 +51,10 @@ from sglang.srt.utils import broadcast_pyobj logger = logging.getLogger(__name__) +# Pool boundaries are nonnegative; an omitted rank must dominate MIN so +# a peer cannot establish presence of that rank's storage shard. +_NO_POOL_VERDICT = -1 + @dataclass(frozen=True) class PPPrefetchPoolSpec: @@ -165,6 +169,10 @@ class PrefetchOperation(StorageOperation): pool_transfers=pool_transfers, ) self.pool_transfers_done = not bool(pool_transfers) + # The hit query's verdict per pool, as the chain prefix length (in + # pages) up to which the pool is present; rank-reduced with the + # folded hit count. Empty until the query ran on this rank. + self.query_pool_hit_pages: dict[PoolName, int] = {} self.buffer_host_occupied_units: Optional[int] = None # The Python transfer worker leaves the unfinished tail to the ACK drain; # a controller that releases it itself must set this False. @@ -1348,9 +1356,11 @@ class HybridCacheController(BaseHiCacheController): if operation.assume_stored: # A prior hit on a suffix of this span proved it stored, and writes - # are prefix-covered, so re-querying only adds a round trip. + # are prefix-covered, so re-querying only adds a round trip. Nothing + # is verified now, so no pool gets a verdict to learn from. kv_hit_pages = len(hash_value) operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages) + operation.query_pool_hit_pages = {} return hash_value, kv_hit_pages * self.page_size extra_info = HiCacheStorageExtraInfo( @@ -1369,12 +1379,45 @@ class HybridCacheController(BaseHiCacheController): kv_hit_pages = hit_result.kv_hit_pages operation.pool_storage_result.update_kv_hit_pages(kv_hit_pages) + # Each pool's own boundary (the KV pool's where the backend reports + # it, else the folded cut); the beliefs heal pool by pool. A pool the + # backend did not report carries no verdict. + pool_hits = hit_result.extra_pool_hit_pages + operation.query_pool_hit_pages = { + PoolName.KV: pool_hits.get(PoolName.KV, kv_hit_pages) + } + for transfer in operation.pool_transfers or (): + if transfer.name in pool_hits: + operation.query_pool_hit_pages[transfer.name] = pool_hits[transfer.name] return ( hash_value[:kv_hit_pages], kv_hit_pages * self.page_size, ) + def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int: + """One MIN collective for the folded hit count and every pool's own + boundary, so per-pool beliefs stay rank-identical. A pool omitted + by any rank heals from the folded cut without learning presence.""" + pools = list(PoolName) + local = operation.query_pool_hit_pages + packed = torch.tensor( + [storage_hit_count] + [local.get(pool, _NO_POOL_VERDICT) for pool in pools], + dtype=torch.int, + ) + self._all_reduce( + packed, + torch.distributed.ReduceOp.MIN, + self.prefetch_hits_sync_groups, + ) + reduced = packed.tolist() + operation.query_pool_hit_pages = { + pool: pages + for pool, pages in zip(pools, reduced[1:], strict=True) + if pages != _NO_POOL_VERDICT + } + return int(reduced[0]) + def _move_pool_indices( self, host_pool, host_indices, device_indices, *, write_back_jit: bool ) -> tuple[torch.Tensor, torch.Tensor]: diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index 36739f744d..151f81d8a2 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -17,7 +17,11 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, MatchResult, ) -from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult +from sglang.srt.mem_cache.hicache_storage import ( + PoolName, + PoolTransfer, + PoolTransferResult, +) from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401 BASE_COMPONENT_TYPE, ComponentType, @@ -25,7 +29,11 @@ from sglang.srt.mem_cache.unified_cache.component_type import ( # noqa: F401 if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req + from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import ( + PoolBeliefPolicy, + ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams + from sglang.srt.mem_cache.pool_host.group import HostPoolGroup, PoolEntry from sglang.srt.mem_cache.unified_cache.cache_action import ( CacheAction, ComponentAction, @@ -699,6 +707,40 @@ class TreeComponent(ABC): """Allocate prefetch staging sized by prepare_prefetch, once the hit is known.""" return None + # ---- Buffer-mode backup hooks (hicache_host_memory_mode=buffer_only) ---- + # Buffer mode writes a node pool by pool: keys per pool come from + # ``buffer_backup_keys`` or the BACKUP_HOST transfers, beliefs heal through + # ``buffer_belief_policy``; the pipeline pins the node from D2H launch to + # ack, so a component dropping rows asks ``buffer_backup_pending`` first. + + def buffer_mode_host_pool_entries( + self, host_pool_group: HostPoolGroup + ) -> list[PoolEntry]: + """Extra host pool entries this component stages through (an entry + may alias the anchor's host pool).""" + return [] + + def buffer_backup_keys( + self, node: UnifiedTreeNode, hash_values: list[str] + ) -> dict[PoolName, list[str]]: + """Per-pool storage keys a backup of ``node`` writes for this component + (empty list: withhold the pool; empty dict: derive them from its + BACKUP_HOST transfers). A component naming any pool names all of them.""" + return {} + + def buffer_backup_parent_covered( + self, parent_node_id: NodeId, node_id: NodeId + ) -> bool: + """Whether this component covers the parent of ``node_id`` without + the parent's KV pages in storage (no prefix hole for a reader).""" + return False + + def buffer_belief_policy(self, pool: PoolName) -> Optional[PoolBeliefPolicy]: + """The policy healing ``pool``'s existence beliefs, for a pool this + component owns; None picks the built-in one for the transfer's hit + policy.""" + return None + def build_hicache_transfers( self, node: UnifiedTreeNode, diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index d6cfe785e9..f2f0f267a4 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -1055,6 +1055,26 @@ class SWAComponent(TreeComponent): host_indices = self._swa_kv_pool_host.alloc(num_tokens) return host_indices + def buffer_backup_keys( + self, node: UnifiedTreeNode, hash_values: list[str] + ) -> dict[PoolName, list[str]]: + # Buffer mode stages the node's own SWA rows, one key per page from + # the chain's tail, so admission needs no transfer (and no device op). + if not self.tree_core.is_host_memory_buffer_only: + return {} + cd = node.component_data[self.component_type] + if not self.tree_core.has_swa_host_pool or cd.value is None: + return {PoolName.SWA: []} + rows = ( + node.component_data[BASE_COMPONENT_TYPE].value + if self._unified_allocator() is not None + else cd.value + ) + num_pages = len(rows) // self._swa_kv_pool_host.page_size + if num_pages == 0 or num_pages > len(hash_values): + return {PoolName.SWA: []} + return {PoolName.SWA: list(hash_values[-num_pages:])} + def build_hicache_transfers( self, node: UnifiedTreeNode, diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index b8fdb70c65..93d640ed10 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -2381,6 +2381,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): """Read a node's device->host backup spec (device value + component transfers) now.""" return self._build_backup_spec(self.node_by_id(node_id)) + def buffer_backup_pool_keys( + self, node_id: NodeId, hash_values: list[str] + ) -> dict[ComponentType, dict[PoolName, list[str]]]: + node = self.node_by_id(node_id) + named: dict[ComponentType, dict[PoolName, list[str]]] = {} + for comp in self.components: + pool_keys = comp.buffer_backup_keys(node, hash_values) + if pool_keys: + named[comp.component_type] = pool_keys + return named + + def build_backup_kv_action(self, node_id: NodeId) -> BackupKV: + return self._build_backup_kv_action(self.node_by_id(node_id)) + def _build_backup_spec(self, node: UnifiedTreeNode): """Gather missing Full and component transfers for Host backup.""" device_value = node.component_data[BASE_COMPONENT_TYPE].value diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index 32e2f1a57c..b7fdc129fe 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -127,7 +127,11 @@ if TYPE_CHECKING: MatchResult, ) from sglang.srt.mem_cache.events import KVCacheEventRecorder - from sglang.srt.mem_cache.hicache_storage import PoolTransfer, PoolTransferResult + from sglang.srt.mem_cache.hicache_storage import ( + PoolName, + PoolTransfer, + PoolTransferResult, + ) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.cache_action import ( BackupKV, @@ -549,6 +553,20 @@ class UnifiedTreeCoreInterface(ABC): """Read a node's device->host backup spec (device value + transfers) now.""" ... + def buffer_backup_pool_keys( + self, node_id: NodeId, hash_values: list[str] + ) -> dict[ComponentType, dict[PoolName, list[str]]]: + """Per component, the storage keys per pool a buffer-mode backup of + the node writes (``TreeComponent.buffer_backup_keys``); Rust cores name none.""" + return {} + + def build_backup_kv_action(self, node_id: NodeId) -> BackupKV: + """The write-through backup action for a node and its unstored + ancestors; only component-requested buffer backups need it.""" + raise NotImplementedError( + f"{type(self).__name__} does not expose build_backup_kv_action" + ) + @abstractmethod def build_storage_backup_spec( self, node_id: NodeId, pass_prefix_keys: bool diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 88b370be3c..28ac43bbb6 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -36,10 +36,17 @@ from sglang.srt.mem_cache.buffer_mode.pipeline import ( validate_buffer_only_stack, ) from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import ( + PoolBeliefPolicy, StorageExistenceCache, + default_belief_policy, ) from sglang.srt.mem_cache.common import RetractionBackup -from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer, SidecarPoolSpec +from sglang.srt.mem_cache.hicache_storage import ( + PoolHitPolicy, + PoolName, + PoolTransfer, + SidecarPoolSpec, +) from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( HybridCacheController, PPPrefetchDecision, @@ -391,6 +398,8 @@ class UnifiedRadixCache(BasePrefixCache): self.ongoing_backup: dict[int, tuple[NodeId, DecLockRefParams]] = {} if self.buffer_pipeline is not None: self.buffer_pipeline.reset() + # The dropped writes' content no longer reaches a storage ack. + self.storage_existence_cache.clear_inflight() if self.cache_controller is not None: self.cache_controller.reset() @@ -487,10 +496,9 @@ class UnifiedRadixCache(BasePrefixCache): if swa is not None and self.tree_core.has_swa_host_pool else 0 ), - # Leak backstop only: live queued tokens are intrinsically - # bounded by the FULL device pool (one intent per node, stale - # intents swept per tick), so a cap that binds on live - # content would drop-newest and punch storage holes. + # Leak backstop only: the queue is charged once per unique node + # span across pools; a cap binding on live content would + # drop-newest and punch storage holes. write_backlog_cap=2 * self.token_to_kv_pool_allocator.size_full, ) # State initialization @@ -498,6 +506,13 @@ class UnifiedRadixCache(BasePrefixCache): self.cache_controller.host_write_staged_tokens_fn = lambda: ( self.buffer_pipeline.write_staged_tokens_ ) + # Pools a component's backups stage through besides the stock + # ones (an entry may alias the anchor's host pool). + for component in self.components.values(): + for entry in component.buffer_mode_host_pool_entries( + self.host_pool_group + ): + self.cache_controller.register_host_pool_entry(entry) self.write_through_threshold = ( 1 if get_memory().hicache_write_policy == "write_through" else 2 ) @@ -596,6 +611,8 @@ class UnifiedRadixCache(BasePrefixCache): return InsertResult(prefix_len=0) # Fail fast on re-entrancy without touching the in-flight walk. assert not self.tree_core.has_ongoing_insert(), "re-entrant insert" + if self.buffer_pipeline is not None: + self.buffer_pipeline.begin_insert_walk() # Pump the resumable insert, applying each step's actions at its barrier. try: step = self.tree_core.begin_insert(params) @@ -609,6 +626,8 @@ class UnifiedRadixCache(BasePrefixCache): finally: # Drain still-pending actions so frees reach the allocator on abort. self._apply_cache_actions(self.tree_core.end_insert()) + if self.buffer_pipeline is not None: + self.buffer_pipeline.end_insert_walk() @rank_consensus(same_params=True, same_results=True) def evict(self, params: EvictParams) -> EvictResult: @@ -1577,6 +1596,22 @@ class UnifiedRadixCache(BasePrefixCache): self.writing_check(write_back=True) return True + def buffer_backup_pending(self, node_id: NodeId) -> bool: + """Whether a buffer-mode write of ``node_id`` is queued, staged or + awaiting its storage ack (see ``BufferModePipeline.backup_pending``).""" + return self.buffer_pipeline is not None and self.buffer_pipeline.backup_pending( + node_id + ) + + def request_buffer_backup(self, node_id: NodeId) -> None: + """Queue a buffer-mode backup of ``node_id`` (and its unstored + ancestors) outside the insert trigger; no-op without buffer storage.""" + if self.buffer_pipeline is None or not self.enable_storage: + return + self._execute_and_commit_kv_backup( + self.tree_core.build_backup_kv_action(node_id) + ) + def _execute_and_commit_kv_backup( self, action: BackupKV, write_back: bool = False ) -> int: @@ -2408,9 +2443,9 @@ class UnifiedRadixCache(BasePrefixCache): for transfer, count in zip(pool_transfers, pool_hit_pages) ) if pool_transfers and not all_succeeded: - # Drop the KV beliefs from the first page any pool failed to serve; - # the next insert then re-writes that span through one FULL check, - # restoring the missing aux pages. + # Drop each pool's beliefs beyond what it served (KV and its + # sidecars beyond the completed span); the next insert re-writes + # only the missing pools. invalidation_hashes = ( operation.sidecar_hash_values if operation.sidecar_hash_values is not None @@ -2418,18 +2453,29 @@ class UnifiedRadixCache(BasePrefixCache): ) trimmed_pages = len(invalidation_hashes) - len(hash_value) keep_pages = trimmed_pages + completed_tokens // self.page_size + beliefs = self.storage_existence_cache + if keep_pages < len(invalidation_hashes): + self._belief_policy( + PoolName.KV, PoolHitPolicy.ALL_PAGES + ).heal_from_transfer( + beliefs.pool(PoolName.KV), invalidation_hashes, keep_pages + ) + for transfer in operation.pool_transfers or (): + if transfer.indices_from_pool == PoolName.KV: + self._belief_policy( + transfer.name, transfer.hit_policy + ).heal_from_transfer( + beliefs.pool(transfer.name), + invalidation_hashes, + keep_pages, + ) for transfer, count in zip(pool_transfers, pool_hit_pages): - if transfer.keys is None: - keep_pages = 0 - elif count < len(transfer.keys): - # Aux transfers key the chain's trailing pages. - keep_pages = min( - keep_pages, - max(0, len(invalidation_hashes) - len(transfer.keys)), + if transfer.keys is not None and count < len(transfer.keys): + self._belief_policy( + transfer.name, transfer.hit_policy + ).heal_from_transfer( + beliefs.pool(transfer.name), transfer.keys, count ) - self.storage_existence_cache.invalidate_beyond( - PoolName.KV, invalidation_hashes, keep_pages=keep_pages - ) self._finish_storage_prefetch( request, fulfilled_tokens=0, reason="storage_transfer" ) @@ -2444,11 +2490,15 @@ class UnifiedRadixCache(BasePrefixCache): self.prefetch_loaded_storage_start_by_reqid.pop(request, None) logger.warning( "HiCache hybrid prefetch discarded req=%s completed=%d requested=%d " - "kv_beliefs_kept_pages=%d", + "kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s", request.rid, completed_tokens, expected_tokens, keep_pages, + { + getattr(transfer.name, "value", transfer.name): count + for transfer, count in zip(pool_transfers, pool_hit_pages) + }, ) return False return True @@ -2637,18 +2687,47 @@ class UnifiedRadixCache(BasePrefixCache): ), ) + def _belief_policy( + self, pool: PoolName, hit_policy: PoolHitPolicy + ) -> PoolBeliefPolicy: + """The policy healing ``pool``'s existence beliefs: the owning + component's, else the built-in one for the transfer's hit policy.""" + for component in self.components.values(): + policy = component.buffer_belief_policy(pool) + if policy is not None: + return policy + return default_belief_policy(hit_policy) + def _invalidate_absent_from_hit_query(self, operation) -> None: - """Drop KV beliefs beyond the folded usable cut (rank-synced): the - next insert then re-writes the node (all pools), healing stale - positives and aux holes at the cut through one FULL check.""" + """Heal each pool's beliefs beyond its rank-agreed boundary (a + chain-page pool also learns presence up to it); a pool without a + verdict heals from the folded cut.""" if self.host_memory_mode != "buffer_only": return chain = operation.all_hash_values if chain is None: return - self.storage_existence_cache.invalidate_beyond( - PoolName.KV, chain, keep_pages=operation.storage_hit_count // self.page_size + beliefs = self.storage_existence_cache + verdicts = operation.query_pool_hit_pages + folded_pages = operation.storage_hit_count // self.page_size + + def verdict(pool: PoolName) -> tuple[int, bool]: + pages = verdicts.get(pool) + return (folded_pages, False) if pages is None else (pages, True) + + kv_pages, kv_evidence = verdict(PoolName.KV) + self._belief_policy(PoolName.KV, PoolHitPolicy.ALL_PAGES).heal_from_query( + beliefs.pool(PoolName.KV), chain, chain, kv_pages, evidence=kv_evidence ) + for transfer in operation.pool_transfers or (): + hit_pages, evidence = verdict(transfer.name) + self._belief_policy(transfer.name, transfer.hit_policy).heal_from_query( + beliefs.pool(transfer.name), + chain, + list(transfer.keys or ()), + hit_pages, + evidence=evidence, + ) def _account_prefetch_outcome(self, operation, revoked: bool) -> None: """Feed the cumulative prefetch-outcome counters at the (rank-synced) diff --git a/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py b/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py index eb26245264..e25988c02e 100644 --- a/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py +++ b/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py @@ -2,31 +2,47 @@ import unittest from array import array +from pathlib import Path +from tempfile import TemporaryDirectory from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +import msgspec import torch from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle from sglang.srt.mem_cache.buffer_mode.pipeline import ( BufferModePipeline, - _UnifiedBackupIntent, validate_buffer_only_stack, ) +from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import ( + StorageExistenceCache, +) from sglang.srt.mem_cache.hicache_storage import ( + HiCacheFile, + HiCacheStorageConfig, PoolHitPolicy, PoolName, PoolTransfer, SidecarPoolSpec, ) +from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( + HybridCacheController, + PrefetchOperation, +) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.components.base import ( ComponentType, ) from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( BufferBackupSnapshot, + BufferBackupState, ) -from sglang.srt.mem_cache.unified_radix_cache import _OngoingPrefetch +from sglang.srt.mem_cache.unified_radix_cache import ( + UnifiedRadixCache, + _OngoingPrefetch, +) +from sglang.srt.mem_cache.utils import get_storage_hash_str from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=8, suite="base-a-test-cpu") @@ -44,6 +60,171 @@ class TestBufferModeSidecar(unittest.TestCase): ), ) + def test_file_query_missing_rank_preserves_swa_repair(self): + snapshot = self._backup_snapshot() + token_ids = list(snapshot.key.token_ids) + chain = get_storage_hash_str(token_ids, page_size=2) + snapshot = msgspec.structs.replace(snapshot, hash_values=chain) + swa = PoolTransfer( + name=PoolName.SWA, + keys=chain[-1:], + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + for peer_swa_pages in (0, 1): + with ( + TemporaryDirectory() as directory, + patch.dict( + "os.environ", {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": directory} + ), + ): + ranks, packed = [], [] + + def capture(values, reduce_op, groups): + self.assertEqual(reduce_op, torch.distributed.ReduceOp.MIN) + packed.append(values.clone()) + + for rank, swa_pages in enumerate((2, peer_swa_pages)): + backend = HiCacheFile( + HiCacheStorageConfig( + tp_rank=rank, + tp_size=2, + pp_rank=0, + pp_size=1, + attn_cp_rank=0, + attn_cp_size=1, + is_mla_model=False, + enable_storage_metrics=False, + is_page_first_layout=False, + model_name="swa-repair", + extra_config={"enable_metadata_cache": False}, + ) + ) + for pool, keys in ( + (PoolName.KV, chain), + (PoolName.SWA, chain[:swa_pages]), + ): + for key in keys: + Path( + directory, + backend._get_component_key(key, pool) + ".bin", + ).touch() + operation = PrefetchOperation( + CacheRequestHandle("repair", 0), token_ids, pool_transfers=[swa] + ) + controller = SimpleNamespace( + storage_backend=backend, + page_size=2, + prefetch_hits_sync_groups=[], + _all_reduce=capture, + ) + _, hit_tokens = HybridCacheController._storage_hit_query( + controller, operation + ) + self.assertEqual(operation.all_hash_values, chain) + self.assertEqual( + operation.query_pool_hit_pages, + { + PoolName.KV: 2, + **({PoolName.SWA: swa_pages} if swa_pages else {}), + }, + ) + HybridCacheController._sync_prefetch_hit_query( + controller, operation, hit_tokens + ) + ranks.append((controller, operation, hit_tokens)) + + # Only transport is simulated: reduce the tensors built by the + # actual controller, then deliver the same MIN to both ranks. + agreed = torch.stack(packed).amin(dim=0) + for rank, (controller, operation, hit_tokens) in enumerate(ranks): + with self.subTest(peer_swa_pages=peer_swa_pages, rank=rank): + controller._all_reduce = lambda values, *_: values.copy_(agreed) + operation.storage_hit_count = ( + HybridCacheController._sync_prefetch_hit_query( + controller, operation, hit_tokens + ) + ) + tree = MagicMock() + tree.build_hicache_transfers.return_value = [swa] + tree.buffer_backup_pool_keys.return_value = {} + pipeline = self._launch_pipeline(tree, []) + cache = pipeline._cache + cache.host_memory_mode, cache.page_size = "buffer_only", 2 + cache.components = { + ct: SimpleNamespace(buffer_belief_policy=lambda _: None) + for ct in (ComponentType.FULL, ComponentType.SWA) + } + cache._belief_policy = UnifiedRadixCache._belief_policy.__get__( + cache + ) + for pool in (PoolName.KV, PoolName.SWA): + cache.storage_existence_cache.pool(pool).add(chain) + UnifiedRadixCache._invalidate_absent_from_hit_query( + cache, operation + ) + intents = pipeline._write_intents(snapshot) + # Retain FULL while making only the missing SWA tail + # eligible, including when one rank omitted its verdict. + self.assertEqual(intents, [(PoolName.SWA, chain[-1:])]) + self.assertTrue( + cache.storage_existence_cache.pool( + PoolName.KV + ).contains_all(chain) + ) + self.assertEqual( + operation.storage_hit_count, peer_swa_pages * 2 + ) + self.assertEqual( + [ + cache.storage_existence_cache.pool( + PoolName.SWA + ).contains(h) + for h in chain + ], + [i < peer_swa_pages for i in range(2)], + ) + + # An assumed-stored retry verifies nothing now, so it carries no pool + # verdict and re-learns nothing a shortfall heal dropped. + operation = PrefetchOperation( + CacheRequestHandle("assumed", 0), + token_ids, + pool_transfers=[swa], + assume_stored=True, + ) + controller = SimpleNamespace( + storage_backend=None, + page_size=2, + prefetch_hits_sync_groups=[], + _all_reduce=lambda values, *_: None, + ) + _, hit_tokens = HybridCacheController._storage_hit_query(controller, operation) + operation.storage_hit_count = HybridCacheController._sync_prefetch_hit_query( + controller, operation, hit_tokens + ) + self.assertEqual( + (operation.storage_hit_count, operation.query_pool_hit_pages), (4, {}) + ) + pipeline = self._launch_pipeline(MagicMock(), []) + cache = pipeline._cache + cache.host_memory_mode, cache.page_size = "buffer_only", 2 + cache.components = { + ct: SimpleNamespace(buffer_belief_policy=lambda _: None) + for ct in (ComponentType.FULL, ComponentType.SWA) + } + cache._belief_policy = UnifiedRadixCache._belief_policy.__get__(cache) + cache.storage_existence_cache.pool(PoolName.KV).add(chain) + UnifiedRadixCache._invalidate_absent_from_hit_query(cache, operation) + self.assertTrue( + cache.storage_existence_cache.pool(PoolName.KV).contains_all(chain) + ) + self.assertFalse( + any( + cache.storage_existence_cache.pool(PoolName.SWA).contains(h) + for h in chain + ) + ) + @staticmethod def _dsv4_specs(): return [ @@ -120,7 +301,39 @@ class TestBufferModeSidecar(unittest.TestCase): swa_component=self._swa_component(), ) + @staticmethod + def _backup_snapshot(): + return BufferBackupSnapshot( + node_id=7, + parent_node_id=0, + parent_is_root=True, + parent_last_hash=None, + hash_values=["page-0", "page-1"], + key=RadixKey(array("q", [1, 2, 3, 4])), + prefix_keys=["page-p"], + ) + + @staticmethod + def _launch_pipeline(tree, sidecar_pool_specs): + cache = SimpleNamespace( + tree_core=tree, + components={ComponentType.FULL: SimpleNamespace()}, + sidecar_pool_specs=sidecar_pool_specs, + storage_existence_cache=StorageExistenceCache(), + ) + cache._build_sidecar_transfers = ( + UnifiedRadixCache._build_sidecar_transfers.__get__(cache) + ) + cache._build_backup_sidecar = UnifiedRadixCache._build_backup_sidecar.__get__( + cache + ) + pipeline = BufferModePipeline.__new__(BufferModePipeline) + pipeline._cache = cache + return pipeline + def test_write_stages_and_persists_dsv4_full_and_swa_sidecars(self): + """Each pool of a node is a write of its own; every sidecar rides its + source pool's write under the source's keys.""" page_size = 2 device_indices = torch.arange(4, dtype=torch.int64) host_indices = torch.arange(10, 14, dtype=torch.int64) @@ -139,49 +352,68 @@ class TestBufferModeSidecar(unittest.TestCase): ) for spec in self._dsv4_specs() ] + kv_sidecars = [s for s in sidecars if s.indices_from_pool == PoolName.KV] + swa_sidecars = [s for s in sidecars if s.indices_from_pool == PoolName.SWA] controller = MagicMock() controller.mem_pool_host.anchor_entry.host_pool.shared_allocation_domain = None controller.mem_pool_host.entry_map = { PoolName.SWA: SimpleNamespace( - host_pool=SimpleNamespace(page_size=page_size) + host_pool=SimpleNamespace( + page_size=page_size, + size=8, + available_size=lambda: 8, + free=MagicMock(), + ) ) } + controller.mem_pool_host.size = 32 + controller.mem_pool_host.anchor_entry.host_pool = SimpleNamespace() + controller.prefetch_tokens_occupied = 0 + ack_ids = [] def _write(device_value, node_id, extra_pools, flush): - self.assertEqual(node_id, 7) self.assertFalse(flush) - self.assertEqual( - [transfer.name for transfer in extra_pools], - [PoolName.SWA, *[transfer.name for transfer in sidecars]], - ) + ack_ids.append(node_id) # HostPoolGroup.resolve_host_transfers gives derived pools their # source pool's indices without allocating another staging span. - swa.host_indices = swa_host_indices - for sidecar in sidecars: - if sidecar.indices_from_pool == PoolName.KV: + if device_value.numel(): + self.assertEqual(extra_pools, kv_sidecars) + for sidecar in extra_pools: sidecar.host_indices = host_indices sidecar.device_indices = device_value - else: - sidecar.host_indices = swa_host_indices - sidecar.device_indices = swa_device_indices - return host_indices + return host_indices + self.assertEqual(extra_pools, [swa, *swa_sidecars]) + swa.host_indices = swa_host_indices + for sidecar in swa_sidecars: + sidecar.host_indices = swa_host_indices + sidecar.device_indices = swa_device_indices + return device_value controller.write.side_effect = _write - controller.write_storage.return_value = 99 + controller.write_storage.side_effect = [98, 99] cache = MagicMock() cache.cache_controller = controller cache.page_size = page_size + cache.enable_storage = True + cache.enable_storage_metrics = False + cache.hicache_storage_pass_prefix_keys = True + cache.components = {ComponentType.FULL: MagicMock()} + cache.sidecar_pool_specs = self._dsv4_specs() cache._build_backup_sidecar.return_value = sidecars + cache.storage_existence_cache = StorageExistenceCache() pipeline = BufferModePipeline.__new__(BufferModePipeline) pipeline._cache = cache - pipeline.ongoing_write_through = {} - pipeline.ongoing_backup = {} - pipeline.inflight_backup_hashes = {} - pipeline.write_staged_tokens_ = 0 - pipeline.write_backlog_tokens_ = len(device_indices) + pipeline._swa_window_pages = 1 + pipeline.write_backlog_cap = 8 + pipeline.reset() + pipeline._build_backup_transfers = lambda node_id, *, kv_only=False: ( + device_indices, + {ComponentType.SWA: [swa]}, + None, + ) hashes = ["page-0", "page-1"] snapshot = BufferBackupSnapshot( @@ -191,24 +423,51 @@ class TestBufferModeSidecar(unittest.TestCase): parent_last_hash=None, hash_values=hashes, key=RadixKey(array("q", [1, 2, 3, 4])), - prefix_keys=None, + prefix_keys=["page-p"], ) - intent = _UnifiedBackupIntent(snapshot=snapshot) - - self.assertTrue( - pipeline._stage_backup_intent( - intent, - device_indices, - comp_xfers={ComponentType.SWA: [swa]}, - ) + tree = cache.tree_core + tree.snapshot_buffer_backup.return_value = snapshot + tree.validate_buffer_backup.return_value = BufferBackupState(0, True, None) + tree.buffer_backup_pool_keys.return_value = { + ComponentType.SWA: {PoolName.SWA: hashes[-1:]} + } + pipeline.enqueue_backup_intent(7) + self.assertEqual( + [intent.pool for intent in pipeline.pending_write_queue], + [PoolName.KV, PoolName.SWA], ) - self.assertEqual(pipeline.ongoing_write_through[7].aux_xfers, [swa, *sidecars]) + pipeline.flush_pending_writes() + self.assertFalse(pipeline.pending_write_queue) + kv_ack, swa_ack = ack_ids + self.assertEqual(pipeline.ongoing_write_through[kv_ack].aux_xfers, kv_sidecars) + self.assertEqual( + pipeline.ongoing_write_through[swa_ack].aux_xfers, [swa, *swa_sidecars] + ) + expected_keys = {PoolName.KV: hashes, PoolName.SWA: hashes[-1:]} + for spec in self._dsv4_specs(): + expected_keys[spec.pool_name] = expected_keys[spec.indices_from_pool] + for pool, keys in expected_keys.items(): + # In flight (covered for admission) but not yet believed stored. + pool_beliefs = cache.storage_existence_cache.pool(pool) + self.assertTrue(pool_beliefs.covered(keys)) + self.assertFalse(pool_beliefs.contains_all(keys)) + self.assertEqual(pipeline.write_backlog_tokens_, 0) - pipeline.finish_backup_ack(7) + pipeline.finish_backup_ack(kv_ack) + pipeline.finish_backup_ack(swa_ack) + kv_call, swa_call = controller.write_storage.call_args_list + self.assertTrue(torch.equal(kv_call.args[0], host_indices)) + self.assertEqual(kv_call.args[2], hashes) + self.assertEqual(kv_call.args[3], ["page-p"]) + # No KV page in the SWA write; the window's keys ride the transfer. + self.assertEqual(swa_call.args[0].numel(), 0) + self.assertEqual(swa_call.args[2], []) + self.assertEqual(swa_call.args[3], ["page-p"]) storage_transfers = { transfer.name: transfer - for transfer in controller.write_storage.call_args.kwargs["extra_pools"] + for call in (kv_call, swa_call) + for transfer in call.kwargs["extra_pools"] } self.assertEqual(storage_transfers[PoolName.SWA].keys, [hashes[-1]]) self.assertTrue( @@ -216,14 +475,16 @@ class TestBufferModeSidecar(unittest.TestCase): ) for spec in self._dsv4_specs(): storage_sidecar = storage_transfers[spec.pool_name] - expected_keys = ( - hashes if spec.indices_from_pool == PoolName.KV else [hashes[-1]] - ) - self.assertEqual(storage_sidecar.keys, expected_keys) + self.assertEqual(storage_sidecar.keys, expected_keys[spec.pool_name]) self.assertEqual(storage_sidecar.hit_policy, spec.hit_policy) self.assertEqual(storage_sidecar.indices_from_pool, spec.indices_from_pool) self.assertIsNone(storage_sidecar.host_indices) - self.assertIn(99, pipeline.ongoing_backup) + self.assertEqual(set(pipeline.ongoing_backup), {98, 99}) + pipeline.finish_storage_write_ack(98) + self.assertTrue(pipeline.backup_pending(7)) + pipeline.finish_storage_write_ack(99) + self.assertFalse(pipeline.backup_pending(7)) + self.assertEqual(pipeline.write_staged_tokens_, 0) def test_completed_prefetch_keeps_dsv4_full_and_swa_sidecars_for_h2d(self): swa = PoolTransfer( diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index b271af60cf..67986f71b5 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -62,6 +62,7 @@ from sglang.srt.mem_cache.memory_pool import ( MHATokenToKVPool, ReqToTokenPool, ) +from sglang.srt.mem_cache.pool_host import PoolEntry from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -77,11 +78,13 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( SWARebuild, ) from sglang.srt.mem_cache.unified_cache.components.base import ( + BASE_COMPONENT_TYPE, CacheTransferPhase, ComponentType, EvictLayer, TreeComponent, ) +from sglang.srt.mem_cache.unified_cache.components.full import FullComponent from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment from sglang.srt.mem_cache.unified_cache.tree_core_registry import ( _TREE_CORE_REGISTRY, @@ -238,6 +241,46 @@ class _FakeFullComponent(TreeComponent): pass +class _ExtraPoolFullComponent(FullComponent): + """FULL component that also stores every page under a pool of its own, + staged through the anchor's host pool.""" + + EXTRA_POOL = PoolName.DRAFT + + def buffer_mode_host_pool_entries(self, host_pool_group): + anchor = host_pool_group.anchor_entry + return [ + PoolEntry( + name=self.EXTRA_POOL, + host_pool=anchor.host_pool, + device_pool=anchor.device_pool, + layer_mapper=anchor.layer_mapper, + ) + ] + + def buffer_backup_keys(self, node, hash_values): + return {self.EXTRA_POOL: list(hash_values)} + + def build_hicache_transfers(self, node, phase, **kwargs): + if phase == CacheTransferPhase.BACKUP_HOST: + # Two transfers of the pool, each keyed by its own pages: the + # storage write must keep the keys per transfer. + value = node.component_data[BASE_COMPONENT_TYPE].value + keys = list(node.hash_value) + split = len(keys) // 2 + page_size = self.tree_core.page_size + halves = [ + (value[: split * page_size], keys[:split]), + (value[split * page_size :], keys[split:]), + ] + return [ + PoolTransfer(name=self.EXTRA_POOL, device_indices=rows, keys=part) + for rows, part in halves + if part + ] + return super().build_hicache_transfers(node, phase, **kwargs) + + def _drop_hicache_atexit_pin(cache): """init_hicache registers ``cache.shutdown`` with atexit, which pins the fixture's device and host pools until process exit; hundreds of HiCache @@ -468,6 +511,9 @@ def build_fixture( enable_session_radix_cache: bool = False, tree_page_size: Optional[int] = None, mamba_cache_chunk_size: Optional[int] = None, + component_registry_override: Optional[ + dict[ComponentType, type[TreeComponent]] + ] = None, tree_core_backend: Optional[str] = None, ): """Create (tree, allocator, req_to_token_pool) from a CacheConfig. @@ -602,6 +648,7 @@ def build_fixture( enable_session_radix_cache=enable_session_radix_cache, eviction_policy=cfg.eviction_policy, is_eagle=cfg.is_eagle, + component_registry_override=component_registry_override, ) requested_backend = tree_core_backend or _selected_tree_core_test_backend() selected_backend = resolve_tree_core_backend(requested_backend, cache_init_params) @@ -1135,13 +1182,18 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase): # every in-flight backup. lock_params = cache.inc_lock_ref(leaf_id).to_dec_params() pipeline = BufferModePipeline.__new__(BufferModePipeline) + keys = list(snapshot.hash_values) + # Entries are keyed by the write's ack id, not by the node. pipeline.ongoing_write_through = { - leaf_id: _UnifiedBufferBackupEntry( - intent=_UnifiedBackupIntent(snapshot=snapshot), + -1: _UnifiedBufferBackupEntry( + intent=_UnifiedBackupIntent( + snapshot=snapshot, pool=PoolName.KV, keys=keys + ), host_indices=torch.empty(0, dtype=torch.int64), aux_xfers=[], lock_params=lock_params, occupied_units=0, + keys_by_pool={PoolName.KV: keys}, ) } cache.buffer_pipeline = pipeline @@ -4031,8 +4083,14 @@ class UnifiedRadixCacheSuite: pipeline = cache.buffer_pipeline controller = cache.cache_controller _write_backup(cache, leaf) - node_ids = [intent.snapshot.node_id for intent in pipeline.pending_write_queue] + # One intent per pool of every node on the path. + intents = list(pipeline.pending_write_queue) + node_ids = sorted({intent.snapshot.node_id for intent in intents}) self.assertGreaterEqual(len(node_ids), 2) + self.assertEqual( + pipeline.inflight_backup_pools[leaf], + {intent.pool for intent in intents if intent.snapshot.node_id == leaf}, + ) device_allocators = [allocator] if self.cfg.has_swa: @@ -4064,9 +4122,19 @@ class UnifiedRadixCacheSuite: self.assertFalse(pipeline.pending_write_queue) self.assertFalse(controller.write_queue) self.assertEqual(len(controller.ack_write_queue), 1) + # Every intent is a write of its own, acked under its own id. ack = controller.ack_write_queue[0] - self.assertEqual(ack.node_ids, node_ids) - self.assertEqual(set(pipeline.ongoing_write_through), set(node_ids)) + self.assertEqual(len(ack.node_ids), len(intents)) + self.assertEqual(list(ack.node_ids), list(pipeline.ongoing_write_through)) + self.assertEqual( + sorted( + { + entry.intent.snapshot.node_id + for entry in pipeline.ongoing_write_through.values() + } + ), + node_ids, + ) staged_avail = self._host_avail_sizes(cache) self.assertNotEqual(staged_avail, avail0) @@ -4076,7 +4144,7 @@ class UnifiedRadixCacheSuite: self.assertGreater(_device_lock_ref(cache, node_id, ComponentType.FULL), 0) cache.writing_check(finish_count=1) self.assertFalse(pipeline.ongoing_write_through) - self.assertEqual(len(pipeline.ongoing_backup), len(node_ids)) + self.assertEqual(len(pipeline.ongoing_backup), len(intents)) for node_id in node_ids: self.assertEqual(_device_lock_ref(cache, node_id, ComponentType.FULL), 0) # Each storage write still owns its staging until its separate ACK. @@ -4103,13 +4171,259 @@ class UnifiedRadixCacheSuite: ) for n in chain: self.assertTrue( - cache.storage_existence_cache.contains_all( - PoolName.KV, cache.tree_core.get_hash_values(n) + cache.storage_existence_cache.pool(PoolName.KV).contains_all( + cache.tree_core.get_hash_values(n) ) ) - # Re-hit absorbed by the (FULL-focused) belief skip. + for transfer in _aux_storage_key_transfers(cache, n) or (): + self.assertTrue( + cache.storage_existence_cache.pool(transfer.name).contains_all( + transfer.keys + ) + ) + # Re-hit absorbed when every pool is believed present. _write_backup(cache, leaf) self.assertNotIn(leaf, cache.buffer_pipeline.inflight_backup_node_ids) + + if self.cfg.has_swa: + # Only the SWA window dropped: the next insert queues the SWA pool + # alone. A private SWA arena's write proceeds even with FULL staging + # at the live cap (shared or anchor-aliased layouts keep the gate). + swa_transfer = _aux_storage_key_transfers(cache, leaf)[0] + cache.storage_existence_cache.pool(PoolName.SWA).invalidate_beyond( + swa_transfer.keys, keep_pages=0 + ) + host_group = controller.mem_pool_host + private_swa = ( + pipeline._shared_host_domain() is None + and host_group.entry_map[PoolName.SWA].host_pool + is not host_group.anchor_entry.host_pool + ) + staged_before = pipeline.write_staged_tokens_ + if private_swa: + pipeline.write_staged_tokens_ = host_group.size + with ( + mock.patch.object(controller, "write", wraps=controller.write) as stage, + mock.patch.object( + controller, "write_storage", wraps=controller.write_storage + ) as write_storage, + mock.patch.object( + controller, "page_set_func", wraps=controller.page_set_func + ) as write_full, + mock.patch.object( + pipeline, "_write_intents", wraps=pipeline._write_intents + ) as admissions, + ): + # The fixture parks the stock trigger; arm it here. + cache.write_through_threshold = 1 + self._insert(cache, allocator, req_to_token_pool, seq_ab) + self.assertEqual( + [ + (i.snapshot.node_id, i.pool) + for i in pipeline.pending_write_queue + ], + [(leaf, PoolName.SWA)], + ) + # One admission per node per walk. + self.assertEqual(admissions.call_count, len(node_ids)) + self.assertTrue(cache.buffer_backup_pending(leaf)) + self._pump_hicache_until( + cache, + lambda: ( + not pipeline.inflight_backup_node_ids + and not pipeline.ongoing_backup + ), + "selective buffer backup did not drain", + ) + self.assertFalse(cache.buffer_backup_pending(leaf)) + if private_swa: + pipeline.write_staged_tokens_ -= host_group.size - staged_before + self.assertTrue(stage.called) + self.assertTrue( + all(call.args[0].numel() == 0 for call in stage.call_args_list) + ) + # SWA writes carry no KV page; the window's keys ride the transfer. + self.assertTrue(write_storage.called) + snapshot = cache.tree_core.snapshot_buffer_backup( + leaf, cache.hicache_storage_pass_prefix_keys + ) + for call in write_storage.call_args_list: + host_indices, _, hash_value, prefix_keys = call.args[:4] + self.assertEqual(host_indices.numel(), 0) + self.assertEqual(hash_value, []) + transfers = call.kwargs["extra_pools"] + self.assertEqual( + [t.name for t in transfers if t.indices_from_pool is None], + [PoolName.SWA], + ) + self.assertEqual(prefix_keys, snapshot.prefix_keys) + write_full.assert_not_called() + self.assertTrue( + cache.storage_existence_cache.pool(PoolName.SWA).contains_all( + swa_transfer.keys + ) + ) + self.assertEqual(self._host_avail_sizes(cache), avail0) + cache.sanity_check() + + @staticmethod + def _writes_by_pool(write_storage, extra): + """Key tuples of the KV writes and of the ``extra`` pool's writes + among the storage writes recorded by ``write_storage``.""" + kv_writes, extra_writes = set(), set() + for call in write_storage.call_args_list: + host_indices, _, hash_value = call.args[:3] + transfers = call.kwargs.get("extra_pools") or [] + names = [transfer.name for transfer in transfers] + if hash_value: + assert extra not in names + kv_writes.add(tuple(hash_value)) + elif extra in names: + # The pool's transfers reach storage as one contiguous span. + assert host_indices.numel() == 0 and names == [extra] + extra_writes.add(tuple(transfers[0].keys)) + return kv_writes, extra_writes + + def test_buffer_only_writes_component_pools_independently(self): + """A component's own pool (aliased host entry, per-pool keys, + BACKUP_HOST transfers) is written, believed and re-written + independently of the KV pool.""" + self._skip_unsupported_hicache_test() + storage_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, storage_dir, ignore_errors=True) + + # Component buffer-mode hooks are Python-core only. + cache, allocator, req_to_token_pool = build_fixture( + self.cfg, + component_registry_override={ComponentType.FULL: _ExtraPoolFullComponent}, + tree_core_backend="python", + ) + self._init_buffer_hicache(cache, storage_dir) + extra = _ExtraPoolFullComponent.EXTRA_POOL + controller = cache.cache_controller + self.assertIn(extra, controller.mem_pool_host.entry_map) + avail0 = self._host_avail_sizes(cache) + + seq = self._buffer_swa_seq() + self._insert(cache, allocator, req_to_token_pool, seq) + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).last_device_node + chain = self._path_chain(cache, leaf) + pipeline = cache.buffer_pipeline + expected_intents = [] + expected_backlog = 0 + for node_id in chain: + snapshot = cache.tree_core.snapshot_buffer_backup( + node_id, cache.hicache_storage_pass_prefix_keys + ) + expected_backlog += len(snapshot.key) + expected_intents.extend( + (node_id, pool) for pool, _ in pipeline._write_intents(snapshot) + ) + pipeline.write_backlog_cap = expected_backlog + if self.cfg.has_swa: + self.assertEqual( + {pool for node_id, pool in expected_intents if node_id == leaf}, + {PoolName.KV, PoolName.DRAFT, PoolName.SWA}, + ) + self.assertFalse(cache.buffer_backup_pending(leaf)) + with mock.patch.object( + controller, "write_storage", wraps=controller.write_storage + ) as write_storage: + _write_backup(cache, leaf) + self.assertEqual( + [ + (intent.snapshot.node_id, intent.pool) + for intent in pipeline.pending_write_queue + ], + expected_intents, + ) + self.assertEqual(pipeline.write_backlog_tokens_, expected_backlog) + self.assertEqual(pipeline._backlog_cap_hits, 0) + self.assertTrue(cache.buffer_backup_pending(leaf)) + self._pump_hicache_until( + cache, + lambda: ( + not cache.buffer_pipeline.inflight_backup_node_ids + and not cache.buffer_pipeline.ongoing_backup + ), + "buffer backup pipeline did not drain", + ) + self.assertFalse(cache.buffer_backup_pending(leaf)) + # Each pool of each node is a write of its own: the KV chain with its + # pages, the component's pool with no KV rows and its transfers + # keyed by their own pages. + expected = {tuple(cache.tree_core.get_hash_values(n)) for n in chain} + kv_writes, extra_writes = self._writes_by_pool(write_storage, extra) + self.assertEqual(kv_writes, expected) + self.assertEqual(extra_writes, expected) + + page_hashes = self._all_page_hashes(cache, leaf) + self.assertEqual( + self._storage_exists_count( + cache, page_hashes, [PoolTransfer(name=extra, keys=page_hashes)] + ), + len(page_hashes), + ) + for n in chain: + keys = cache.tree_core.get_hash_values(n) + self.assertTrue( + cache.storage_existence_cache.pool(PoolName.KV).contains_all(keys) + ) + self.assertTrue( + cache.storage_existence_cache.pool(extra).contains_all(keys) + ) + self.assertEqual(self._host_avail_sizes(cache), avail0) + + # Only the component's pool dropped from the beliefs: its requested + # backup writes that pool alone, staged through the anchor's host + # pool with no KV rows. + for n in chain: + cache.storage_existence_cache.pool(extra).invalidate_beyond( + cache.tree_core.get_hash_values(n), keep_pages=0 + ) + with ( + mock.patch.object(controller, "write", wraps=controller.write) as stage, + mock.patch.object( + controller, "write_storage", wraps=controller.write_storage + ) as write_storage, + mock.patch.object( + controller, "page_set_func", wraps=controller.page_set_func + ) as write_full, + ): + cache.request_buffer_backup(leaf) + intents = list(pipeline.pending_write_queue) + self.assertEqual( + [(intent.snapshot.node_id, intent.pool) for intent in intents], + [(n, extra) for n in chain], + ) + self._pump_hicache_until( + cache, + lambda: ( + not pipeline.inflight_backup_node_ids + and not pipeline.ongoing_backup + ), + "component-pool backup did not drain", + ) + self.assertTrue(stage.called) + self.assertTrue(all(call.args[0].numel() == 0 for call in stage.call_args_list)) + self.assertEqual(len(write_storage.call_args_list), len(chain)) + kv_writes, extra_writes = self._writes_by_pool(write_storage, extra) + self.assertEqual(kv_writes, set()) + self.assertEqual(extra_writes, expected) + write_full.assert_not_called() + for n in chain: + self.assertTrue( + cache.storage_existence_cache.pool(extra).contains_all( + cache.tree_core.get_hash_values(n) + ) + ) + self.assertEqual(self._host_avail_sizes(cache), avail0) + + # Everything believed stored: a requested backup is absorbed. + cache.request_buffer_backup(leaf) + self.assertNotIn(leaf, pipeline.inflight_backup_node_ids) cache.sanity_check() def test_buffer_only_read_path_roundtrip(self): @@ -4228,10 +4542,18 @@ class UnifiedRadixCacheSuite: for component_type in cons.tree_components: self.assertIsNone(_host_value(cons, cur, component_type)) self.assertTrue( - cons.storage_existence_cache.contains_all( - PoolName.KV, self._all_page_hashes(cons, leaf) + cons.storage_existence_cache.pool(PoolName.KV).contains_all( + self._all_page_hashes(cons, leaf) ) ) + # The fetch is evidence for every pool it delivered, so the + # consumer's own re-write of the span is skipped pool by pool. + for transfer in _aux_storage_key_transfers(cons, leaf) or (): + self.assertTrue( + cons.storage_existence_cache.pool(transfer.name).contains_all( + transfer.keys + ) + ) loaded_k, loaded_v = self._snapshot_full_kv(cons_alloc, mc.device_indices) self.assertTrue(torch.equal(loaded_k, expected_k)) self.assertTrue(torch.equal(loaded_v, expected_v))