[Feat] hicache: Add opt-in tail-only SWA backups in buffer mode (#44173)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
xiezhq-hermann
parent
bba535ebb4
commit
173e0f2e9b
@@ -786,6 +786,9 @@ class Envs:
|
||||
# to consumption so eviction cannot waste the fetch. Cap = fraction of
|
||||
# the pool the pins may hold; 0 disables pinning.
|
||||
SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP = EnvFloat(0.5)
|
||||
# Buffer mode: store only the SWA window ending at each node (what a
|
||||
# prefetch ending there restores) instead of every SWA page.
|
||||
SGLANG_ENABLE_HICACHE_SWA_TAIL_ONLY_BACKUP = EnvBool(False)
|
||||
SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||
# Enable O_DIRECT when opening NIXL POSIX backend files (bypasses OS page cache).
|
||||
# Disable with SGLANG_HICACHE_NIXL_USE_DIRECT_IO=0 or via the
|
||||
|
||||
@@ -19,7 +19,7 @@ import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
|
||||
from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -1318,6 +1318,10 @@ class HiCacheController:
|
||||
)
|
||||
return int(hit_count.item())
|
||||
|
||||
def extend_storage_prefix(self, prefix_keys: Any, hash_values: List[str]) -> Any:
|
||||
"""Advance the backend's prefix context past omitted pages."""
|
||||
return prefix_keys + hash_values if prefix_keys is not None else None
|
||||
|
||||
def write_storage(
|
||||
self,
|
||||
host_indices: torch.Tensor,
|
||||
|
||||
@@ -288,6 +288,7 @@ class BufferModePipeline:
|
||||
self._cache = cache
|
||||
self._swa_component = swa_component
|
||||
self._auxiliary_swa_component = auxiliary_swa_component
|
||||
self._swa_tail_only = envs.SGLANG_ENABLE_HICACHE_SWA_TAIL_ONLY_BACKUP.get()
|
||||
# Metadata-only pending-write backlog cap, measured in unique FULL
|
||||
# snapshot spans; beyond it new spans are dropped at admission.
|
||||
self.write_backlog_cap = write_backlog_cap
|
||||
@@ -323,6 +324,9 @@ class BufferModePipeline:
|
||||
# Arbitrary: a deferral means the admission budget and the allocator disagree
|
||||
# about free slots, which waiting on decode rarely fixes.
|
||||
max_staged_admission_defers: int = 32
|
||||
# Opt-in: a backup stores only the SWA window ending at its node, the one
|
||||
# a prefetch ending there restores, instead of the node's every SWA page.
|
||||
_swa_tail_only: bool = False
|
||||
|
||||
def reset(self) -> None:
|
||||
# Load pipeline: hits awaiting a staging grant (park-and-retry),
|
||||
@@ -533,12 +537,37 @@ class BufferModePipeline:
|
||||
return None
|
||||
return list(hash_values[-num_pages:])
|
||||
|
||||
def _swa_tail_pages(self, pool: PoolName) -> int:
|
||||
"""The pool's backup window; zero keeps every page."""
|
||||
if self._swa_tail_only:
|
||||
for component in self._swa_components():
|
||||
if component.pool_name == pool:
|
||||
return component.full_window_pages
|
||||
return 0
|
||||
|
||||
def _trim_swa_backup_transfer(
|
||||
self, transfer: PoolTransfer, keys: list[str]
|
||||
) -> list[str]:
|
||||
"""Stage and store only the node's tail window; rows follow the keys."""
|
||||
tail = self._swa_tail_pages(transfer.name)
|
||||
if not tail or len(keys) <= tail:
|
||||
return keys
|
||||
entry = self._cache.cache_controller.mem_pool_host.entry_map[transfer.name]
|
||||
page_size = entry.host_pool.page_size
|
||||
rows = transfer.device_indices
|
||||
assert rows is not None and len(rows) == len(keys) * page_size
|
||||
transfer.device_indices = rows[-tail * page_size :]
|
||||
transfer.keys = keys[-tail:]
|
||||
transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES
|
||||
return transfer.keys
|
||||
|
||||
def _backup_pool_keys(
|
||||
self, snapshot: BufferBackupSnapshot
|
||||
) -> dict[PoolName, list[str]]:
|
||||
"""Admission view of the keys a backup writes per pool: the KV chain,
|
||||
each aux component's BACKUP_HOST transfers, component overrides on
|
||||
top, every sidecar under its source's keys."""
|
||||
top, the SWA tail under tail-only backups, every sidecar under its
|
||||
source's keys."""
|
||||
hash_values = snapshot.hash_values
|
||||
tree_core = self._cache.tree_core
|
||||
pool_keys: dict[PoolName, list[str]] = {PoolName.KV: list(hash_values)}
|
||||
@@ -560,6 +589,10 @@ class BufferModePipeline:
|
||||
pool_keys[pool] = list(keys)
|
||||
else:
|
||||
pool_keys.pop(pool, None)
|
||||
for pool, keys in pool_keys.items():
|
||||
tail = self._swa_tail_pages(pool)
|
||||
if tail and len(keys) > tail:
|
||||
pool_keys[pool] = keys[-tail:]
|
||||
for spec in self._cache.sidecar_pool_specs:
|
||||
source = pool_keys.get(spec.indices_from_pool)
|
||||
if source:
|
||||
@@ -1001,6 +1034,7 @@ class BufferModePipeline:
|
||||
t_keys = self._transfer_keys(transfer, hash_values)
|
||||
if not t_keys:
|
||||
continue
|
||||
t_keys = self._trim_swa_backup_transfer(transfer, t_keys)
|
||||
if transfer.keys is None:
|
||||
transfer.keys = t_keys
|
||||
transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES
|
||||
@@ -1109,11 +1143,20 @@ class BufferModePipeline:
|
||||
hash_value = (
|
||||
list(entry.keys_by_pool[PoolName.KV]) if intent.pool == PoolName.KV else []
|
||||
)
|
||||
prefix_keys = snapshot.prefix_keys
|
||||
if intent.pool in (PoolName.SWA, PoolName.AUXILIARY_SWA):
|
||||
keys = entry.keys_by_pool[intent.pool]
|
||||
skipped_pages = len(snapshot.hash_values) - len(keys)
|
||||
if skipped_pages > 0:
|
||||
# A pool-only suffix starts after the omitted node pages.
|
||||
prefix_keys = self._cache.cache_controller.extend_storage_prefix(
|
||||
prefix_keys, snapshot.hash_values[:skipped_pages]
|
||||
)
|
||||
operation_id = self._cache.cache_controller.write_storage(
|
||||
entry.host_indices,
|
||||
snapshot.key.token_ids,
|
||||
hash_value,
|
||||
snapshot.prefix_keys,
|
||||
prefix_keys,
|
||||
extra_pools=storage_xfers or None,
|
||||
)
|
||||
self.ongoing_backup[operation_id] = entry
|
||||
|
||||
Reference in New Issue
Block a user