diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a9d55a3938..6e88182897 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -786,6 +786,9 @@ class Envs: # to consumption so eviction cannot waste the fetch. Cap = fraction of # the pool the pins may hold; 0 disables pinning. SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP = EnvFloat(0.5) + # Buffer mode: store only the SWA window ending at each node (what a + # prefetch ending there restores) instead of every SWA page. + SGLANG_ENABLE_HICACHE_SWA_TAIL_ONLY_BACKUP = EnvBool(False) SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR = EnvStr(None) # Enable O_DIRECT when opening NIXL POSIX backend files (bypasses OS page cache). # Disable with SGLANG_HICACHE_NIXL_USE_DIRECT_IO=0 or via the diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 4af640c00b..710eff6167 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -19,7 +19,7 @@ import threading import time from dataclasses import dataclass from queue import Empty, Queue -from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional +from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional import torch @@ -1318,6 +1318,10 @@ class HiCacheController: ) return int(hit_count.item()) + def extend_storage_prefix(self, prefix_keys: Any, hash_values: List[str]) -> Any: + """Advance the backend's prefix context past omitted pages.""" + return prefix_keys + hash_values if prefix_keys is not None else None + def write_storage( self, host_indices: torch.Tensor, diff --git a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py index e68a8f7517..2a33d7d2b7 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py +++ b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py @@ -288,6 +288,7 @@ class BufferModePipeline: self._cache = cache self._swa_component = swa_component self._auxiliary_swa_component = auxiliary_swa_component + self._swa_tail_only = envs.SGLANG_ENABLE_HICACHE_SWA_TAIL_ONLY_BACKUP.get() # Metadata-only pending-write backlog cap, measured in unique FULL # snapshot spans; beyond it new spans are dropped at admission. self.write_backlog_cap = write_backlog_cap @@ -323,6 +324,9 @@ class BufferModePipeline: # Arbitrary: a deferral means the admission budget and the allocator disagree # about free slots, which waiting on decode rarely fixes. max_staged_admission_defers: int = 32 + # Opt-in: a backup stores only the SWA window ending at its node, the one + # a prefetch ending there restores, instead of the node's every SWA page. + _swa_tail_only: bool = False def reset(self) -> None: # Load pipeline: hits awaiting a staging grant (park-and-retry), @@ -533,12 +537,37 @@ class BufferModePipeline: return None return list(hash_values[-num_pages:]) + def _swa_tail_pages(self, pool: PoolName) -> int: + """The pool's backup window; zero keeps every page.""" + if self._swa_tail_only: + for component in self._swa_components(): + if component.pool_name == pool: + return component.full_window_pages + return 0 + + def _trim_swa_backup_transfer( + self, transfer: PoolTransfer, keys: list[str] + ) -> list[str]: + """Stage and store only the node's tail window; rows follow the keys.""" + tail = self._swa_tail_pages(transfer.name) + if not tail or len(keys) <= tail: + return keys + entry = self._cache.cache_controller.mem_pool_host.entry_map[transfer.name] + page_size = entry.host_pool.page_size + rows = transfer.device_indices + assert rows is not None and len(rows) == len(keys) * page_size + transfer.device_indices = rows[-tail * page_size :] + transfer.keys = keys[-tail:] + transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES + return transfer.keys + def _backup_pool_keys( self, snapshot: BufferBackupSnapshot ) -> dict[PoolName, list[str]]: """Admission view of the keys a backup writes per pool: the KV chain, each aux component's BACKUP_HOST transfers, component overrides on - top, every sidecar under its source's keys.""" + top, the SWA tail under tail-only backups, every sidecar under its + source's keys.""" hash_values = snapshot.hash_values tree_core = self._cache.tree_core pool_keys: dict[PoolName, list[str]] = {PoolName.KV: list(hash_values)} @@ -560,6 +589,10 @@ class BufferModePipeline: pool_keys[pool] = list(keys) else: pool_keys.pop(pool, None) + for pool, keys in pool_keys.items(): + tail = self._swa_tail_pages(pool) + if tail and len(keys) > tail: + pool_keys[pool] = keys[-tail:] for spec in self._cache.sidecar_pool_specs: source = pool_keys.get(spec.indices_from_pool) if source: @@ -1001,6 +1034,7 @@ class BufferModePipeline: t_keys = self._transfer_keys(transfer, hash_values) if not t_keys: continue + t_keys = self._trim_swa_backup_transfer(transfer, t_keys) if transfer.keys is None: transfer.keys = t_keys transfer.hit_policy = PoolHitPolicy.TRAILING_PAGES @@ -1109,11 +1143,20 @@ class BufferModePipeline: hash_value = ( list(entry.keys_by_pool[PoolName.KV]) if intent.pool == PoolName.KV else [] ) + prefix_keys = snapshot.prefix_keys + if intent.pool in (PoolName.SWA, PoolName.AUXILIARY_SWA): + keys = entry.keys_by_pool[intent.pool] + skipped_pages = len(snapshot.hash_values) - len(keys) + if skipped_pages > 0: + # A pool-only suffix starts after the omitted node pages. + prefix_keys = self._cache.cache_controller.extend_storage_prefix( + prefix_keys, snapshot.hash_values[:skipped_pages] + ) operation_id = self._cache.cache_controller.write_storage( entry.host_indices, snapshot.key.token_ids, hash_value, - snapshot.prefix_keys, + prefix_keys, extra_pools=storage_xfers or None, ) self.ongoing_backup[operation_id] = entry diff --git a/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py b/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py index f34bdbca02..4d57f31a68 100644 --- a/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py +++ b/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py @@ -434,6 +434,9 @@ class TestBufferModeSidecar(unittest.TestCase): controller.write.side_effect = _write controller.write_storage.side_effect = [98, 99] + controller.extend_storage_prefix = ( + HybridCacheController.extend_storage_prefix.__get__(controller) + ) cache = MagicMock() cache.cache_controller = controller @@ -510,7 +513,7 @@ class TestBufferModeSidecar(unittest.TestCase): # No KV page in the SWA write; the window's keys ride the transfer. self.assertEqual(swa_call.args[0].numel(), 0) self.assertEqual(swa_call.args[2], []) - self.assertEqual(swa_call.args[3], ["page-p"]) + self.assertEqual(swa_call.args[3], ["page-p", hashes[0]]) storage_transfers = { transfer.name: transfer for call in (kv_call, swa_call) diff --git a/test/registered/unit/mem_cache/test_buffer_swa_tail_only.py b/test/registered/unit/mem_cache/test_buffer_swa_tail_only.py new file mode 100644 index 0000000000..517b5354cf --- /dev/null +++ b/test/registered/unit/mem_cache/test_buffer_swa_tail_only.py @@ -0,0 +1,156 @@ +"""Opt-in tail-only SWA backups: admission keys and staged rows agree.""" + +import unittest +from array import array +from types import SimpleNamespace +from unittest.mock import MagicMock + +import torch + +from sglang.srt.environ import envs +from sglang.srt.mem_cache.buffer_mode.pipeline import ( + BufferModePipeline, + _UnifiedBackupIntent, +) +from sglang.srt.mem_cache.buffer_mode.storage_existence_cache import ( + StorageExistenceCache, +) +from sglang.srt.mem_cache.hicache_storage import ( + PoolHitPolicy, + PoolName, + PoolTransfer, + SidecarPoolSpec, +) +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.unified_cache.components.base import ComponentType +from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( + BufferBackupSnapshot, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + +SIDECAR = PoolName.DEEPSEEK_V4_C4_INDEXER_STATE + + +def _pipeline(*, tail_only: bool, window: int = 2, swa_page_size: int = 2): + cache = MagicMock() + cache.page_size = 2 + cache.enable_storage = True + cache.cache_controller.mem_pool_host.entry_map = { + pool: SimpleNamespace(host_pool=SimpleNamespace(page_size=swa_page_size)) + for pool in (PoolName.SWA, PoolName.AUXILIARY_SWA) + } + cache.token_to_kv_pool_allocator.get_kvcache.return_value = SimpleNamespace( + size=128 + ) + cache.sidecar_pool_specs = [ + SidecarPoolSpec( + pool_name=SIDECAR, + indices_from_pool=PoolName.SWA, + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + ] + cache.storage_existence_cache = StorageExistenceCache() + with envs.SGLANG_ENABLE_HICACHE_SWA_TAIL_ONLY_BACKUP.override(tail_only): + pipeline = BufferModePipeline( + cache, + swa_component=SimpleNamespace( + pool_name=PoolName.SWA, full_window_pages=window + ), + auxiliary_swa_component=SimpleNamespace( + pool_name=PoolName.AUXILIARY_SWA, full_window_pages=window + 1 + ), + write_backlog_cap=128, + ) + return pipeline, cache + + +def _snapshot(hashes): + return BufferBackupSnapshot( + node_id=7, + parent_node_id=3, + parent_is_root=False, + parent_last_hash="page-p", + hash_values=hashes, + key=RadixKey(array("q", list(range(2 * len(hashes))))), + prefix_keys=None, + ) + + +class TestSWATailOnlyBackup(unittest.TestCase): + def _write(self, pipeline, cache, hashes, rows, pool=PoolName.SWA): + """Admit a node whose SWA pages are ``hashes`` and launch its SWA + intent off ``rows``; return the admitted SWA keys and the transfers.""" + snapshot = _snapshot(hashes) + cache.sidecar_pool_specs = [ + SidecarPoolSpec( + pool_name=SIDECAR, + indices_from_pool=pool, + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + ] + cache.tree_core.buffer_backup_pool_keys.return_value = { + ComponentType.SWA: {pool: hashes} + } + intents = dict(pipeline._write_intents(snapshot)) + self.assertEqual(intents[PoolName.KV], hashes) + sidecar = PoolTransfer(name=SIDECAR, indices_from_pool=pool) + cache._build_backup_sidecar.return_value = [sidecar] + swa = PoolTransfer(name=pool, device_indices=rows) + pipeline._build_backup_transfers = lambda node_id, *, kv_only=False: ( + torch.arange(6), + {ComponentType.SWA: [swa]}, + None, + ) + intent = _UnifiedBackupIntent(snapshot=snapshot, pool=pool, keys=intents[pool]) + device_value, keys_by_pool, transfers = pipeline._launch_transfers(intent) + self.assertEqual(device_value.numel(), 0) + self.assertEqual(transfers, [swa, sidecar]) + self.assertEqual(keys_by_pool, {pool: swa.keys, SIDECAR: sidecar.keys}) + self.assertEqual(swa.hit_policy, PoolHitPolicy.TRAILING_PAGES) + return intents[pool], swa, sidecar + + def test_admission_and_launch_keep_the_node_tail(self): + for pool, window in ((PoolName.SWA, 2), (PoolName.AUXILIARY_SWA, 3)): + with self.subTest(pool=pool): + pipeline, cache = _pipeline(tail_only=True) + hashes = [f"page-{i}" for i in range(6)] + admitted, swa, sidecar = self._write( + pipeline, cache, hashes, torch.arange(100, 112), pool + ) + tail = hashes[-window:] + self.assertEqual((admitted, swa.keys, sidecar.keys), (tail, tail, tail)) + self.assertTrue( + torch.equal(swa.device_indices, torch.arange(112 - 2 * window, 112)) + ) + + def test_short_node_and_disabled_keep_every_page(self): + cases = ( + (True, ["page-0"]), + (True, ["page-0", "page-1"]), + (False, [f"page-{i}" for i in range(6)]), + ) + for tail_only, hashes in cases: + with self.subTest(tail_only=tail_only, pages=len(hashes)): + pipeline, cache = _pipeline(tail_only=tail_only) + rows = torch.arange(100, 100 + 2 * len(hashes)) + admitted, swa, sidecar = self._write(pipeline, cache, hashes, rows) + self.assertEqual((admitted, swa.keys, sidecar.keys), (hashes,) * 3) + self.assertTrue(torch.equal(swa.device_indices, rows)) + + def test_default_off(self): + self.assertFalse(envs.SGLANG_ENABLE_HICACHE_SWA_TAIL_ONLY_BACKUP.get()) + # Pipelines built around __init__ (test fixtures) keep every page. + self.assertEqual( + BufferModePipeline.__new__(BufferModePipeline)._swa_tail_pages( + PoolName.SWA + ), + 0, + ) + pipeline, _ = _pipeline(tail_only=True, window=0) + self.assertEqual(pipeline._swa_tail_pages(PoolName.SWA), 0) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 187aea8bfa..170744995a 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -2,6 +2,7 @@ import atexit import json +import os import shutil import sys import tempfile @@ -4352,6 +4353,160 @@ class UnifiedRadixCacheSuite: num_pages = max(num_pages, sw_pages + 1) return self._make_seq(1, num_pages) + def test_buffer_only_swa_tail_only_backups(self): + """Tail-only backups: SWA reaches storage as the window ending at + each node, a hit falls back to the last node end whose window is + stored, and a split makes its point restorable once the parent is + backed up again.""" + self._skip_unsupported_hicache_test() + if not self.cfg.has_swa: + self.skipTest("requires an SWA component") + page = self.cfg.page_size + sw_pages = (self.cfg.sliding_window_size + page - 1) // page + # Nodes longer than two windows (the SWA component may split a leaf + # at the window boundary), shorter than the window, and one window. + node_pages = [2 * sw_pages + 3, max(sw_pages - 1, 1), sw_pages] + total_pages = sum(node_pages) + # Each growing insert allocates its whole sequence next to the + # resident prefix before the duplicate is released. + if 2 * total_pages * page > self.cfg.kv_size: + self.skipTest("chain does not fit the device pool") + storage_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, storage_dir, ignore_errors=True) + + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + with envs.SGLANG_ENABLE_HICACHE_SWA_TAIL_ONLY_BACKUP.override(True): + self._init_buffer_hicache(cache, storage_dir) + seq = self._make_seq(1, total_pages) + end_pages = 0 + for pages in node_pages: + end_pages += pages + self._insert(cache, allocator, req_to_token_pool, seq[: end_pages * page]) + + def chain_pages(key): + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", key))) + ).last_device_node + hashes: list[str] = [] + ends: list[int] = [] + for node in self._path_chain(cache, leaf): + hashes.extend(cache.tree_core.get_hash_values(node)) + ends.append(len(hashes)) + return leaf, hashes, ends + + def window_pages(ends): + kept = set() + for end in ends: + kept.update(range(max(0, end - sw_pages), end)) + return kept + + def restorable(prefix, kept): + """Longest prefix of ``prefix`` pages whose window is stored.""" + return next( + ( + length + for length in range(prefix, 0, -1) + if all(p in kept for p in range(max(0, length - sw_pages), length)) + ), + 0, + ) + + leaf, page_hashes, node_ends = chain_pages(seq) + self.assertEqual(len(page_hashes), total_pages) + controller = cache.cache_controller + with mock.patch.object( + controller, "write_storage", wraps=controller.write_storage + ) as write_storage: + self._buffer_backup_and_wait(cache, leaf) + + kept = window_pages(node_ends) + self.assertLess(len(kept), total_pages) + backend = controller.storage_backend + swa_page_size = controller.mem_pool_host.entry_map[ + PoolName.SWA + ].host_pool.page_size + + def stored(pool, key): + return os.path.exists( + os.path.join( + backend.file_path, backend._get_component_key(key, pool) + ".bin" + ) + ) + + swa_beliefs = cache.storage_existence_cache.pool(PoolName.SWA) + for i, key in enumerate(page_hashes): + self.assertTrue(stored(PoolName.KV, key), f"KV page {i}") + self.assertEqual(stored(PoolName.SWA, key), i in kept, f"SWA page {i}") + self.assertEqual(swa_beliefs.contains(key), i in kept, f"SWA belief {i}") + # Each SWA write staged exactly the rows of the keys it stored. + written: list[str] = [] + for call in write_storage.call_args_list: + for transfer in call.kwargs.get("extra_pools") or (): + if transfer.name == PoolName.SWA: + self.assertEqual( + transfer.host_indices.numel(), + len(transfer.keys) * swa_page_size, + ) + written.extend(transfer.keys) + self.assertEqual(sorted(written), sorted(page_hashes[i] for i in kept)) + + # A hit's usable prefix is the last node end whose window is stored. + swa_query = [ + PoolTransfer( + name=PoolName.SWA, + keys=["_"] * sw_pages, + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + ] + for prefix in range(1, total_pages + 1): + self.assertEqual( + self._storage_exists_count(cache, page_hashes[:prefix], swa_query), + restorable(prefix, kept), + f"prefix {prefix}", + ) + + # The whole chain is still served end to end from the sparse store. + cons, _, _ = build_fixture(self.cfg) + self._init_buffer_hicache(cons, storage_dir) + handle = CacheRequestHandle("tail-only-req", 0) + cons.prefetch_from_storage( + handle, cons.root_node_handle(), array("q", seq), None, None + ) + self._run_prefetch_to_completion(cons, handle) + self.assertEqual( + cons.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).device_prefix_len, + len(seq), + ) + + # A partial match inside the first node splits it; the parent's next + # backup stores the window ending at the split, so a hit there is + # served exactly from storage. + split = max( + ( + p + for p in range(1, node_ends[0]) + if p not in node_ends and restorable(p, kept) < p + ), + default=None, + ) + if split is None: + self.skipTest("the first node has no unrestorable interior page") + branch = seq[: split * page] + self._make_seq(10_000, 1) + self._insert(cache, allocator, req_to_token_pool, branch) + branch_leaf, branch_hashes, branch_ends = chain_pages(branch) + self.assertEqual(branch_hashes[:split], page_hashes[:split]) + self.assertIn(split, branch_ends) + self._buffer_backup_and_wait(cache, branch_leaf) + for i in range(max(0, split - sw_pages), split): + self.assertTrue(stored(PoolName.SWA, page_hashes[i]), f"split page {i}") + self.assertEqual( + self._storage_exists_count(cache, page_hashes[:split], swa_query), split + ) + cache.sanity_check() + cons.sanity_check() + def test_buffer_only_write_path_roundtrip(self): """Write path end to end: admission -> D2H staging -> storage write -> free. Staging and locks fully released, pages stored under every