From 4a651146dec6ebbfe2ff1f0f200147b4246e5776 Mon Sep 17 00:00:00 2001 From: lpdink <51897400+lpdink@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:09:13 +0800 Subject: [PATCH] [Feat] hicache: Notify storage backends when the initial host-pool registration burst completes (#44030) Co-authored-by: Zhangheng --- .../docs/advanced_features/hicache_design.mdx | 2 + .../sglang/srt/managers/cache_controller.py | 6 + .../sglang/srt/mem_cache/hicache_storage.py | 10 + .../hybrid_cache/hybrid_cache_controller.py | 5 + .../tensorcast_store/test_tensorcast_store.py | 3 + .../test_hicache_storage_registration_hook.py | 227 ++++++++++++++++++ .../simulation/sglang/hicache_storage.py | 4 + 7 files changed, 257 insertions(+) create mode 100644 test/registered/unit/mem_cache/test_hicache_storage_registration_hook.py diff --git a/docs/docs/advanced_features/hicache_design.mdx b/docs/docs/advanced_features/hicache_design.mdx index 9654c33b81..6dc3ddb65c 100644 --- a/docs/docs/advanced_features/hicache_design.mdx +++ b/docs/docs/advanced_features/hicache_design.mdx @@ -162,6 +162,8 @@ SGLang supports a PD (Prefill-Decode) disaggregation deployment mode through the HiCache encapsulates all read, write, and query operations on L3 backends within the `class HiCacheStorage(ABC)`, exposing a set of simple and consistent interfaces. This design supports a wide range of L3 storage backends and allows users to select the one that best fits their specific use cases. +When a backend is attached, HiCache hands it the host memory pools in a fixed order: `register_mem_pool_host` receives the anchor pool first, then every pool of the host pool group (Mamba state, indexer, SWA, draft sidecars, ...) arrives through `register_mem_host_pool_v2`, and `on_host_pools_registered` fires exactly once when this initial burst is over. Pools attached dynamically later still arrive through `register_mem_host_pool_v2` alone. Backends that talk to an external service whose instance specification must be complete up front (e.g. a remote cache manager) should finalize it inside `on_host_pools_registered`; raising there fails the attach. + - **Mooncake**: Mooncake is a high-performance caching system for LLM inference that leverages RDMA and multi-NIC resources to enable zero-copy, ultra-fast data transfers. Try Mooncake [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/mooncake_store). - **DeepSeek 3FS (HF3FS)**: HF3FS is a Kubernetes-native distributed storage solution with operator-based deployment. Try HF3FS [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/hf3fs). diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 9abb9d7237..29cc484ac1 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -530,6 +530,10 @@ class HiCacheController: ) raise RuntimeError("Failed to stop HiCache storage threads cleanly.") + # A subclass whose attach continues registering host pools after super() + # returns fires on_host_pools_registered() itself, once. + _registers_host_pools_after_super_attach = False + def attach_storage_backend( self, storage_backend: str, @@ -578,6 +582,8 @@ class HiCacheController: storage_backend, self.storage_config, self.storage_host_pool ) self.storage_backend.register_mem_pool_host(self.storage_host_pool) + if not self._registers_host_pools_after_super_attach: + self.storage_backend.on_host_pools_registered() self.enable_storage = True # todo: threshold policy for prefetching diff --git a/python/sglang/srt/mem_cache/hicache_storage.py b/python/sglang/srt/mem_cache/hicache_storage.py index f93600cfe1..2ed6d86972 100644 --- a/python/sglang/srt/mem_cache/hicache_storage.py +++ b/python/sglang/srt/mem_cache/hicache_storage.py @@ -188,6 +188,16 @@ class HiCacheStorage(ABC): self.registered_pools = {} self.registered_pools[host_pool_name] = host_pool + def on_host_pools_registered(self) -> None: + """The initial host-pool registration burst is over: the anchor pool + (``register_mem_pool_host``) and every host pool group entry + (``register_mem_host_pool_v2``). Fires once per attach; pools + registered later (dynamic sidecars) arrive through v2 alone and do + not re-fire it. Backends that must finalize once the pool set is + known -- e.g. register with a remote cache manager whose spec cannot + be amended -- should do so here; raising fails the attach. + """ + def batch_exists_v2( self, keys: List[str], diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index a0e95be806..67bad2d4f7 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -297,6 +297,10 @@ class HybridCacheController(BaseHiCacheController): """Only opt in backends whose FULL object is the complete shared page.""" return storage_backend is None or storage_backend == "mori" + # The per-pool v2 burst runs after super() returns; on_host_pools_registered() + # fires at its end, so the base class leaves the firing to us. + _registers_host_pools_after_super_attach = True + def attach_storage_backend( self, storage_backend: str, @@ -343,6 +347,7 @@ class HybridCacheController(BaseHiCacheController): for entry in host_pools or []: self.storage_backend.register_mem_host_pool_v2(entry.host_pool, entry.name) + self.storage_backend.on_host_pools_registered() def detach_storage_backend(self): super().detach_storage_backend() diff --git a/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py b/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py index 5ebd327807..8c7bdf8213 100644 --- a/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py +++ b/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py @@ -1122,6 +1122,9 @@ def test_controller_tensorcast_factory_and_host_pool_group_use_v1_anchor( def register_mem_pool_host(self, pool: HostKVCache) -> None: self.registered.append(pool) + def on_host_pools_registered(self) -> None: + pass + backend = FakeStorageBackend() def create_backend( diff --git a/test/registered/unit/mem_cache/test_hicache_storage_registration_hook.py b/test/registered/unit/mem_cache/test_hicache_storage_registration_hook.py new file mode 100644 index 0000000000..ac4f282f48 --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_storage_registration_hook.py @@ -0,0 +1,227 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The storage backend's view of the initial host-pool registration burst. + +``on_host_pools_registered`` fires exactly once per attach, immediately after +the last call of the initial burst: the anchor pool on the base controller +(its whole burst), after the per-pool v2 loop on the hybrid controller (which +continues the burst after super() returns). Pools registered later (dynamic +sidecars) arrive through v2 alone and do not re-fire the hook. + + python3 -m pytest test/registered/unit/mem_cache/test_hicache_storage_registration_hook.py -v +""" + +import threading +import unittest +from types import SimpleNamespace +from typing import Any, List, Optional +from unittest import mock + +from sglang.srt.managers.cache_controller import HiCacheController +from sglang.srt.mem_cache.hicache_storage import PoolName +from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( + HybridCacheController, +) +from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry +from sglang.srt.mem_cache.storage import StorageBackendFactory +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +PAGE = 16 + + +class RecordingBackend: + """A backend that records the registration calls it receives.""" + + def __init__(self, order: List[str], fail_on_complete: bool = False): + self.order = order + self.fail_on_complete = fail_on_complete + self.closed = False + + def register_mem_pool_host(self, mem_pool_host: Any) -> None: + self.order.append("v1") + + def register_mem_host_pool_v2(self, host_pool: Any, host_pool_name: Any) -> None: + self.order.append(f"v2:{host_pool_name}") + + def on_host_pools_registered(self) -> None: + self.order.append("complete") + if self.fail_on_complete: + raise RuntimeError("registration failed in the backend") + + def close(self) -> None: + self.closed = True + + +def _fake_pool(name: str) -> SimpleNamespace: + """The HostPoolGroup surface, without CUDA.""" + return SimpleNamespace( + name=name, + layout="page_first", + page_size=PAGE, + device="cpu", + size=128, + logical_size=128, + can_use_write_back_jit=True, + ) + + +def _entry(name: PoolName, pool: Any, is_anchor: bool = False) -> PoolEntry: + return PoolEntry( + name=name, + host_pool=pool, + device_pool=SimpleNamespace(), + layer_mapper=lambda layer: layer, + is_primary_index_anchor=is_anchor, + ) + + +def _group() -> HostPoolGroup: + return HostPoolGroup( + [ + _entry(PoolName.KV, _fake_pool("kv"), is_anchor=True), + _entry(PoolName.MAMBA, _fake_pool("mamba")), + ] + ) + + +def _controller(cls: type, storage_host_pool: Any, mem_pool_host: Any) -> Any: + """Only the attributes ``attach_storage_backend`` reads.""" + controller = object.__new__(cls) + controller.enable_storage = False + controller.storage_stop_event = threading.Event() + controller.storage_host_pool = storage_host_pool + controller.mem_pool_host = mem_pool_host + controller.page_size = PAGE + controller.host_memory_mode = "cache" + controller.tp_group = None + controller.attn_cp_group = None + controller.attn_tp_group = None + controller.pp_group = None + controller.prefetch_hits_sync_groups = [] + controller.prefetch_completion_sync_groups = [] + if cls is HybridCacheController: + controller.pp_prefetch_command_thread = None + controller.pp_prefetch_command_group = None + controller.extra_host_mem_release_queues = {} + return controller + + +def _attach( + controller: Any, + backend: RecordingBackend, + order: List[str], + host_pools: Optional[List[PoolEntry]] = None, +) -> None: + """Run ``attach_storage_backend`` with the heavy pieces stubbed out.""" + storage_config = SimpleNamespace(is_mla_model=False, extra_config={}) + # Only the hybrid attach accepts host_pools; the base attach's burst is + # the anchor pool alone. + kwargs = {"host_pools": host_pools} if host_pools is not None else {} + with ( + mock.patch.object( + StorageBackendFactory, "create_backend", return_value=backend + ), + mock.patch.object( + controller, "_generate_storage_config", return_value=storage_config + ), + mock.patch.object( + controller, + "_start_storage_threads", + side_effect=lambda: order.append("threads"), + ), + ): + controller.attach_storage_backend(storage_backend="dynamic", **kwargs) + + +class TestRegistrationBurstOrder(CustomTestCase): + def test_hybrid_hook_fires_after_its_v2_burst(self): + # super() completes (threads included, as before), then the hybrid + # burst continues and the hook fires at its end -- once. + order: List[str] = [] + group = _group() + controller = _controller( + HybridCacheController, group.anchor_entry.host_pool, group + ) + + _attach(controller, RecordingBackend(order), order, host_pools=group.entries) + + self.assertEqual(order, ["v1", "threads", "v2:kv", "v2:mamba", "complete"]) + + def test_base_hook_fires_right_after_the_anchor_pool(self): + # Without a host pool group the burst is the anchor pool alone, so + # the hook fires inside the base attach. + order: List[str] = [] + pool = _fake_pool("kv") + controller = _controller(HiCacheController, pool, pool) + + _attach(controller, RecordingBackend(order), order) + + self.assertEqual(order, ["v1", "complete", "threads"]) + + def test_base_hook_failure_rolls_the_attach_back(self): + # On the base path the hook runs inside the attach's guarded section, + # so a backend failure rolls the backend back through the existing + # error path. + order: List[str] = [] + pool = _fake_pool("kv") + controller = _controller(HiCacheController, pool, pool) + backend = RecordingBackend(order, fail_on_complete=True) + + with self.assertRaises(RuntimeError): + _attach(controller, backend, order) + + self.assertEqual(order, ["v1", "complete"]) + self.assertTrue(backend.closed) + self.assertFalse(controller.enable_storage) + self.assertIsNone(controller.storage_backend) + + def test_hybrid_hook_failure_propagates(self): + # On the hybrid path the burst continues after super() has completed; + # a backend failure there propagates out of attach_storage_backend, + # failing the attach (the post-super section has no rollback of its + # own, exactly as for the v2 calls themselves). + order: List[str] = [] + group = _group() + controller = _controller( + HybridCacheController, group.anchor_entry.host_pool, group + ) + backend = RecordingBackend(order, fail_on_complete=True) + + with self.assertRaises(RuntimeError): + _attach(controller, backend, order, host_pools=group.entries) + + self.assertEqual(order, ["v1", "threads", "v2:kv", "v2:mamba", "complete"]) + + def test_dynamic_sidecar_does_not_re_fire_the_hook(self): + order: List[str] = [] + group = _group() + controller = _controller( + HybridCacheController, group.anchor_entry.host_pool, group + ) + backend = RecordingBackend(order) + + _attach(controller, backend, order, host_pools=group.entries) + controller.register_host_pool_entry(_entry(PoolName.SWA, _fake_pool("swa"))) + + self.assertEqual( + order, + ["v1", "threads", "v2:kv", "v2:mamba", "complete", "v2:swa"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hicache_storage.py b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hicache_storage.py index dcb2e89214..c361ba206c 100644 --- a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hicache_storage.py +++ b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hicache_storage.py @@ -45,6 +45,10 @@ class MockHiCacheStorage: """Register one pool from UnifiedRadixCache's multi-pool HiCache stack.""" self.registered_pools[host_pool_name] = host_pool + def on_host_pools_registered(self): + """The initial registration burst is over (see HiCacheStorage).""" + pass + @staticmethod def _pool_storage_key(key: str, pool_name) -> str: name = str(pool_name)