[Feat] hicache: Notify storage backends when the initial host-pool registration burst completes (#44030)

Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
lpdink
2026-10-20 14:09:13 +08:00
committed by GitHub
co-authored by Zhangheng
parent 36978d6c4d
commit 4a651146de
7 changed files with 257 additions and 0 deletions
@@ -162,6 +162,8 @@ SGLang supports a PD (Prefill-Decode) disaggregation deployment mode through the
HiCache encapsulates all read, write, and query operations on L3 backends within the `class HiCacheStorage(ABC)`, exposing a set of simple and consistent interfaces. This design supports a wide range of L3 storage backends and allows users to select the one that best fits their specific use cases. HiCache encapsulates all read, write, and query operations on L3 backends within the `class HiCacheStorage(ABC)`, exposing a set of simple and consistent interfaces. This design supports a wide range of L3 storage backends and allows users to select the one that best fits their specific use cases.
When a backend is attached, HiCache hands it the host memory pools in a fixed order: `register_mem_pool_host` receives the anchor pool first, then every pool of the host pool group (Mamba state, indexer, SWA, draft sidecars, ...) arrives through `register_mem_host_pool_v2`, and `on_host_pools_registered` fires exactly once when this initial burst is over. Pools attached dynamically later still arrive through `register_mem_host_pool_v2` alone. Backends that talk to an external service whose instance specification must be complete up front (e.g. a remote cache manager) should finalize it inside `on_host_pools_registered`; raising there fails the attach.
- **Mooncake**: Mooncake is a high-performance caching system for LLM inference that leverages RDMA and multi-NIC resources to enable zero-copy, ultra-fast data transfers. Try Mooncake [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/mooncake_store). - **Mooncake**: Mooncake is a high-performance caching system for LLM inference that leverages RDMA and multi-NIC resources to enable zero-copy, ultra-fast data transfers. Try Mooncake [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/mooncake_store).
- **DeepSeek 3FS (HF3FS)**: HF3FS is a Kubernetes-native distributed storage solution with operator-based deployment. Try HF3FS [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/hf3fs). - **DeepSeek 3FS (HF3FS)**: HF3FS is a Kubernetes-native distributed storage solution with operator-based deployment. Try HF3FS [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/hf3fs).
@@ -530,6 +530,10 @@ class HiCacheController:
) )
raise RuntimeError("Failed to stop HiCache storage threads cleanly.") raise RuntimeError("Failed to stop HiCache storage threads cleanly.")
# A subclass whose attach continues registering host pools after super()
# returns fires on_host_pools_registered() itself, once.
_registers_host_pools_after_super_attach = False
def attach_storage_backend( def attach_storage_backend(
self, self,
storage_backend: str, storage_backend: str,
@@ -578,6 +582,8 @@ class HiCacheController:
storage_backend, self.storage_config, self.storage_host_pool storage_backend, self.storage_config, self.storage_host_pool
) )
self.storage_backend.register_mem_pool_host(self.storage_host_pool) self.storage_backend.register_mem_pool_host(self.storage_host_pool)
if not self._registers_host_pools_after_super_attach:
self.storage_backend.on_host_pools_registered()
self.enable_storage = True self.enable_storage = True
# todo: threshold policy for prefetching # todo: threshold policy for prefetching
@@ -188,6 +188,16 @@ class HiCacheStorage(ABC):
self.registered_pools = {} self.registered_pools = {}
self.registered_pools[host_pool_name] = host_pool self.registered_pools[host_pool_name] = host_pool
def on_host_pools_registered(self) -> None:
"""The initial host-pool registration burst is over: the anchor pool
(``register_mem_pool_host``) and every host pool group entry
(``register_mem_host_pool_v2``). Fires once per attach; pools
registered later (dynamic sidecars) arrive through v2 alone and do
not re-fire it. Backends that must finalize once the pool set is
known -- e.g. register with a remote cache manager whose spec cannot
be amended -- should do so here; raising fails the attach.
"""
def batch_exists_v2( def batch_exists_v2(
self, self,
keys: List[str], keys: List[str],
@@ -297,6 +297,10 @@ class HybridCacheController(BaseHiCacheController):
"""Only opt in backends whose FULL object is the complete shared page.""" """Only opt in backends whose FULL object is the complete shared page."""
return storage_backend is None or storage_backend == "mori" return storage_backend is None or storage_backend == "mori"
# The per-pool v2 burst runs after super() returns; on_host_pools_registered()
# fires at its end, so the base class leaves the firing to us.
_registers_host_pools_after_super_attach = True
def attach_storage_backend( def attach_storage_backend(
self, self,
storage_backend: str, storage_backend: str,
@@ -343,6 +347,7 @@ class HybridCacheController(BaseHiCacheController):
for entry in host_pools or []: for entry in host_pools or []:
self.storage_backend.register_mem_host_pool_v2(entry.host_pool, entry.name) self.storage_backend.register_mem_host_pool_v2(entry.host_pool, entry.name)
self.storage_backend.on_host_pools_registered()
def detach_storage_backend(self): def detach_storage_backend(self):
super().detach_storage_backend() super().detach_storage_backend()
@@ -1122,6 +1122,9 @@ def test_controller_tensorcast_factory_and_host_pool_group_use_v1_anchor(
def register_mem_pool_host(self, pool: HostKVCache) -> None: def register_mem_pool_host(self, pool: HostKVCache) -> None:
self.registered.append(pool) self.registered.append(pool)
def on_host_pools_registered(self) -> None:
pass
backend = FakeStorageBackend() backend = FakeStorageBackend()
def create_backend( def create_backend(
@@ -0,0 +1,227 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The storage backend's view of the initial host-pool registration burst.
``on_host_pools_registered`` fires exactly once per attach, immediately after
the last call of the initial burst: the anchor pool on the base controller
(its whole burst), after the per-pool v2 loop on the hybrid controller (which
continues the burst after super() returns). Pools registered later (dynamic
sidecars) arrive through v2 alone and do not re-fire the hook.
python3 -m pytest test/registered/unit/mem_cache/test_hicache_storage_registration_hook.py -v
"""
import threading
import unittest
from types import SimpleNamespace
from typing import Any, List, Optional
from unittest import mock
from sglang.srt.managers.cache_controller import HiCacheController
from sglang.srt.mem_cache.hicache_storage import PoolName
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
)
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
from sglang.srt.mem_cache.storage import StorageBackendFactory
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
PAGE = 16
class RecordingBackend:
"""A backend that records the registration calls it receives."""
def __init__(self, order: List[str], fail_on_complete: bool = False):
self.order = order
self.fail_on_complete = fail_on_complete
self.closed = False
def register_mem_pool_host(self, mem_pool_host: Any) -> None:
self.order.append("v1")
def register_mem_host_pool_v2(self, host_pool: Any, host_pool_name: Any) -> None:
self.order.append(f"v2:{host_pool_name}")
def on_host_pools_registered(self) -> None:
self.order.append("complete")
if self.fail_on_complete:
raise RuntimeError("registration failed in the backend")
def close(self) -> None:
self.closed = True
def _fake_pool(name: str) -> SimpleNamespace:
"""The HostPoolGroup surface, without CUDA."""
return SimpleNamespace(
name=name,
layout="page_first",
page_size=PAGE,
device="cpu",
size=128,
logical_size=128,
can_use_write_back_jit=True,
)
def _entry(name: PoolName, pool: Any, is_anchor: bool = False) -> PoolEntry:
return PoolEntry(
name=name,
host_pool=pool,
device_pool=SimpleNamespace(),
layer_mapper=lambda layer: layer,
is_primary_index_anchor=is_anchor,
)
def _group() -> HostPoolGroup:
return HostPoolGroup(
[
_entry(PoolName.KV, _fake_pool("kv"), is_anchor=True),
_entry(PoolName.MAMBA, _fake_pool("mamba")),
]
)
def _controller(cls: type, storage_host_pool: Any, mem_pool_host: Any) -> Any:
"""Only the attributes ``attach_storage_backend`` reads."""
controller = object.__new__(cls)
controller.enable_storage = False
controller.storage_stop_event = threading.Event()
controller.storage_host_pool = storage_host_pool
controller.mem_pool_host = mem_pool_host
controller.page_size = PAGE
controller.host_memory_mode = "cache"
controller.tp_group = None
controller.attn_cp_group = None
controller.attn_tp_group = None
controller.pp_group = None
controller.prefetch_hits_sync_groups = []
controller.prefetch_completion_sync_groups = []
if cls is HybridCacheController:
controller.pp_prefetch_command_thread = None
controller.pp_prefetch_command_group = None
controller.extra_host_mem_release_queues = {}
return controller
def _attach(
controller: Any,
backend: RecordingBackend,
order: List[str],
host_pools: Optional[List[PoolEntry]] = None,
) -> None:
"""Run ``attach_storage_backend`` with the heavy pieces stubbed out."""
storage_config = SimpleNamespace(is_mla_model=False, extra_config={})
# Only the hybrid attach accepts host_pools; the base attach's burst is
# the anchor pool alone.
kwargs = {"host_pools": host_pools} if host_pools is not None else {}
with (
mock.patch.object(
StorageBackendFactory, "create_backend", return_value=backend
),
mock.patch.object(
controller, "_generate_storage_config", return_value=storage_config
),
mock.patch.object(
controller,
"_start_storage_threads",
side_effect=lambda: order.append("threads"),
),
):
controller.attach_storage_backend(storage_backend="dynamic", **kwargs)
class TestRegistrationBurstOrder(CustomTestCase):
def test_hybrid_hook_fires_after_its_v2_burst(self):
# super() completes (threads included, as before), then the hybrid
# burst continues and the hook fires at its end -- once.
order: List[str] = []
group = _group()
controller = _controller(
HybridCacheController, group.anchor_entry.host_pool, group
)
_attach(controller, RecordingBackend(order), order, host_pools=group.entries)
self.assertEqual(order, ["v1", "threads", "v2:kv", "v2:mamba", "complete"])
def test_base_hook_fires_right_after_the_anchor_pool(self):
# Without a host pool group the burst is the anchor pool alone, so
# the hook fires inside the base attach.
order: List[str] = []
pool = _fake_pool("kv")
controller = _controller(HiCacheController, pool, pool)
_attach(controller, RecordingBackend(order), order)
self.assertEqual(order, ["v1", "complete", "threads"])
def test_base_hook_failure_rolls_the_attach_back(self):
# On the base path the hook runs inside the attach's guarded section,
# so a backend failure rolls the backend back through the existing
# error path.
order: List[str] = []
pool = _fake_pool("kv")
controller = _controller(HiCacheController, pool, pool)
backend = RecordingBackend(order, fail_on_complete=True)
with self.assertRaises(RuntimeError):
_attach(controller, backend, order)
self.assertEqual(order, ["v1", "complete"])
self.assertTrue(backend.closed)
self.assertFalse(controller.enable_storage)
self.assertIsNone(controller.storage_backend)
def test_hybrid_hook_failure_propagates(self):
# On the hybrid path the burst continues after super() has completed;
# a backend failure there propagates out of attach_storage_backend,
# failing the attach (the post-super section has no rollback of its
# own, exactly as for the v2 calls themselves).
order: List[str] = []
group = _group()
controller = _controller(
HybridCacheController, group.anchor_entry.host_pool, group
)
backend = RecordingBackend(order, fail_on_complete=True)
with self.assertRaises(RuntimeError):
_attach(controller, backend, order, host_pools=group.entries)
self.assertEqual(order, ["v1", "threads", "v2:kv", "v2:mamba", "complete"])
def test_dynamic_sidecar_does_not_re_fire_the_hook(self):
order: List[str] = []
group = _group()
controller = _controller(
HybridCacheController, group.anchor_entry.host_pool, group
)
backend = RecordingBackend(order)
_attach(controller, backend, order, host_pools=group.entries)
controller.register_host_pool_entry(_entry(PoolName.SWA, _fake_pool("swa")))
self.assertEqual(
order,
["v1", "threads", "v2:kv", "v2:mamba", "complete", "v2:swa"],
)
if __name__ == "__main__":
unittest.main()
@@ -45,6 +45,10 @@ class MockHiCacheStorage:
"""Register one pool from UnifiedRadixCache's multi-pool HiCache stack.""" """Register one pool from UnifiedRadixCache's multi-pool HiCache stack."""
self.registered_pools[host_pool_name] = host_pool self.registered_pools[host_pool_name] = host_pool
def on_host_pools_registered(self):
"""The initial registration burst is over (see HiCacheStorage)."""
pass
@staticmethod @staticmethod
def _pool_storage_key(key: str, pool_name) -> str: def _pool_storage_key(key: str, pool_name) -> str:
name = str(pool_name) name = str(pool_name)