[Feat] hicache: Notify storage backends when the initial host-pool registration burst completes (#44030)
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
@@ -162,6 +162,8 @@ SGLang supports a PD (Prefill-Decode) disaggregation deployment mode through the
|
||||
|
||||
HiCache encapsulates all read, write, and query operations on L3 backends within the `class HiCacheStorage(ABC)`, exposing a set of simple and consistent interfaces. This design supports a wide range of L3 storage backends and allows users to select the one that best fits their specific use cases.
|
||||
|
||||
When a backend is attached, HiCache hands it the host memory pools in a fixed order: `register_mem_pool_host` receives the anchor pool first, then every pool of the host pool group (Mamba state, indexer, SWA, draft sidecars, ...) arrives through `register_mem_host_pool_v2`, and `on_host_pools_registered` fires exactly once when this initial burst is over. Pools attached dynamically later still arrive through `register_mem_host_pool_v2` alone. Backends that talk to an external service whose instance specification must be complete up front (e.g. a remote cache manager) should finalize it inside `on_host_pools_registered`; raising there fails the attach.
|
||||
|
||||
- **Mooncake**: Mooncake is a high-performance caching system for LLM inference that leverages RDMA and multi-NIC resources to enable zero-copy, ultra-fast data transfers. Try Mooncake [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/mooncake_store).
|
||||
|
||||
- **DeepSeek 3FS (HF3FS)**: HF3FS is a Kubernetes-native distributed storage solution with operator-based deployment. Try HF3FS [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/hf3fs).
|
||||
|
||||
@@ -530,6 +530,10 @@ class HiCacheController:
|
||||
)
|
||||
raise RuntimeError("Failed to stop HiCache storage threads cleanly.")
|
||||
|
||||
# A subclass whose attach continues registering host pools after super()
|
||||
# returns fires on_host_pools_registered() itself, once.
|
||||
_registers_host_pools_after_super_attach = False
|
||||
|
||||
def attach_storage_backend(
|
||||
self,
|
||||
storage_backend: str,
|
||||
@@ -578,6 +582,8 @@ class HiCacheController:
|
||||
storage_backend, self.storage_config, self.storage_host_pool
|
||||
)
|
||||
self.storage_backend.register_mem_pool_host(self.storage_host_pool)
|
||||
if not self._registers_host_pools_after_super_attach:
|
||||
self.storage_backend.on_host_pools_registered()
|
||||
|
||||
self.enable_storage = True
|
||||
# todo: threshold policy for prefetching
|
||||
|
||||
@@ -188,6 +188,16 @@ class HiCacheStorage(ABC):
|
||||
self.registered_pools = {}
|
||||
self.registered_pools[host_pool_name] = host_pool
|
||||
|
||||
def on_host_pools_registered(self) -> None:
|
||||
"""The initial host-pool registration burst is over: the anchor pool
|
||||
(``register_mem_pool_host``) and every host pool group entry
|
||||
(``register_mem_host_pool_v2``). Fires once per attach; pools
|
||||
registered later (dynamic sidecars) arrive through v2 alone and do
|
||||
not re-fire it. Backends that must finalize once the pool set is
|
||||
known -- e.g. register with a remote cache manager whose spec cannot
|
||||
be amended -- should do so here; raising fails the attach.
|
||||
"""
|
||||
|
||||
def batch_exists_v2(
|
||||
self,
|
||||
keys: List[str],
|
||||
|
||||
@@ -297,6 +297,10 @@ class HybridCacheController(BaseHiCacheController):
|
||||
"""Only opt in backends whose FULL object is the complete shared page."""
|
||||
return storage_backend is None or storage_backend == "mori"
|
||||
|
||||
# The per-pool v2 burst runs after super() returns; on_host_pools_registered()
|
||||
# fires at its end, so the base class leaves the firing to us.
|
||||
_registers_host_pools_after_super_attach = True
|
||||
|
||||
def attach_storage_backend(
|
||||
self,
|
||||
storage_backend: str,
|
||||
@@ -343,6 +347,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
for entry in host_pools or []:
|
||||
self.storage_backend.register_mem_host_pool_v2(entry.host_pool, entry.name)
|
||||
self.storage_backend.on_host_pools_registered()
|
||||
|
||||
def detach_storage_backend(self):
|
||||
super().detach_storage_backend()
|
||||
|
||||
@@ -1122,6 +1122,9 @@ def test_controller_tensorcast_factory_and_host_pool_group_use_v1_anchor(
|
||||
def register_mem_pool_host(self, pool: HostKVCache) -> None:
|
||||
self.registered.append(pool)
|
||||
|
||||
def on_host_pools_registered(self) -> None:
|
||||
pass
|
||||
|
||||
backend = FakeStorageBackend()
|
||||
|
||||
def create_backend(
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""The storage backend's view of the initial host-pool registration burst.
|
||||
|
||||
``on_host_pools_registered`` fires exactly once per attach, immediately after
|
||||
the last call of the initial burst: the anchor pool on the base controller
|
||||
(its whole burst), after the per-pool v2 loop on the hybrid controller (which
|
||||
continues the burst after super() returns). Pools registered later (dynamic
|
||||
sidecars) arrive through v2 alone and do not re-fire the hook.
|
||||
|
||||
python3 -m pytest test/registered/unit/mem_cache/test_hicache_storage_registration_hook.py -v
|
||||
"""
|
||||
|
||||
import threading
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, List, Optional
|
||||
from unittest import mock
|
||||
|
||||
from sglang.srt.managers.cache_controller import HiCacheController
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
|
||||
from sglang.srt.mem_cache.storage import StorageBackendFactory
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
PAGE = 16
|
||||
|
||||
|
||||
class RecordingBackend:
|
||||
"""A backend that records the registration calls it receives."""
|
||||
|
||||
def __init__(self, order: List[str], fail_on_complete: bool = False):
|
||||
self.order = order
|
||||
self.fail_on_complete = fail_on_complete
|
||||
self.closed = False
|
||||
|
||||
def register_mem_pool_host(self, mem_pool_host: Any) -> None:
|
||||
self.order.append("v1")
|
||||
|
||||
def register_mem_host_pool_v2(self, host_pool: Any, host_pool_name: Any) -> None:
|
||||
self.order.append(f"v2:{host_pool_name}")
|
||||
|
||||
def on_host_pools_registered(self) -> None:
|
||||
self.order.append("complete")
|
||||
if self.fail_on_complete:
|
||||
raise RuntimeError("registration failed in the backend")
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _fake_pool(name: str) -> SimpleNamespace:
|
||||
"""The HostPoolGroup surface, without CUDA."""
|
||||
return SimpleNamespace(
|
||||
name=name,
|
||||
layout="page_first",
|
||||
page_size=PAGE,
|
||||
device="cpu",
|
||||
size=128,
|
||||
logical_size=128,
|
||||
can_use_write_back_jit=True,
|
||||
)
|
||||
|
||||
|
||||
def _entry(name: PoolName, pool: Any, is_anchor: bool = False) -> PoolEntry:
|
||||
return PoolEntry(
|
||||
name=name,
|
||||
host_pool=pool,
|
||||
device_pool=SimpleNamespace(),
|
||||
layer_mapper=lambda layer: layer,
|
||||
is_primary_index_anchor=is_anchor,
|
||||
)
|
||||
|
||||
|
||||
def _group() -> HostPoolGroup:
|
||||
return HostPoolGroup(
|
||||
[
|
||||
_entry(PoolName.KV, _fake_pool("kv"), is_anchor=True),
|
||||
_entry(PoolName.MAMBA, _fake_pool("mamba")),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _controller(cls: type, storage_host_pool: Any, mem_pool_host: Any) -> Any:
|
||||
"""Only the attributes ``attach_storage_backend`` reads."""
|
||||
controller = object.__new__(cls)
|
||||
controller.enable_storage = False
|
||||
controller.storage_stop_event = threading.Event()
|
||||
controller.storage_host_pool = storage_host_pool
|
||||
controller.mem_pool_host = mem_pool_host
|
||||
controller.page_size = PAGE
|
||||
controller.host_memory_mode = "cache"
|
||||
controller.tp_group = None
|
||||
controller.attn_cp_group = None
|
||||
controller.attn_tp_group = None
|
||||
controller.pp_group = None
|
||||
controller.prefetch_hits_sync_groups = []
|
||||
controller.prefetch_completion_sync_groups = []
|
||||
if cls is HybridCacheController:
|
||||
controller.pp_prefetch_command_thread = None
|
||||
controller.pp_prefetch_command_group = None
|
||||
controller.extra_host_mem_release_queues = {}
|
||||
return controller
|
||||
|
||||
|
||||
def _attach(
|
||||
controller: Any,
|
||||
backend: RecordingBackend,
|
||||
order: List[str],
|
||||
host_pools: Optional[List[PoolEntry]] = None,
|
||||
) -> None:
|
||||
"""Run ``attach_storage_backend`` with the heavy pieces stubbed out."""
|
||||
storage_config = SimpleNamespace(is_mla_model=False, extra_config={})
|
||||
# Only the hybrid attach accepts host_pools; the base attach's burst is
|
||||
# the anchor pool alone.
|
||||
kwargs = {"host_pools": host_pools} if host_pools is not None else {}
|
||||
with (
|
||||
mock.patch.object(
|
||||
StorageBackendFactory, "create_backend", return_value=backend
|
||||
),
|
||||
mock.patch.object(
|
||||
controller, "_generate_storage_config", return_value=storage_config
|
||||
),
|
||||
mock.patch.object(
|
||||
controller,
|
||||
"_start_storage_threads",
|
||||
side_effect=lambda: order.append("threads"),
|
||||
),
|
||||
):
|
||||
controller.attach_storage_backend(storage_backend="dynamic", **kwargs)
|
||||
|
||||
|
||||
class TestRegistrationBurstOrder(CustomTestCase):
|
||||
def test_hybrid_hook_fires_after_its_v2_burst(self):
|
||||
# super() completes (threads included, as before), then the hybrid
|
||||
# burst continues and the hook fires at its end -- once.
|
||||
order: List[str] = []
|
||||
group = _group()
|
||||
controller = _controller(
|
||||
HybridCacheController, group.anchor_entry.host_pool, group
|
||||
)
|
||||
|
||||
_attach(controller, RecordingBackend(order), order, host_pools=group.entries)
|
||||
|
||||
self.assertEqual(order, ["v1", "threads", "v2:kv", "v2:mamba", "complete"])
|
||||
|
||||
def test_base_hook_fires_right_after_the_anchor_pool(self):
|
||||
# Without a host pool group the burst is the anchor pool alone, so
|
||||
# the hook fires inside the base attach.
|
||||
order: List[str] = []
|
||||
pool = _fake_pool("kv")
|
||||
controller = _controller(HiCacheController, pool, pool)
|
||||
|
||||
_attach(controller, RecordingBackend(order), order)
|
||||
|
||||
self.assertEqual(order, ["v1", "complete", "threads"])
|
||||
|
||||
def test_base_hook_failure_rolls_the_attach_back(self):
|
||||
# On the base path the hook runs inside the attach's guarded section,
|
||||
# so a backend failure rolls the backend back through the existing
|
||||
# error path.
|
||||
order: List[str] = []
|
||||
pool = _fake_pool("kv")
|
||||
controller = _controller(HiCacheController, pool, pool)
|
||||
backend = RecordingBackend(order, fail_on_complete=True)
|
||||
|
||||
with self.assertRaises(RuntimeError):
|
||||
_attach(controller, backend, order)
|
||||
|
||||
self.assertEqual(order, ["v1", "complete"])
|
||||
self.assertTrue(backend.closed)
|
||||
self.assertFalse(controller.enable_storage)
|
||||
self.assertIsNone(controller.storage_backend)
|
||||
|
||||
def test_hybrid_hook_failure_propagates(self):
|
||||
# On the hybrid path the burst continues after super() has completed;
|
||||
# a backend failure there propagates out of attach_storage_backend,
|
||||
# failing the attach (the post-super section has no rollback of its
|
||||
# own, exactly as for the v2 calls themselves).
|
||||
order: List[str] = []
|
||||
group = _group()
|
||||
controller = _controller(
|
||||
HybridCacheController, group.anchor_entry.host_pool, group
|
||||
)
|
||||
backend = RecordingBackend(order, fail_on_complete=True)
|
||||
|
||||
with self.assertRaises(RuntimeError):
|
||||
_attach(controller, backend, order, host_pools=group.entries)
|
||||
|
||||
self.assertEqual(order, ["v1", "threads", "v2:kv", "v2:mamba", "complete"])
|
||||
|
||||
def test_dynamic_sidecar_does_not_re_fire_the_hook(self):
|
||||
order: List[str] = []
|
||||
group = _group()
|
||||
controller = _controller(
|
||||
HybridCacheController, group.anchor_entry.host_pool, group
|
||||
)
|
||||
backend = RecordingBackend(order)
|
||||
|
||||
_attach(controller, backend, order, host_pools=group.entries)
|
||||
controller.register_host_pool_entry(_entry(PoolName.SWA, _fake_pool("swa")))
|
||||
|
||||
self.assertEqual(
|
||||
order,
|
||||
["v1", "threads", "v2:kv", "v2:mamba", "complete", "v2:swa"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -45,6 +45,10 @@ class MockHiCacheStorage:
|
||||
"""Register one pool from UnifiedRadixCache's multi-pool HiCache stack."""
|
||||
self.registered_pools[host_pool_name] = host_pool
|
||||
|
||||
def on_host_pools_registered(self):
|
||||
"""The initial registration burst is over (see HiCacheStorage)."""
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _pool_storage_key(key: str, pool_name) -> str:
|
||||
name = str(pool_name)
|
||||
|
||||
Reference in New Issue
Block a user