Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com> Co-authored-by: jasonjk-park <jasonjk@fb.com>
423 lines
16 KiB
Python
423 lines
16 KiB
Python
"""Unit tests for the radix-cache registry, routing, and selection chain."""
|
|
|
|
from sglang.srt.runtime_context import get_context
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
|
|
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
|
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from sglang.srt.mem_cache.registry import (
|
|
_RADIX_CACHE_REGISTRY,
|
|
TreeCacheBuildContext,
|
|
create_tree_cache,
|
|
create_unified_radix_cache,
|
|
default_radix_cache_factory,
|
|
get_radix_cache_factory,
|
|
register_radix_cache_backend,
|
|
registered_radix_cache_backends,
|
|
)
|
|
from sglang.test.test_utils import CustomTestCase, enter_override
|
|
|
|
|
|
def _publish(testcase, **fields):
|
|
"""Install a published config for one case and restore on its cleanup."""
|
|
from sglang.srt.runtime_context import get_context, get_server_args
|
|
|
|
override = get_context().override_server_args(**fields)
|
|
override.install()
|
|
testcase.addCleanup(override.restore)
|
|
return get_server_args()
|
|
|
|
|
|
def _make_ctx(
|
|
testcase,
|
|
*,
|
|
backend=None,
|
|
enable_streaming=False,
|
|
enable_lmcache=False,
|
|
is_hybrid_swa=False,
|
|
is_hybrid_ssm=False,
|
|
is_dsa=False,
|
|
enable_hierarchical_cache=False,
|
|
disable_radix_cache=False,
|
|
effective_chunked_prefill_size=None,
|
|
full_tokens_per_layer=None,
|
|
):
|
|
# The factory reads the published bags for the cache-backend leaves, so the
|
|
# fixture publishes them; the instance stays for the whole-object contract
|
|
# `TreeCacheBuildContext` carries.
|
|
server_args = _publish(
|
|
testcase,
|
|
radix_cache_backend=backend,
|
|
enable_streaming_session=enable_streaming,
|
|
enable_lmcache=enable_lmcache,
|
|
enable_flexkv=False,
|
|
enable_unified_cache_external_linker=False,
|
|
)
|
|
return TreeCacheBuildContext(
|
|
server_args=server_args,
|
|
params=MagicMock(),
|
|
is_hybrid_swa=is_hybrid_swa,
|
|
is_hybrid_ssm=is_hybrid_ssm,
|
|
is_dsa=is_dsa,
|
|
enable_hierarchical_cache=enable_hierarchical_cache,
|
|
disable_radix_cache=disable_radix_cache,
|
|
effective_chunked_prefill_size=effective_chunked_prefill_size,
|
|
tp_worker=MagicMock(),
|
|
model_config=MagicMock(),
|
|
tp_size=1,
|
|
tp_rank=0,
|
|
tp_group=MagicMock(),
|
|
full_tokens_per_layer=full_tokens_per_layer,
|
|
)
|
|
|
|
|
|
class _RegistryIsolationMixin:
|
|
"""Restore the global registry around each test so registrations
|
|
from one test don't leak into the next.
|
|
"""
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self._registry_snapshot = dict(_RADIX_CACHE_REGISTRY)
|
|
|
|
def tearDown(self):
|
|
_RADIX_CACHE_REGISTRY.clear()
|
|
_RADIX_CACHE_REGISTRY.update(self._registry_snapshot)
|
|
super().tearDown()
|
|
|
|
|
|
class TestRegisterRadixCacheBackend(_RegistryIsolationMixin, CustomTestCase):
|
|
def test_register_then_lookup(self):
|
|
factory = MagicMock()
|
|
register_radix_cache_backend("oss_unit_test", factory)
|
|
self.assertIs(get_radix_cache_factory("oss_unit_test"), factory)
|
|
self.assertIn("oss_unit_test", registered_radix_cache_backends())
|
|
|
|
def test_lookup_unknown_returns_none(self):
|
|
self.assertIsNone(get_radix_cache_factory("definitely_not_registered"))
|
|
|
|
def test_empty_name_raises(self):
|
|
with self.assertRaises(ValueError):
|
|
register_radix_cache_backend("", MagicMock())
|
|
|
|
def test_whitespace_only_name_raises(self):
|
|
with self.assertRaises(ValueError):
|
|
register_radix_cache_backend(" ", MagicMock())
|
|
|
|
def test_duplicate_registration_raises(self):
|
|
register_radix_cache_backend("dupe", MagicMock())
|
|
with self.assertRaises(ValueError):
|
|
register_radix_cache_backend("dupe", MagicMock())
|
|
|
|
|
|
class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase):
|
|
def test_dispatches_to_registered_factory(self):
|
|
cache = MagicMock()
|
|
cache.supports_streaming_session.return_value = True
|
|
factory = MagicMock(return_value=cache)
|
|
register_radix_cache_backend("custom", factory)
|
|
|
|
result = create_tree_cache(_make_ctx(self, backend="custom"))
|
|
|
|
factory.assert_called_once()
|
|
self.assertIs(result, cache)
|
|
|
|
def test_unknown_backend_raises(self):
|
|
with self.assertRaises(ValueError):
|
|
create_tree_cache(_make_ctx(self, backend="not_a_real_backend"))
|
|
|
|
@patch("sglang.srt.mem_cache.registry.default_radix_cache_factory")
|
|
def test_unset_backend_falls_back_to_default(self, default_factory):
|
|
cache = MagicMock()
|
|
cache.supports_streaming_session.return_value = True
|
|
default_factory.return_value = cache
|
|
|
|
result = create_tree_cache(_make_ctx(self, backend=None))
|
|
|
|
default_factory.assert_called_once()
|
|
self.assertIs(result, cache)
|
|
|
|
def test_streaming_wrap_when_cache_does_not_support_it(self):
|
|
inner = MagicMock()
|
|
inner.supports_streaming_session.return_value = False
|
|
register_radix_cache_backend("nonstreaming", MagicMock(return_value=inner))
|
|
|
|
with patch(
|
|
"sglang.srt.session.streaming_session.StreamingSession"
|
|
) as session_cls:
|
|
session_cls.return_value = MagicMock(name="wrapped")
|
|
result = create_tree_cache(
|
|
_make_ctx(self, backend="nonstreaming", enable_streaming=True)
|
|
)
|
|
|
|
session_cls.assert_called_once_with(inner)
|
|
self.assertIs(result, session_cls.return_value)
|
|
|
|
def test_no_streaming_wrap_when_cache_supports_it(self):
|
|
inner = MagicMock()
|
|
inner.supports_streaming_session.return_value = True
|
|
register_radix_cache_backend("streaming", MagicMock(return_value=inner))
|
|
|
|
result = create_tree_cache(
|
|
_make_ctx(self, backend="streaming", enable_streaming=True)
|
|
)
|
|
|
|
self.assertIs(result, inner)
|
|
|
|
|
|
class TestDefaultRadixCacheFactory(CustomTestCase):
|
|
"""Branch coverage for the built-in radix cache selection chain.
|
|
|
|
Each cache class is imported lazily inside the factory, so we patch
|
|
the class at its definition site to verify routing without depending
|
|
on each cache's real constructor or runtime state.
|
|
"""
|
|
|
|
def test_chunk_cache_when_chunked_prefill_and_disable_radix(self):
|
|
ctx = _make_ctx(
|
|
self, effective_chunked_prefill_size=512, disable_radix_cache=True
|
|
)
|
|
with patch("sglang.srt.mem_cache.chunk_cache.ChunkCache") as ChunkCache:
|
|
ChunkCache.return_value = MagicMock()
|
|
result = default_radix_cache_factory(ctx)
|
|
ChunkCache.assert_called_once_with(ctx.params)
|
|
self.assertIs(result, ChunkCache.return_value)
|
|
|
|
def test_swa_chunk_cache_when_chunked_prefill_disable_and_hybrid_swa(self):
|
|
ctx = _make_ctx(
|
|
self,
|
|
effective_chunked_prefill_size=512,
|
|
disable_radix_cache=True,
|
|
is_hybrid_swa=True,
|
|
)
|
|
with patch("sglang.srt.mem_cache.chunk_cache.SWAChunkCache") as SWAChunkCache:
|
|
SWAChunkCache.return_value = MagicMock()
|
|
result = default_radix_cache_factory(ctx)
|
|
SWAChunkCache.assert_called_once_with(ctx.params)
|
|
self.assertIs(result, SWAChunkCache.return_value)
|
|
|
|
def test_pure_swa_chunk_cache_when_chunked_prefill_disable_and_all_swa(self):
|
|
ctx = _make_ctx(
|
|
self,
|
|
effective_chunked_prefill_size=512,
|
|
disable_radix_cache=True,
|
|
is_hybrid_swa=True,
|
|
full_tokens_per_layer=0,
|
|
)
|
|
with patch(
|
|
"sglang.srt.mem_cache.chunk_cache.PureSWAChunkCache"
|
|
) as PureSWAChunkCache:
|
|
PureSWAChunkCache.return_value = MagicMock()
|
|
result = default_radix_cache_factory(ctx)
|
|
PureSWAChunkCache.assert_called_once_with(ctx.params)
|
|
self.assertIs(result, PureSWAChunkCache.return_value)
|
|
|
|
def test_unified_radix_cache_is_the_default(self):
|
|
ctx = _make_ctx(
|
|
self,
|
|
)
|
|
# Shim both factory imports — each transitively loads sgl_kernel.
|
|
fake_components = MagicMock()
|
|
fake_radix = MagicMock()
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"sglang.srt.mem_cache.unified_cache.components": fake_components,
|
|
"sglang.srt.mem_cache.unified_radix_cache": fake_radix,
|
|
},
|
|
):
|
|
result = default_radix_cache_factory(ctx)
|
|
fake_radix.UnifiedRadixCache.assert_called_once_with(ctx.params)
|
|
self.assertIs(result, fake_radix.UnifiedRadixCache.return_value)
|
|
|
|
def test_unified_radix_cache_when_hierarchical(self):
|
|
ctx = _make_ctx(self, enable_hierarchical_cache=True)
|
|
# Full attention with hierarchical cache also uses UnifiedRadixCache.
|
|
fake_components = MagicMock()
|
|
fake_radix = MagicMock()
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"sglang.srt.mem_cache.unified_cache.components": fake_components,
|
|
"sglang.srt.mem_cache.unified_radix_cache": fake_radix,
|
|
},
|
|
):
|
|
result = default_radix_cache_factory(ctx)
|
|
fake_radix.UnifiedRadixCache.assert_called_once_with(ctx.params)
|
|
fake_radix.UnifiedRadixCache.return_value.init_hicache.assert_called_once_with(
|
|
ctx.server_args, ctx.params
|
|
)
|
|
ctx.tp_worker.register_hicache_layer_transfer_counter.assert_called_once()
|
|
self.assertIs(result, fake_radix.UnifiedRadixCache.return_value)
|
|
|
|
def test_unified_radix_cache_with_mori_external_linker(self):
|
|
from sglang.srt.mem_cache.storage.umbp import umbp_direct_linker
|
|
|
|
ctx = _make_ctx(self)
|
|
# The factory reads the linker settings from the bags.
|
|
enter_override(
|
|
self,
|
|
get_context().override_server_args(
|
|
enable_unified_cache_external_linker=True,
|
|
unified_cache_external_linker_backend="mori",
|
|
),
|
|
)
|
|
fake_components = MagicMock()
|
|
fake_components.ComponentType.FULL = "full"
|
|
fake_radix = MagicMock()
|
|
cache = fake_radix.UnifiedRadixCache.return_value
|
|
cache.components = ("full",)
|
|
counter = MagicMock(name="layer_done_counter")
|
|
cache.linker.layer_done_counter = counter
|
|
linker = MagicMock(name="linker")
|
|
|
|
with (
|
|
patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"sglang.srt.mem_cache.unified_cache.components": fake_components,
|
|
"sglang.srt.mem_cache.unified_radix_cache": fake_radix,
|
|
},
|
|
),
|
|
patch.object(
|
|
umbp_direct_linker,
|
|
"UMBPDirectLinker",
|
|
return_value=linker,
|
|
) as linker_cls,
|
|
):
|
|
result = default_radix_cache_factory(ctx)
|
|
|
|
linker_cls.assert_called_once_with(
|
|
ctx.server_args,
|
|
ctx.params,
|
|
components={"full"},
|
|
)
|
|
cache.init_cache_linker.assert_called_once_with(linker)
|
|
ctx.params.token_to_kv_pool_allocator.get_kvcache.return_value.register_layer_transfer_counter.assert_called_once_with(
|
|
counter
|
|
)
|
|
ctx.tp_worker.register_hicache_layer_transfer_counter.assert_called_once_with(
|
|
counter
|
|
)
|
|
self.assertIs(result, cache)
|
|
|
|
def test_custom_unified_cache_keeps_host_pool_setup(self):
|
|
ctx = _make_ctx(self)
|
|
enter_override(
|
|
self,
|
|
get_context().override_server_args(
|
|
disaggregation_decode_retraction_backup="host_pool"
|
|
),
|
|
)
|
|
cache_class = MagicMock()
|
|
result = create_unified_radix_cache(ctx, cache_class=cache_class)
|
|
cache_class.assert_called_once_with(ctx.params)
|
|
result.init_hicache.assert_called_once_with(ctx.server_args, ctx.params)
|
|
ctx.tp_worker.register_hicache_layer_transfer_counter.assert_called_once_with(
|
|
result.cache_controller.layer_done_counter
|
|
)
|
|
self.assertIs(result, cache_class.return_value)
|
|
|
|
def test_pure_swa_radix_cache_when_all_swa(self):
|
|
ctx = _make_ctx(self, is_hybrid_swa=True, full_tokens_per_layer=0)
|
|
with patch(
|
|
"sglang.srt.mem_cache.pure_swa_radix_cache.PureSWARadixCache"
|
|
) as PureSWA:
|
|
PureSWA.return_value = MagicMock()
|
|
result = default_radix_cache_factory(ctx)
|
|
PureSWA.assert_called_once_with(params=ctx.params)
|
|
self.assertIs(result, PureSWA.return_value)
|
|
|
|
def test_lmcache_unified_radix_cache_when_enable_lmcache(self):
|
|
ctx = _make_ctx(self, enable_lmcache=True)
|
|
fake_module = MagicMock()
|
|
fake_components = MagicMock()
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module,
|
|
"sglang.srt.mem_cache.unified_cache.components": fake_components,
|
|
},
|
|
):
|
|
result = default_radix_cache_factory(ctx)
|
|
fake_module.LMCacheUnifiedRadixCache.assert_called_once_with(
|
|
ctx.params,
|
|
model_config=ctx.model_config,
|
|
tp_size=ctx.tp_size,
|
|
tp_rank=ctx.tp_rank,
|
|
lmcache_config_file=None,
|
|
forward_stream=ctx.tp_worker.model_runner.forward_stream,
|
|
)
|
|
self.assertEqual(
|
|
ctx.params.tree_components,
|
|
(fake_components.ComponentType.FULL,),
|
|
)
|
|
self.assertIs(result, fake_module.LMCacheUnifiedRadixCache.return_value)
|
|
|
|
def test_lmcache_supports_hybrid_swa_components(self):
|
|
ctx = _make_ctx(self, enable_lmcache=True, is_hybrid_swa=True)
|
|
fake_module = MagicMock()
|
|
fake_components = MagicMock()
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module,
|
|
"sglang.srt.mem_cache.unified_cache.components": fake_components,
|
|
},
|
|
):
|
|
default_radix_cache_factory(ctx)
|
|
|
|
self.assertEqual(
|
|
ctx.params.tree_components,
|
|
(
|
|
fake_components.ComponentType.FULL,
|
|
fake_components.ComponentType.SWA,
|
|
),
|
|
)
|
|
|
|
def test_lmcache_supports_hybrid_ssm_components(self):
|
|
ctx = _make_ctx(self, enable_lmcache=True, is_hybrid_ssm=True)
|
|
fake_module = MagicMock()
|
|
fake_components = MagicMock()
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module,
|
|
"sglang.srt.mem_cache.unified_cache.components": fake_components,
|
|
},
|
|
):
|
|
default_radix_cache_factory(ctx)
|
|
|
|
self.assertEqual(
|
|
ctx.params.tree_components,
|
|
(
|
|
fake_components.ComponentType.FULL,
|
|
fake_components.ComponentType.MAMBA,
|
|
),
|
|
)
|
|
|
|
def test_lmcache_supports_dsa_as_full_sidecar(self):
|
|
ctx = _make_ctx(self, enable_lmcache=True, is_dsa=True)
|
|
fake_module = MagicMock()
|
|
fake_components = MagicMock()
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module,
|
|
"sglang.srt.mem_cache.unified_cache.components": fake_components,
|
|
},
|
|
):
|
|
default_radix_cache_factory(ctx)
|
|
|
|
self.assertEqual(
|
|
ctx.params.tree_components,
|
|
(fake_components.ComponentType.FULL,),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|