[MemCache] Serve disabled full-attention models with UnifiedRadixCache (#43199)

This commit is contained in:
Liangsheng Yin
2026-10-08 17:27:24 -07:00
committed by GitHub
parent 977e2719c7
commit 265efe5e01
2 changed files with 12 additions and 11 deletions
+3 -6
View File
@@ -84,8 +84,9 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0
if ctx.disable_radix_cache and (
get_disagg().disaggregation_decode_retraction_backup == "host_pool"
# Streaming sessions and mamba states need UnifiedRadixCache, whose
# disabled mode replaces the chunk caches; pure-SWA has no unified layout.
# UnifiedRadixCache's disabled mode serves full-attention layouts,
# streaming sessions, and mamba states; pure-SWA has no unified layout.
or not ctx.is_hybrid_swa
or (
not is_pure_swa
and (get_serving().enable_streaming_session or ctx.is_hybrid_ssm)
@@ -100,10 +101,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
return PureSWARadixCache(params=params)
if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache:
if not ctx.is_hybrid_swa:
from sglang.srt.mem_cache.chunk_cache import ChunkCache
return ChunkCache(params)
from sglang.srt.mem_cache.chunk_cache import SWAChunkCache
return SWAChunkCache(params)
@@ -148,15 +148,16 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase):
_make_ctx(self, backend="nonstreaming", enable_streaming=True)
)
def test_chunk_cache_when_chunked_prefill_and_disable_radix(self):
def test_full_attention_with_disable_radix_routes_to_unified(self):
ctx = _make_ctx(
self, effective_chunked_prefill_size=512, disable_radix_cache=True
)
with patch("sglang.srt.mem_cache.chunk_cache.ChunkCache") as ChunkCache:
ChunkCache.return_value = MagicMock()
with patch(
"sglang.srt.mem_cache.registry.create_unified_radix_cache"
) as create_unified:
result = default_radix_cache_factory(ctx)
ChunkCache.assert_called_once_with(ctx.params)
self.assertIs(result, ChunkCache.return_value)
create_unified.assert_called_once_with(ctx)
self.assertIs(result, create_unified.return_value)
def test_swa_chunk_cache_when_chunked_prefill_disable_and_hybrid_swa(self):
ctx = _make_ctx(
@@ -244,11 +245,14 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase):
self.assertIs(result, PureSWARadixCache.return_value)
def test_mamba_with_disable_radix_routes_to_unified(self):
# Hybrid SWA, so the mamba clause (not the full-attention one) routes it.
ctx = _make_ctx(
self,
effective_chunked_prefill_size=512,
disable_radix_cache=True,
is_hybrid_ssm=True,
is_hybrid_swa=True,
full_tokens_per_layer=128,
)
with patch(
"sglang.srt.mem_cache.registry.create_unified_radix_cache"