[MemCache] Serve disabled full-attention models with UnifiedRadixCache (#43199)
This commit is contained in:
@@ -84,8 +84,9 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0
|
||||
if ctx.disable_radix_cache and (
|
||||
get_disagg().disaggregation_decode_retraction_backup == "host_pool"
|
||||
# Streaming sessions and mamba states need UnifiedRadixCache, whose
|
||||
# disabled mode replaces the chunk caches; pure-SWA has no unified layout.
|
||||
# UnifiedRadixCache's disabled mode serves full-attention layouts,
|
||||
# streaming sessions, and mamba states; pure-SWA has no unified layout.
|
||||
or not ctx.is_hybrid_swa
|
||||
or (
|
||||
not is_pure_swa
|
||||
and (get_serving().enable_streaming_session or ctx.is_hybrid_ssm)
|
||||
@@ -100,10 +101,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
return PureSWARadixCache(params=params)
|
||||
|
||||
if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache:
|
||||
if not ctx.is_hybrid_swa:
|
||||
from sglang.srt.mem_cache.chunk_cache import ChunkCache
|
||||
|
||||
return ChunkCache(params)
|
||||
from sglang.srt.mem_cache.chunk_cache import SWAChunkCache
|
||||
|
||||
return SWAChunkCache(params)
|
||||
|
||||
@@ -148,15 +148,16 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase):
|
||||
_make_ctx(self, backend="nonstreaming", enable_streaming=True)
|
||||
)
|
||||
|
||||
def test_chunk_cache_when_chunked_prefill_and_disable_radix(self):
|
||||
def test_full_attention_with_disable_radix_routes_to_unified(self):
|
||||
ctx = _make_ctx(
|
||||
self, effective_chunked_prefill_size=512, disable_radix_cache=True
|
||||
)
|
||||
with patch("sglang.srt.mem_cache.chunk_cache.ChunkCache") as ChunkCache:
|
||||
ChunkCache.return_value = MagicMock()
|
||||
with patch(
|
||||
"sglang.srt.mem_cache.registry.create_unified_radix_cache"
|
||||
) as create_unified:
|
||||
result = default_radix_cache_factory(ctx)
|
||||
ChunkCache.assert_called_once_with(ctx.params)
|
||||
self.assertIs(result, ChunkCache.return_value)
|
||||
create_unified.assert_called_once_with(ctx)
|
||||
self.assertIs(result, create_unified.return_value)
|
||||
|
||||
def test_swa_chunk_cache_when_chunked_prefill_disable_and_hybrid_swa(self):
|
||||
ctx = _make_ctx(
|
||||
@@ -244,11 +245,14 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase):
|
||||
self.assertIs(result, PureSWARadixCache.return_value)
|
||||
|
||||
def test_mamba_with_disable_radix_routes_to_unified(self):
|
||||
# Hybrid SWA, so the mamba clause (not the full-attention one) routes it.
|
||||
ctx = _make_ctx(
|
||||
self,
|
||||
effective_chunked_prefill_size=512,
|
||||
disable_radix_cache=True,
|
||||
is_hybrid_ssm=True,
|
||||
is_hybrid_swa=True,
|
||||
full_tokens_per_layer=128,
|
||||
)
|
||||
with patch(
|
||||
"sglang.srt.mem_cache.registry.create_unified_radix_cache"
|
||||
|
||||
Reference in New Issue
Block a user