From 508ba79688dba3afae027977845056a11e371ca5 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 8 Oct 2026 17:30:01 -0700 Subject: [PATCH] [MemCache] Serve disabled hybrid-SWA models with `UnifiedRadixCache` (#43200) --- python/sglang/srt/mem_cache/registry.py | 15 +---- .../mem_cache/unified_cache/components/swa.py | 11 +++- .../unit/mem_cache/test_registry.py | 65 +++++++------------ 3 files changed, 34 insertions(+), 57 deletions(-) diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index fddc164efe..0148e0bb8a 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -82,15 +82,11 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: params = ctx.params is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0 + # The decode host-pool backup needs UnifiedRadixCache; otherwise its + # disabled mode serves every layout but pure-SWA. if ctx.disable_radix_cache and ( get_disagg().disaggregation_decode_retraction_backup == "host_pool" - # UnifiedRadixCache's disabled mode serves full-attention layouts, - # streaming sessions, and mamba states; pure-SWA has no unified layout. - or not ctx.is_hybrid_swa - or ( - not is_pure_swa - and (get_serving().enable_streaming_session or ctx.is_hybrid_ssm) - ) + or not is_pure_swa ): return create_unified_radix_cache(ctx) @@ -100,11 +96,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: return PureSWARadixCache(params=params) - if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache: - from sglang.srt.mem_cache.chunk_cache import SWAChunkCache - - return SWAChunkCache(params) - if get_memory().enable_lmcache: from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import ( LMCacheUnifiedRadixCache, diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index d432ef3976..5458debfbb 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -69,12 +69,19 @@ class SWAComponent(TreeComponent): """ def __init__(self, cache: UnifiedRadixCache, params: CacheInitParams): + from sglang.srt.mem_cache.allocator.hisparse import ( + DeepSeekV4HiSparseTokenToKVPoolAllocator, + ) from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator + # DeepSeek V4 HiSparse forces --disable-radix-cache, where this component + # only frees through free_swa_segment / free / free_group_*, which the + # HiSparse allocator defines. assert isinstance( - params.token_to_kv_pool_allocator, SWATokenToKVPoolAllocator + params.token_to_kv_pool_allocator, + (SWATokenToKVPoolAllocator, DeepSeekV4HiSparseTokenToKVPoolAllocator), ), ( - f"SWAComponent requires SWATokenToKVPoolAllocator, got {type(params.token_to_kv_pool_allocator)}" + f"SWAComponent requires an SWA allocator, got {type(params.token_to_kv_pool_allocator)}" ) if params.sliding_window_size is None or params.sliding_window_size <= 0: raise ValueError("SWAComponent requires a positive sliding_window_size") diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index f2750e3d9f..b83b44ee15 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -159,18 +159,35 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): create_unified.assert_called_once_with(ctx) self.assertIs(result, create_unified.return_value) - def test_swa_chunk_cache_when_chunked_prefill_disable_and_hybrid_swa(self): + def test_hybrid_swa_with_disable_radix_routes_to_unified(self): ctx = _make_ctx( self, effective_chunked_prefill_size=512, disable_radix_cache=True, is_hybrid_swa=True, + full_tokens_per_layer=128, ) - with patch("sglang.srt.mem_cache.chunk_cache.SWAChunkCache") as SWAChunkCache: - SWAChunkCache.return_value = MagicMock() + with patch( + "sglang.srt.mem_cache.registry.create_unified_radix_cache" + ) as create_unified: result = default_radix_cache_factory(ctx) - SWAChunkCache.assert_called_once_with(ctx.params) - self.assertIs(result, SWAChunkCache.return_value) + create_unified.assert_called_once_with(ctx) + self.assertIs(result, create_unified.return_value) + + def test_swa_component_accepts_hisparse_allocator(self): + # Disabled DeepSeek V4 HiSparse routes to UnifiedRadixCache, whose SWA + # component must accept the HiSparse allocator. + from sglang.srt.mem_cache.allocator.hisparse import ( + DeepSeekV4HiSparseTokenToKVPoolAllocator, + ) + from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent + + params = MagicMock(sliding_window_size=128, page_size=64) + params.token_to_kv_pool_allocator = MagicMock( + spec=DeepSeekV4HiSparseTokenToKVPoolAllocator + ) + component = SWAComponent(MagicMock(), params) + self.assertEqual(component.full_window_pages, 2) def test_pure_swa_radix_cache_when_chunked_prefill_disable_and_all_swa(self): ctx = _make_ctx( @@ -223,44 +240,6 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): create_unified.assert_called_once_with(ctx) self.assertIs(result, create_unified.return_value) - def test_streaming_with_disable_radix_keeps_pure_swa_off_unified(self): - ctx = _make_ctx( - self, - effective_chunked_prefill_size=512, - disable_radix_cache=True, - enable_streaming=True, - is_hybrid_swa=True, - full_tokens_per_layer=0, - ) - with ( - patch( - "sglang.srt.mem_cache.registry.create_unified_radix_cache" - ) as create_unified, - patch( - "sglang.srt.mem_cache.pure_swa_radix_cache.PureSWARadixCache" - ) as PureSWARadixCache, - ): - result = default_radix_cache_factory(ctx) - create_unified.assert_not_called() - self.assertIs(result, PureSWARadixCache.return_value) - - def test_mamba_with_disable_radix_routes_to_unified(self): - # Hybrid SWA, so the mamba clause (not the full-attention one) routes it. - ctx = _make_ctx( - self, - effective_chunked_prefill_size=512, - disable_radix_cache=True, - is_hybrid_ssm=True, - is_hybrid_swa=True, - full_tokens_per_layer=128, - ) - with patch( - "sglang.srt.mem_cache.registry.create_unified_radix_cache" - ) as create_unified: - result = default_radix_cache_factory(ctx) - create_unified.assert_called_once_with(ctx) - self.assertIs(result, create_unified.return_value) - def test_mamba_rejected_on_cache_without_mamba(self): inner = MagicMock() inner.supports_mamba.return_value = False