From 977e2719c7b4a1ca11e3d2e746d21105a4be473c Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 8 Oct 2026 17:25:54 -0700 Subject: [PATCH] [MemCache] Serve disabled pure-SWA models with `PureSWARadixCache` (#43198) --- .../srt/mem_cache/pure_swa_radix_cache.py | 3 +- python/sglang/srt/mem_cache/registry.py | 10 ++- .../mem_cache/test_pure_swa_radix_cache.py | 77 ++++++++++++++++--- .../unit/mem_cache/test_registry.py | 53 ++++++++++--- 4 files changed, 117 insertions(+), 26 deletions(-) diff --git a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py index b51dada1fa..5b671ac2dd 100644 --- a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py @@ -16,7 +16,8 @@ class PureSWARadixCache(RadixCache): Extends RadixCache with SWA semantics. Only caches the prefill portion [0, evict_floor) on request completion. Window-range KV is freed. - No tombstone mechanism needed. + No tombstone mechanism needed. With ``params.disable`` it serves + ``--disable-radix-cache`` and caches nothing. """ def __init__(self, params: CacheInitParams): diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index 638db7fb85..1b648b2e84 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -93,15 +93,17 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: ): return create_unified_radix_cache(ctx) + # A disabled cache publishes nothing, so no storage backend applies. + if ctx.disable_radix_cache and is_pure_swa: + from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache + + return PureSWARadixCache(params=params) + if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache: if not ctx.is_hybrid_swa: from sglang.srt.mem_cache.chunk_cache import ChunkCache return ChunkCache(params) - if ctx.full_tokens_per_layer == 0: - from sglang.srt.mem_cache.chunk_cache import PureSWAChunkCache - - return PureSWAChunkCache(params) from sglang.srt.mem_cache.chunk_cache import SWAChunkCache return SWAChunkCache(params) diff --git a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py index aca959b135..f3780f00d7 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py @@ -35,6 +35,7 @@ class _FakeAllocator: def __init__(self): self.freed = [] self.skipped = [] + self.window_freed = [] def free_segment(self, indices, *, start_pos): self.freed.extend(indices.tolist()) @@ -47,21 +48,28 @@ class _FakeAllocator: for indices, _ in segments: self.skipped.extend(indices.tolist()) + def free_swa_segment(self, indices, *, start_pos): + self.window_freed.extend(indices.tolist()) + + +def _make_cache(*, disable, num_tokens, page_size=1): + return PureSWARadixCache( + CacheInitParams( + disable=disable, + req_to_token_pool=_FakeReqToTokenPool( + torch.arange(num_tokens, dtype=torch.int64).unsqueeze(0) + ), + token_to_kv_pool_allocator=_FakeAllocator(), + page_size=page_size, + sliding_window_size=4, + ) + ) + class TestPureSWARadixCache(CustomTestCase): def test_finish_inserts_up_to_the_evict_floor_and_frees_the_rest(self): - allocator = _FakeAllocator() - cache = PureSWARadixCache( - CacheInitParams( - disable=False, - req_to_token_pool=_FakeReqToTokenPool( - torch.arange(8, dtype=torch.int64).unsqueeze(0) - ), - token_to_kv_pool_allocator=allocator, - page_size=1, - sliding_window_size=4, - ) - ) + cache = _make_cache(disable=False, num_tokens=8) + allocator = cache.token_to_kv_pool_allocator token_ids = array("q", range(8)) req = SimpleNamespace( origin_input_ids=token_ids, @@ -90,5 +98,50 @@ class TestPureSWARadixCache(CustomTestCase): self.assertEqual(allocator.skipped, [4, 5]) +class TestDisabledPureSWARadixCache(CustomTestCase): + def test_finished_req_skips_protected_prefix_and_evicted_range(self): + cache = _make_cache(disable=True, num_tokens=10) + allocator = cache.token_to_kv_pool_allocator + token_ids = array("q", range(8)) + match = cache.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) + req = SimpleNamespace( + origin_input_ids=token_ids, + output_ids=array("q"), + full_untruncated_fill_ids=token_ids, + extra_key=None, + cache_salt=None, + last_node=match.last_device_node, + lock=cache.lock(match.last_device_node), + priority=0, + kv=ReqKvInfo( + req_pool_idx=0, + cache_protected_len=2, + swa_evict_floor=3, + component_evicted_seqlens={ComponentType.SWA: 6}, + ), + ) + + # protected 2, floor 3, cursor 6: [2, 3) and [6, 8) go back, [3, 6) is dead + cache.checkpoint(req, up_to=8) + cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) + cache.unlock(req.lock) + + self.assertEqual(req.kv.cache_protected_len, 2) + self.assertEqual(allocator.freed, [2, 6, 7]) + self.assertEqual(allocator.skipped, [3, 4, 5]) + self.assertEqual(cache.total_size(), 0) + + def test_window_eviction_frees_up_to_the_window(self): + cache = _make_cache(disable=True, num_tokens=20, page_size=8) + req = SimpleNamespace(kv=ReqKvInfo(req_pool_idx=0)) + + cache.evict_sliding_windows(req, 20) + + # pre_len 20 - window 4 = 16, page-aligned: [0, 16) slides out. A + # prefix-sharing cache would keep max(window, page) and stop at 8. + self.assertEqual(cache.token_to_kv_pool_allocator.window_freed, list(range(16))) + self.assertEqual(req.kv.get_evicted_seqlen(ComponentType.SWA), 16) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index 39d434be01..7dc5a76eb5 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -171,7 +171,7 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): SWAChunkCache.assert_called_once_with(ctx.params) self.assertIs(result, SWAChunkCache.return_value) - def test_pure_swa_chunk_cache_when_chunked_prefill_disable_and_all_swa(self): + def test_pure_swa_radix_cache_when_chunked_prefill_disable_and_all_swa(self): ctx = _make_ctx( self, effective_chunked_prefill_size=512, @@ -180,12 +180,47 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): full_tokens_per_layer=0, ) with patch( - "sglang.srt.mem_cache.chunk_cache.PureSWAChunkCache" - ) as PureSWAChunkCache: - PureSWAChunkCache.return_value = MagicMock() + "sglang.srt.mem_cache.pure_swa_radix_cache.PureSWARadixCache" + ) as PureSWARadixCache: + PureSWARadixCache.return_value = MagicMock() result = default_radix_cache_factory(ctx) - PureSWAChunkCache.assert_called_once_with(ctx.params) - self.assertIs(result, PureSWAChunkCache.return_value) + PureSWARadixCache.assert_called_once_with(params=ctx.params) + self.assertIs(result, PureSWARadixCache.return_value) + + def test_pure_swa_with_disable_radix_skips_storage_backends(self): + ctx = _make_ctx( + self, disable_radix_cache=True, is_hybrid_swa=True, full_tokens_per_layer=0 + ) + enter_override( + self, + get_context().override_server_args( + enable_unified_cache_external_linker=True + ), + ) + with patch( + "sglang.srt.mem_cache.pure_swa_radix_cache.PureSWARadixCache" + ) as PureSWARadixCache: + PureSWARadixCache.return_value = MagicMock() + result = default_radix_cache_factory(ctx) + PureSWARadixCache.assert_called_once_with(params=ctx.params) + self.assertIs(result, PureSWARadixCache.return_value) + + def test_pure_swa_with_disable_radix_and_host_pool_goes_to_unified(self): + ctx = _make_ctx( + self, disable_radix_cache=True, is_hybrid_swa=True, full_tokens_per_layer=0 + ) + enter_override( + self, + get_context().override_server_args( + disaggregation_decode_retraction_backup="host_pool" + ), + ) + with patch( + "sglang.srt.mem_cache.registry.create_unified_radix_cache" + ) as create_unified: + result = default_radix_cache_factory(ctx) + create_unified.assert_called_once_with(ctx) + self.assertIs(result, create_unified.return_value) def test_streaming_with_disable_radix_keeps_pure_swa_off_unified(self): ctx = _make_ctx( @@ -201,12 +236,12 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): "sglang.srt.mem_cache.registry.create_unified_radix_cache" ) as create_unified, patch( - "sglang.srt.mem_cache.chunk_cache.PureSWAChunkCache" - ) as PureSWAChunkCache, + "sglang.srt.mem_cache.pure_swa_radix_cache.PureSWARadixCache" + ) as PureSWARadixCache, ): result = default_radix_cache_factory(ctx) create_unified.assert_not_called() - self.assertIs(result, PureSWAChunkCache.return_value) + self.assertIs(result, PureSWARadixCache.return_value) def test_mamba_with_disable_radix_routes_to_unified(self): ctx = _make_ctx(