From b4fe28283659f7cde21add448da69e551e39f25a Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 8 Oct 2026 17:31:16 -0700 Subject: [PATCH] [MemCache] Delete `ChunkCache` (#43201) --- .github/labeler.yml | 1 - .../autoregressive/Moonshotai/Kimi-K3.mdx | 2 +- docs/docs/advanced_features/dcp.mdx | 2 +- .../_kimi_k3_mamba_ratio_calculator.jsx | 5 +- .../snippets/configs/moonshotai/kimi-k3.jsx | 6 +- .../srt/arg_groups/pd_disaggregation_hook.py | 2 +- .../sglang/srt/disaggregation/role_switch.py | 2 +- python/sglang/srt/managers/schedule_batch.py | 3 +- python/sglang/srt/mem_cache/README.md | 3 +- python/sglang/srt/mem_cache/allocation.py | 7 +- .../sglang/srt/mem_cache/base_prefix_cache.py | 4 +- python/sglang/srt/mem_cache/chunk_cache.py | 133 ------------------ .../sglang/srt/mem_cache/kv_cache_builder.py | 1 - python/sglang/srt/mem_cache/registry.py | 26 ++-- .../srt/model_executor/pool_configurator.py | 3 +- .../test_disaggregation_inkling_mxfp8.py | 2 +- .../scheduler/test_retract_decode.py | 8 +- .../disaggregation/test_pd_role_switch.py | 2 +- .../test_scheduler_chunked_req_gate.py | 43 ++---- .../mem_cache/test_pure_swa_chunk_cache.py | 74 ---------- .../unit/mem_cache/test_radix_force_miss.py | 6 +- .../unit/mem_cache/test_registry.py | 23 +-- .../unit/server_args/test_server_args.py | 2 +- 23 files changed, 57 insertions(+), 303 deletions(-) delete mode 100644 python/sglang/srt/mem_cache/chunk_cache.py delete mode 100644 test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py diff --git a/.github/labeler.yml b/.github/labeler.yml index a2e5d689eb..1c5b0ac2c6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -115,7 +115,6 @@ unified-radix-cache: - any-glob-to-any-file: - 'python/sglang/srt/mem_cache/unified_cache/**/*' - 'python/sglang/srt/mem_cache/**/*radix*' - - 'python/sglang/srt/mem_cache/chunk_cache.py' - 'python/sglang/srt/mem_cache/evict_policy.py' - 'python/sglang/srt/mem_cache/base_prefix_cache.py' - 'test/registered/radix_cache/**/*' diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 60edb6c5f8..0a21246dda 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -316,7 +316,7 @@ PD splits prefill and decode into separate server groups; because K3 is hybrid, - **Transfer**: the CUDA cells emit **NiXL** (RDMA); Mooncake stays selectable in the Playground. The Ascend NPU cells emit `--disaggregation-transfer-backend ascend` — the KV/KDA transfer rides the Ascend memory fabric instead (see [below](#npu-pd)). - **Ports**: prefill `30000`, decode `30100` (derived ZMQ/dist ranges must not collide on a shared host). The positional `8998` after `--prefill` must match `--disaggregation-bootstrap-port`, or only the decode worker registers. -- **Decode state pool**: chunk cache — one slot per request; `--mamba-radix-cache-strategy` is inert. Keep `--disaggregation-decode-extra-slots` pinned: unpinned it defaults to twice the batch below 32 requests and **zero** above. +- **Decode state pool**: radix cache disabled — one slot per request; `--mamba-radix-cache-strategy` is inert. Keep `--disaggregation-decode-extra-slots` pinned: unpinned it defaults to twice the batch below 32 requests and **zero** above. diff --git a/docs/docs/advanced_features/dcp.mdx b/docs/docs/advanced_features/dcp.mdx index d88bcb588e..a0170e53e1 100644 --- a/docs/docs/advanced_features/dcp.mdx +++ b/docs/docs/advanced_features/dcp.mdx @@ -162,7 +162,7 @@ A dense prefill page cannot be copied directly into striped decode storage. The The plan includes the exact token count, so stale rows do not leak out of a partial final page. Mooncake and NIXL use the same plan. KDA state keeps its attention-TP mapping and bypasses DCP filtering. -This composition requires Mooncake or NIXL, matching physical page size and KV dtype, prefill attention CP `1`, and decode chunk cache. Decode radix cache and HiCache are not supported here. +This composition requires Mooncake or NIXL, matching physical page size and KV dtype, and prefill attention CP `1`. Decode radix cache and HiCache are not supported here. ### DCP × HiCache L2 diff --git a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx index 6be4e81f0d..e2b883582e 100644 --- a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx +++ b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx @@ -98,8 +98,7 @@ export const KimiK3MambaRatioCalculator = () => { // the decode-lock skip, plus the ping-pong track buffer (2 under the overlap // scheduler, 1 for lazy or without overlap). no_buffer has no track buffer and // adds the skip's drop back, so it stays 3; a disabled radix cache is 1. - // A PD decode server runs a chunk cache: one live slot per request, and the - // radix-strategy knobs are inert there. + // A PD decode server disables it too; the radix-strategy knobs are inert there. const slots = pdRole === "decode" ? 1 : radixOff @@ -239,7 +238,7 @@ export const KimiK3MambaRatioCalculator = () => { `KV ${kvDtype === "fp8_e4m3" ? "FP8" : "BF16"}`, `State ${ssmDtype === "float32" ? "FP32" : ssmDtype === "bfloat16" ? "BF16" : "FP16"}`, pdRole === "decode" - ? "PD decode: chunk cache (S = 1)" + ? "PD decode: radix off (S = 1)" : radixOff ? "Radix off (S = 1)" : `${strategy}${skipLock ? " + slot saving" : ""} (S = ${slots})`, diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index f1f2594976..75a205a9e0 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -1013,13 +1013,13 @@ export const config = { ], }, { - // A decode server runs a chunk cache by default (1 state slot/req); + // A decode server disables the radix cache by default (1 state slot/req); // radix restores prefix reuse at the unified per-request slot cost. id: "pdDecodeRadix", title: "PD Decode Radix Cache", showWhen: (b) => b.pdMode === "decode", stripPrefixes: ["--disaggregation-decode-enable-radix-cache"], options: [ - { id: "off", label: "Off (chunk cache)" }, + { id: "off", label: "Off" }, { id: "on", label: "On", flags: ["--disaggregation-decode-enable-radix-cache"] }, ], }, @@ -1917,7 +1917,7 @@ export const config = { // shared PP2 x TP8 prefill. Comparisons hold; absolutes would be higher // behind the PP16 x TP1 prefill cell above. // - // Decode runs the KV cache as a chunk cache, so the unified 5-slots-per- + // Decode runs with the radix cache disabled, so the unified 5-slots-per- // request reservation (1 state + ping-pong copies for radix reuse) drops to // a single slot, and --mamba-radix-cache-strategy stops having any effect. // In-transfer requests holding a slot before decode starts are the only diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index da8125869f..c0ef52335f 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -130,7 +130,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: "handle_pd_disaggregation", disable_radix_cache=True, ) - logger.warning("KV cache is forced as chunk cache for decode server") + logger.warning("Radix cache is disabled for decode server") # Default the number of *extra* decode req_to_token slots reserved for # in-transfer (being-received-from-prefill) requests, on top of the diff --git a/python/sglang/srt/disaggregation/role_switch.py b/python/sglang/srt/disaggregation/role_switch.py index 8bf90e20ad..cc820a4e58 100644 --- a/python/sglang/srt/disaggregation/role_switch.py +++ b/python/sglang/srt/disaggregation/role_switch.py @@ -256,7 +256,7 @@ def teardown_disaggregation(scheduler: Scheduler) -> None: def _release_prefix_cache_for_role_switch(scheduler: Scheduler) -> None: """Release the prefix (radix/hicache) cache so a flip works with radix ON. - With radix disabled (ChunkCache) the flip needs nothing here: ChunkCache + With radix disabled the flip needs nothing here: the disabled cache keeps no persistent prefixes and, since the instance is idle before the switch, the allocator is already empty. This is the historical ``--disable-radix-cache`` path, left untouched by the guard below. diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 71e3cfde19..3541eef624 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1220,7 +1220,8 @@ class Req(ReqDllmMixin): # stamp new tree nodes. Allocation itself must NOT read it back when # a tree node is available (the checkpoint dedup rebind # would make it stale); the only allocation-time reader is the - # ChunkCache fallback, which has no tree nodes and no rebind. + # radix-disabled fallback, which keeps no nodes past the root and has + # no rebind. self.kv_rotation_base: Optional[int] = None # Whether or not if it is chunked. It increments whenever diff --git a/python/sglang/srt/mem_cache/README.md b/python/sglang/srt/mem_cache/README.md index bc08fb6dde..5b2838e810 100644 --- a/python/sglang/srt/mem_cache/README.md +++ b/python/sglang/srt/mem_cache/README.md @@ -38,8 +38,7 @@ to keep. The layout is specified in Two groups sit outside that stack: - **Radix cache** is its own axis. The per-model variants (`radix_cache.py`, - `chunk_cache.py`) - are converging onto the **Unified Radix Cache** (`unified_cache/`, + `pure_swa_radix_cache.py`) are converging onto the **Unified Radix Cache** (`unified_cache/`, [#20415](https://github.com/sgl-project/sglang/issues/20415)), whose Full/SWA/Mamba component model is documented in [`unified_cache/components/README.md`](unified_cache/components/README.md). diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index e073e76349..cdaa2e1565 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -262,9 +262,10 @@ def _kv_shard_rotation_bases( allocator draws from the least-full class at that request's turn (so the draw sees earlier requests' pops in the same batch) and resolves the entry in place. - - ChunkCache has no tree nodes (``last_node`` is None); its chunked - continuations fall back to the base recorded on the request at the - previous chunk's alloc (no cross-request reuse, no rebind there). + - A radix-disabled cache keeps no nodes past the root, which carries no + base; its chunked continuations fall back to the base recorded on the + request at the previous chunk's alloc (no cross-request reuse, no + rebind there). """ assert batch is not None and len(batch.reqs) == len(prefix_lens_cpu) bases = [] diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index bc74b767f8..c58f387ca2 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -322,7 +322,7 @@ def zero_match_result( tree_cache, match_result: MatchResult, extra_key: Optional[str] = None ) -> MatchResult: if not tree_cache.supports_prefix_sharing(): - # match_prefix already returns a miss; no root_node to walk back to. + # A non-sharing cache's match is already a miss; pass it through. return match_result root = tree_cache.root_node_handle(extra_key=extra_key) return match_result._replace( @@ -467,7 +467,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait): """Logical-page KV sharding: the rotation base stamped on ``node``. ``node`` is whatever this cache stores in ``req.last_node`` (a NodeId - for the unified tree, None for caches without tree nodes). None means + for the unified tree). None means "no base available here", which sends the alloc path to the base the request recorded at its previous alloc. Tree caches that keep the per-chain base override this. See UnifiedTreeNode.rotation_base. diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py deleted file mode 100644 index 7fafebddc2..0000000000 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import annotations - -"""Cache for chunked prefill, used when RadixCache is disabled.""" - -import logging -from typing import TYPE_CHECKING, Any, Optional - -import torch - -from sglang.srt.mem_cache.allocator.hisparse import ( - DeepSeekV4HiSparseTokenToKVPoolAllocator, -) -from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator -from sglang.srt.mem_cache.base_prefix_cache import ( - BasePrefixCache, - DecLockRefParams, - DecLockRefResult, - EvictParams, - EvictResult, - IncLockRefResult, - InsertParams, - InsertResult, - MatchPrefixParams, - MatchResult, -) - -if TYPE_CHECKING: - from sglang.srt.managers.schedule_batch import Req - from sglang.srt.mem_cache.cache_init_params import CacheInitParams - - -logger = logging.getLogger(__name__) - - -class ChunkCache(BasePrefixCache): - """ - ChunkCache is used when radix cache is disabled. - - That includes standard chunked-prefill setups and the decode side of P/D - disaggregation when decode radix cache is not enabled. - """ - - def __init__(self, params: CacheInitParams): - self.req_to_token_pool = params.req_to_token_pool - self.token_to_kv_pool_allocator = params.token_to_kv_pool_allocator - self.page_size = params.page_size - if self.token_to_kv_pool_allocator: - self.device = self.token_to_kv_pool_allocator.device - else: - self.device = torch.device("cpu") - - self.protected_size_ = 0 - - def supports_prefix_sharing(self) -> bool: - return False - - # NOTE (csy): this is to determine if a cache has prefix matching feature. - # Chunk cache always return True to indicate no prefix matching. - # TODO (csy): Using a prefix cache trait to replace this - @property - def disable(self): - return True - - def reset(self): - pass - - def match_prefix(self, params: MatchPrefixParams) -> MatchResult: - return MatchResult( - device_prefix_len=0, - last_device_node=None, - last_host_node=None, - best_match_node=None, - ) - - def path_device_indices(self, node: Any) -> torch.Tensor: - return torch.empty((0,), dtype=torch.int64) - - def insert(self, params: InsertParams) -> InsertResult: - # ChunkCache does not support prefix caching, so insert is a no-op - return InsertResult(prefix_len=0) - - def checkpoint(self, req: Req, *, up_to: int): - pass - - def evict(self, params: EvictParams) -> EvictResult: - return EvictResult() - - def inc_lock_ref(self, node: Any) -> IncLockRefResult: - return IncLockRefResult(delta=0) - - def dec_lock_ref( - self, node: Any, params: Optional[DecLockRefParams] = None - ) -> DecLockRefResult: - return DecLockRefResult(delta=0) - - def protected_size(self): - # NOTE: no protected size in chunk cache. Chunk cache's eviction is the same with request's lifecycle. - return 0 - - def pretty_print(self): - return "" - - -class SWAChunkCache(ChunkCache): - """ChunkCache with support for sliding window attention.""" - - def __init__(self, params: CacheInitParams): - # DeepSeek V4 HiSparse wraps SWATokenToKVPoolAllocator and exposes the same API. - assert isinstance( - params.token_to_kv_pool_allocator, - ( - SWATokenToKVPoolAllocator, - DeepSeekV4HiSparseTokenToKVPoolAllocator, - ), - ) - super().__init__(params) - - self.sliding_window_size = params.sliding_window_size - self.chunked_prefill_size = params.chunked_prefill_size - - def supports_swa(self) -> bool: - assert self.sliding_window_size is not None, ( - "sliding_window_size must be set for SWAChunkCache" - ) - return True - - def evict(self, params: EvictParams) -> EvictResult: - return EvictResult() - - -class PureSWAChunkCache(SWAChunkCache): - """ChunkCache for all-SWA models (no full attention layers): no - full_to_swa_index_mapping, so free_kv_row must skip the window-evicted span.""" diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 016d9c4cfa..2bb3ad2037 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -367,7 +367,6 @@ def build_kv_cache( is_dsa=is_dsa, enable_hierarchical_cache=enable_hierarchical_cache, disable_radix_cache=disable_radix_cache, - effective_chunked_prefill_size=effective_chunked_prefill_size, tp_worker=tp_worker, model_config=model_config, tp_size=parallel.tp_size, diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index 0148e0bb8a..25343d74f5 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -37,7 +37,6 @@ class TreeCacheBuildContext: is_hybrid_ssm: bool enable_hierarchical_cache: bool disable_radix_cache: bool - effective_chunked_prefill_size: Optional[int] tp_worker: Any model_config: ModelConfig tp_size: int @@ -82,19 +81,18 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: params = ctx.params is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0 - # The decode host-pool backup needs UnifiedRadixCache; otherwise its - # disabled mode serves every layout but pure-SWA. - if ctx.disable_radix_cache and ( - get_disagg().disaggregation_decode_retraction_backup == "host_pool" - or not is_pure_swa - ): - return create_unified_radix_cache(ctx) - # A disabled cache publishes nothing, so no storage backend applies. - if ctx.disable_radix_cache and is_pure_swa: - from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache + # Host-pool retraction goes to UnifiedRadixCache, which validates it (and + # rejects pure-SWA). + if ctx.disable_radix_cache: + if ( + is_pure_swa + and get_disagg().disaggregation_decode_retraction_backup != "host_pool" + ): + from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache - return PureSWARadixCache(params=params) + return PureSWARadixCache(params=params) + return create_unified_radix_cache(ctx) if get_memory().enable_lmcache: from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import ( @@ -103,7 +101,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: from sglang.srt.mem_cache.unified_cache.components import ComponentType tree_components = [] - if not (ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0): + if not is_pure_swa: tree_components.append(ComponentType.FULL) if ctx.is_hybrid_swa: tree_components.append(ComponentType.SWA) @@ -122,7 +120,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: if get_memory().enable_unified_cache_external_linker: return create_unified_radix_cache(ctx) - if ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0: + if is_pure_swa: from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache return PureSWARadixCache(params=params) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 29301ba3a3..15c5492f15 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -943,7 +943,8 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator): @staticmethod def is_applicable(kvc: KVCacheConfigurator) -> bool: - """True when SWAChunkCache can be sized from explicit max requests.""" + """True when a radix-disabled hybrid SWA cache can be sized from + explicit max requests.""" if get_schedule().max_running_requests is None: return False if not get_memory().disable_radix_cache: diff --git a/test/registered/disaggregation/test_disaggregation_inkling_mxfp8.py b/test/registered/disaggregation/test_disaggregation_inkling_mxfp8.py index 02c7ddf192..8ff1bef948 100644 --- a/test/registered/disaggregation/test_disaggregation_inkling_mxfp8.py +++ b/test/registered/disaggregation/test_disaggregation_inkling_mxfp8.py @@ -82,7 +82,7 @@ class TestDisaggregationInklingMXFP8(PDDisaggregationServerBase, GSM8KMixin): @classmethod def start_prefill(cls): - # HiCache rides the prefill role only: the decode role forces chunk cache, + # HiCache rides the prefill role only: the decode role disables radix cache, # and its radix opt-in is refused for sliding-window models. prefill_args = [ "--trust-remote-code", diff --git a/test/registered/scheduler/test_retract_decode.py b/test/registered/scheduler/test_retract_decode.py index d6bba96525..77e9edb0d1 100644 --- a/test/registered/scheduler/test_retract_decode.py +++ b/test/registered/scheduler/test_retract_decode.py @@ -62,8 +62,8 @@ class TestRetractDecode(CustomTestCase): assert self.process.poll() is None, "Server crashed during test" -class TestRetractDecodeChunkCachePaged(TestRetractDecode): - """python -m unittest test_retract_decode.TestRetractDecodeChunkCachePaged""" +class TestRetractDecodeRadixDisabledPaged(TestRetractDecode): + """python -m unittest test_retract_decode.TestRetractDecodeRadixDisabledPaged""" other_args = ["--disable-radix-cache", "--page-size", "16"] @@ -102,8 +102,8 @@ class TestRetractDecodeLongOutput(CustomTestCase): @unittest.skipIf(is_in_ci(), "Skipped in CI due to long runtime") -class TestRetractDecodeLongOutputChunkCache(TestRetractDecodeLongOutput): - """python -m unittest test_retract_decode.TestRetractDecodeLongOutputChunkCache""" +class TestRetractDecodeLongOutputRadixDisabled(TestRetractDecodeLongOutput): + """python -m unittest test_retract_decode.TestRetractDecodeLongOutputRadixDisabled""" other_args = ["--disable-radix-cache"] diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py index 7114d5facc..473475fc91 100644 --- a/test/registered/unit/disaggregation/test_pd_role_switch.py +++ b/test/registered/unit/disaggregation/test_pd_role_switch.py @@ -573,7 +573,7 @@ def _radix_scheduler(disable_radix_cache): @unittest.skipUnless(_HAS_ROLE_SWITCH, "role_switch not importable in this env") class TestReleasePrefixCacheOnRoleSwitch(unittest.TestCase): """The flip may run with radix cache ENABLED: teardown resets the tree cache - + KV pools when radix is on, and is a no-op on the historical chunk-cache path.""" + + KV pools when radix is on, and is a no-op on the radix-disabled path.""" def test_noop_when_radix_disabled(self): s = _radix_scheduler(disable_radix_cache=True) diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index 313f9fe43a..3d000f318b 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -5,8 +5,6 @@ from array import array from types import SimpleNamespace from unittest.mock import MagicMock -import torch - from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel @@ -15,7 +13,7 @@ maybe_stub_sgl_kernel() from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.managers.schedule_batch import NextBatchPlan, Req, ReqKvInfo from sglang.srt.managers.scheduler import Scheduler -from sglang.srt.mem_cache.chunk_cache import ChunkCache +from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -52,23 +50,9 @@ def _make_req( return req -def _make_req_to_token_pool(num_slots: int, max_context: int) -> SimpleNamespace: - pool = SimpleNamespace() - pool.req_to_token = ( - torch.arange(max_context, dtype=torch.int32).unsqueeze(0).repeat(num_slots, 1) - + torch.arange(num_slots, dtype=torch.int32).unsqueeze(1) * 1000 - ) - return pool - - -def _make_chunk_cache(req_to_token_pool) -> ChunkCache: - return ChunkCache( - SimpleNamespace( - req_to_token_pool=req_to_token_pool, - token_to_kv_pool_allocator=None, - page_size=1, - ) - ) +def _make_tree_cache() -> BasePrefixCache: + # The gate under test lives in the scheduler; the cache is a stub. + return MagicMock(spec=BasePrefixCache) def _scheduler_for_get_next_batch(*, tree_cache, chunked_req) -> Scheduler: @@ -120,12 +104,9 @@ class TestStashGatePreservesPrefix(CustomTestCase): POOL_IDX = 4 INITIAL_PREFIX_LEN = 8 # what was really cached last iter POST_RESET_FILL_LEN = 32 # length after init_next_round_input rebuilds - NUM_SLOTS = 8 - MAX_CONTEXT = 64 def _build(self, *, fill_len: int): - pool = _make_req_to_token_pool(self.NUM_SLOTS, self.MAX_CONTEXT) - cache = _make_chunk_cache(pool) + cache = _make_tree_cache() req = _make_req( req_pool_idx=self.POOL_IDX, fill_ids=list(range(self.POST_RESET_FILL_LEN)), @@ -133,41 +114,45 @@ class TestStashGatePreservesPrefix(CustomTestCase): fill_len=fill_len, ) s = _scheduler_for_get_next_batch(tree_cache=cache, chunked_req=req) - return s, req, pool + return s, req def test_parked_chunked_req_keeps_its_prefix(self): # A parked chunk has fill_len == prefix_len: no new KV was computed, # so the gate must skip stash and leave the prefix intact. - s, req, _ = self._build(fill_len=self.INITIAL_PREFIX_LEN) + s, req = self._build(fill_len=self.INITIAL_PREFIX_LEN) Scheduler.get_next_batch_to_run( s, running_batch=s.running_batch, last_batch=s.last_batch ) self.assertEqual(req.prefix_len, self.INITIAL_PREFIX_LEN) + s.tree_cache.checkpoint.assert_not_called() def test_scheduled_chunked_req_advances_prefix_via_real_stash(self): # Symmetric guard against over-gating: when fill_len has advanced past # the cached prefix, stash must run and advance prefix_len. - s, req, _ = self._build(fill_len=self.POST_RESET_FILL_LEN) + s, req = self._build(fill_len=self.POST_RESET_FILL_LEN) Scheduler.get_next_batch_to_run( s, running_batch=s.running_batch, last_batch=s.last_batch ) self.assertEqual(req.prefix_len, self.POST_RESET_FILL_LEN) + s.tree_cache.checkpoint.assert_called_once_with( + req, up_to=self.POST_RESET_FILL_LEN + ) def test_no_chunked_req_never_mutates_state(self): # The outer `if chunked_req is not None` guard must hold on the retract # path that clears chunked_req. - pool = _make_req_to_token_pool(self.NUM_SLOTS, self.MAX_CONTEXT) - cache = _make_chunk_cache(pool) + cache = _make_tree_cache() s = _scheduler_for_get_next_batch(tree_cache=cache, chunked_req=None) Scheduler.get_next_batch_to_run( s, running_batch=s.running_batch, last_batch=s.last_batch ) self.assertIsNone(s.chunked_req) + cache.checkpoint.assert_not_called() if __name__ == "__main__": diff --git a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py deleted file mode 100644 index e0a2305c90..0000000000 --- a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Unit tests for all-SWA ChunkCache release semantics.""" - -import unittest -from types import SimpleNamespace - -import torch - -from sglang.srt.managers.schedule_batch import ReqKvInfo -from sglang.srt.mem_cache.chunk_cache import PureSWAChunkCache -from sglang.srt.mem_cache.unified_cache.component_type import ComponentType -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=10, suite="base-a-test-cpu") - - -class _FakeAllocator: - """Rows routed to the full side are skipped: all-SWA has no full pool.""" - - page_size = 1 - - def __init__(self): - self.freed = [] - self.skipped = [] - - def free_segment(self, indices, *, start_pos): - self.freed.extend(indices.tolist()) - - def free_segments(self, segments): - for indices, start_pos in segments: - self.free_segment(indices, start_pos=start_pos) - - def free_full_segments(self, segments): - for indices, _ in segments: - self.skipped.extend(indices.tolist()) - - -def _make_req(): - return SimpleNamespace( - last_node=None, - lock=None, - kv=ReqKvInfo( - req_pool_idx=0, - cache_protected_len=2, - swa_evict_floor=3, - component_evicted_seqlens={ComponentType.SWA: 6}, - ), - ) - - -class TestPureSWAChunkCache(CustomTestCase): - def _make_cache(self): - cache = PureSWAChunkCache.__new__(PureSWAChunkCache) - cache.req_to_token_pool = SimpleNamespace( - req_to_token=torch.arange(10, dtype=torch.int64).unsqueeze(0) - ) - cache.token_to_kv_pool_allocator = _FakeAllocator() - return cache - - def test_finished_req_skips_protected_prefix_and_evicted_range(self): - cache = self._make_cache() - - # protected 2, floor 3, cursor 6: [2, 3) and [6, 8) go back, [3, 6) is dead - req = _make_req() - cache.checkpoint(req, up_to=8) - cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) - cache.unlock(req.lock) - - self.assertEqual(cache.token_to_kv_pool_allocator.freed, [2, 6, 7]) - self.assertEqual(cache.token_to_kv_pool_allocator.skipped, [3, 4, 5]) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index a70b46c289..c12cda2ef2 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -58,8 +58,8 @@ class TestZeroMatchResult(unittest.TestCase): self.assertIs(zeroed.best_match_node, tree.root_node) self.assertEqual(zeroed.host_hit_length, 0) - def test_chunk_cache_is_passthrough(self): - class _StubChunkCache: + def test_non_sharing_cache_is_passthrough(self): + class _StubNonSharingCache: def supports_prefix_sharing(self) -> bool: return False @@ -70,7 +70,7 @@ class TestZeroMatchResult(unittest.TestCase): best_match_node=None, host_hit_length=0, ) - self.assertIs(zero_match_result(_StubChunkCache(), original), original) + self.assertIs(zero_match_result(_StubNonSharingCache(), original), original) class TestMatchKvCacheForceMiss(unittest.TestCase): diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index b83b44ee15..3d42211dcf 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -43,7 +43,6 @@ def _make_ctx( is_dsa=False, enable_hierarchical_cache=False, disable_radix_cache=False, - effective_chunked_prefill_size=None, full_tokens_per_layer=None, ): # The factory reads the published bags for the cache-backend leaves, so the @@ -65,7 +64,6 @@ def _make_ctx( is_dsa=is_dsa, enable_hierarchical_cache=enable_hierarchical_cache, disable_radix_cache=disable_radix_cache, - effective_chunked_prefill_size=effective_chunked_prefill_size, tp_worker=MagicMock(), model_config=MagicMock(), tp_size=1, @@ -149,9 +147,7 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): ) def test_full_attention_with_disable_radix_routes_to_unified(self): - ctx = _make_ctx( - self, effective_chunked_prefill_size=512, disable_radix_cache=True - ) + ctx = _make_ctx(self, disable_radix_cache=True) with patch( "sglang.srt.mem_cache.registry.create_unified_radix_cache" ) as create_unified: @@ -162,7 +158,6 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): def test_hybrid_swa_with_disable_radix_routes_to_unified(self): ctx = _make_ctx( self, - effective_chunked_prefill_size=512, disable_radix_cache=True, is_hybrid_swa=True, full_tokens_per_layer=128, @@ -189,22 +184,6 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): component = SWAComponent(MagicMock(), params) self.assertEqual(component.full_window_pages, 2) - def test_pure_swa_radix_cache_when_chunked_prefill_disable_and_all_swa(self): - ctx = _make_ctx( - self, - effective_chunked_prefill_size=512, - disable_radix_cache=True, - is_hybrid_swa=True, - full_tokens_per_layer=0, - ) - with patch( - "sglang.srt.mem_cache.pure_swa_radix_cache.PureSWARadixCache" - ) as PureSWARadixCache: - PureSWARadixCache.return_value = MagicMock() - result = default_radix_cache_factory(ctx) - PureSWARadixCache.assert_called_once_with(params=ctx.params) - self.assertIs(result, PureSWARadixCache.return_value) - def test_pure_swa_with_disable_radix_skips_storage_backends(self): ctx = _make_ctx( self, disable_radix_cache=True, is_hybrid_swa=True, full_tokens_per_layer=0 diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 0a4be745f2..b962394f4c 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1137,7 +1137,7 @@ class TestLoadBalanceMethod(unittest.TestCase): handle_pd_disaggregation(server_args) self.assertIn("without improving prefill performance", "\n".join(logs.output)) - def test_pd_decode_dcp_forces_chunk_cache(self): + def test_pd_decode_dcp_disables_radix_cache(self): server_args = self._load_balance_args( disaggregation_mode="decode", disaggregation_transfer_backend="mooncake",