[MemCache] Delete ChunkCache (#43201)

This commit is contained in:
Liangsheng Yin
2026-10-08 17:31:16 -07:00
committed by GitHub
parent 508ba79688
commit b4fe282836
23 changed files with 57 additions and 303 deletions
@@ -130,7 +130,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
"handle_pd_disaggregation",
disable_radix_cache=True,
)
logger.warning("KV cache is forced as chunk cache for decode server")
logger.warning("Radix cache is disabled for decode server")
# Default the number of *extra* decode req_to_token slots reserved for
# in-transfer (being-received-from-prefill) requests, on top of the
@@ -256,7 +256,7 @@ def teardown_disaggregation(scheduler: Scheduler) -> None:
def _release_prefix_cache_for_role_switch(scheduler: Scheduler) -> None:
"""Release the prefix (radix/hicache) cache so a flip works with radix ON.
With radix disabled (ChunkCache) the flip needs nothing here: ChunkCache
With radix disabled the flip needs nothing here: the disabled cache
keeps no persistent prefixes and, since the instance is idle before the
switch, the allocator is already empty. This is the historical
``--disable-radix-cache`` path, left untouched by the guard below.
+2 -1
View File
@@ -1220,7 +1220,8 @@ class Req(ReqDllmMixin):
# stamp new tree nodes. Allocation itself must NOT read it back when
# a tree node is available (the checkpoint dedup rebind
# would make it stale); the only allocation-time reader is the
# ChunkCache fallback, which has no tree nodes and no rebind.
# radix-disabled fallback, which keeps no nodes past the root and has
# no rebind.
self.kv_rotation_base: Optional[int] = None
# Whether or not if it is chunked. It increments whenever
+1 -2
View File
@@ -38,8 +38,7 @@ to keep. The layout is specified in
Two groups sit outside that stack:
- **Radix cache** is its own axis. The per-model variants (`radix_cache.py`,
`chunk_cache.py`)
are converging onto the **Unified Radix Cache** (`unified_cache/`,
`pure_swa_radix_cache.py`) are converging onto the **Unified Radix Cache** (`unified_cache/`,
[#20415](https://github.com/sgl-project/sglang/issues/20415)), whose Full/SWA/Mamba
component model is documented in
[`unified_cache/components/README.md`](unified_cache/components/README.md).
+4 -3
View File
@@ -262,9 +262,10 @@ def _kv_shard_rotation_bases(
allocator draws from the least-full class at that request's turn (so
the draw sees earlier requests' pops in the same batch) and resolves
the entry in place.
- ChunkCache has no tree nodes (``last_node`` is None); its chunked
continuations fall back to the base recorded on the request at the
previous chunk's alloc (no cross-request reuse, no rebind there).
- A radix-disabled cache keeps no nodes past the root, which carries no
base; its chunked continuations fall back to the base recorded on the
request at the previous chunk's alloc (no cross-request reuse, no
rebind there).
"""
assert batch is not None and len(batch.reqs) == len(prefix_lens_cpu)
bases = []
@@ -322,7 +322,7 @@ def zero_match_result(
tree_cache, match_result: MatchResult, extra_key: Optional[str] = None
) -> MatchResult:
if not tree_cache.supports_prefix_sharing():
# match_prefix already returns a miss; no root_node to walk back to.
# A non-sharing cache's match is already a miss; pass it through.
return match_result
root = tree_cache.root_node_handle(extra_key=extra_key)
return match_result._replace(
@@ -467,7 +467,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
"""Logical-page KV sharding: the rotation base stamped on ``node``.
``node`` is whatever this cache stores in ``req.last_node`` (a NodeId
for the unified tree, None for caches without tree nodes). None means
for the unified tree). None means
"no base available here", which sends the alloc path to the base the
request recorded at its previous alloc. Tree caches that keep the
per-chain base override this. See UnifiedTreeNode.rotation_base.
-133
View File
@@ -1,133 +0,0 @@
from __future__ import annotations
"""Cache for chunked prefill, used when RadixCache is disabled."""
import logging
from typing import TYPE_CHECKING, Any, Optional
import torch
from sglang.srt.mem_cache.allocator.hisparse import (
DeepSeekV4HiSparseTokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
DecLockRefParams,
DecLockRefResult,
EvictParams,
EvictResult,
IncLockRefResult,
InsertParams,
InsertResult,
MatchPrefixParams,
MatchResult,
)
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
logger = logging.getLogger(__name__)
class ChunkCache(BasePrefixCache):
"""
ChunkCache is used when radix cache is disabled.
That includes standard chunked-prefill setups and the decode side of P/D
disaggregation when decode radix cache is not enabled.
"""
def __init__(self, params: CacheInitParams):
self.req_to_token_pool = params.req_to_token_pool
self.token_to_kv_pool_allocator = params.token_to_kv_pool_allocator
self.page_size = params.page_size
if self.token_to_kv_pool_allocator:
self.device = self.token_to_kv_pool_allocator.device
else:
self.device = torch.device("cpu")
self.protected_size_ = 0
def supports_prefix_sharing(self) -> bool:
return False
# NOTE (csy): this is to determine if a cache has prefix matching feature.
# Chunk cache always return True to indicate no prefix matching.
# TODO (csy): Using a prefix cache trait to replace this
@property
def disable(self):
return True
def reset(self):
pass
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
return MatchResult(
device_prefix_len=0,
last_device_node=None,
last_host_node=None,
best_match_node=None,
)
def path_device_indices(self, node: Any) -> torch.Tensor:
return torch.empty((0,), dtype=torch.int64)
def insert(self, params: InsertParams) -> InsertResult:
# ChunkCache does not support prefix caching, so insert is a no-op
return InsertResult(prefix_len=0)
def checkpoint(self, req: Req, *, up_to: int):
pass
def evict(self, params: EvictParams) -> EvictResult:
return EvictResult()
def inc_lock_ref(self, node: Any) -> IncLockRefResult:
return IncLockRefResult(delta=0)
def dec_lock_ref(
self, node: Any, params: Optional[DecLockRefParams] = None
) -> DecLockRefResult:
return DecLockRefResult(delta=0)
def protected_size(self):
# NOTE: no protected size in chunk cache. Chunk cache's eviction is the same with request's lifecycle.
return 0
def pretty_print(self):
return ""
class SWAChunkCache(ChunkCache):
"""ChunkCache with support for sliding window attention."""
def __init__(self, params: CacheInitParams):
# DeepSeek V4 HiSparse wraps SWATokenToKVPoolAllocator and exposes the same API.
assert isinstance(
params.token_to_kv_pool_allocator,
(
SWATokenToKVPoolAllocator,
DeepSeekV4HiSparseTokenToKVPoolAllocator,
),
)
super().__init__(params)
self.sliding_window_size = params.sliding_window_size
self.chunked_prefill_size = params.chunked_prefill_size
def supports_swa(self) -> bool:
assert self.sliding_window_size is not None, (
"sliding_window_size must be set for SWAChunkCache"
)
return True
def evict(self, params: EvictParams) -> EvictResult:
return EvictResult()
class PureSWAChunkCache(SWAChunkCache):
"""ChunkCache for all-SWA models (no full attention layers): no
full_to_swa_index_mapping, so free_kv_row must skip the window-evicted span."""
@@ -367,7 +367,6 @@ def build_kv_cache(
is_dsa=is_dsa,
enable_hierarchical_cache=enable_hierarchical_cache,
disable_radix_cache=disable_radix_cache,
effective_chunked_prefill_size=effective_chunked_prefill_size,
tp_worker=tp_worker,
model_config=model_config,
tp_size=parallel.tp_size,
+12 -14
View File
@@ -37,7 +37,6 @@ class TreeCacheBuildContext:
is_hybrid_ssm: bool
enable_hierarchical_cache: bool
disable_radix_cache: bool
effective_chunked_prefill_size: Optional[int]
tp_worker: Any
model_config: ModelConfig
tp_size: int
@@ -82,19 +81,18 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
params = ctx.params
is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0
# The decode host-pool backup needs UnifiedRadixCache; otherwise its
# disabled mode serves every layout but pure-SWA.
if ctx.disable_radix_cache and (
get_disagg().disaggregation_decode_retraction_backup == "host_pool"
or not is_pure_swa
):
return create_unified_radix_cache(ctx)
# A disabled cache publishes nothing, so no storage backend applies.
if ctx.disable_radix_cache and is_pure_swa:
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
# Host-pool retraction goes to UnifiedRadixCache, which validates it (and
# rejects pure-SWA).
if ctx.disable_radix_cache:
if (
is_pure_swa
and get_disagg().disaggregation_decode_retraction_backup != "host_pool"
):
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
return PureSWARadixCache(params=params)
return PureSWARadixCache(params=params)
return create_unified_radix_cache(ctx)
if get_memory().enable_lmcache:
from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import (
@@ -103,7 +101,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
from sglang.srt.mem_cache.unified_cache.components import ComponentType
tree_components = []
if not (ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0):
if not is_pure_swa:
tree_components.append(ComponentType.FULL)
if ctx.is_hybrid_swa:
tree_components.append(ComponentType.SWA)
@@ -122,7 +120,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
if get_memory().enable_unified_cache_external_linker:
return create_unified_radix_cache(ctx)
if ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0:
if is_pure_swa:
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
return PureSWARadixCache(params=params)
@@ -943,7 +943,8 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
@staticmethod
def is_applicable(kvc: KVCacheConfigurator) -> bool:
"""True when SWAChunkCache can be sized from explicit max requests."""
"""True when a radix-disabled hybrid SWA cache can be sized from
explicit max requests."""
if get_schedule().max_running_requests is None:
return False
if not get_memory().disable_radix_cache: