[MemCache] Delete ChunkCache (#43201)
This commit is contained in:
@@ -130,7 +130,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
"handle_pd_disaggregation",
|
||||
disable_radix_cache=True,
|
||||
)
|
||||
logger.warning("KV cache is forced as chunk cache for decode server")
|
||||
logger.warning("Radix cache is disabled for decode server")
|
||||
|
||||
# Default the number of *extra* decode req_to_token slots reserved for
|
||||
# in-transfer (being-received-from-prefill) requests, on top of the
|
||||
|
||||
@@ -256,7 +256,7 @@ def teardown_disaggregation(scheduler: Scheduler) -> None:
|
||||
def _release_prefix_cache_for_role_switch(scheduler: Scheduler) -> None:
|
||||
"""Release the prefix (radix/hicache) cache so a flip works with radix ON.
|
||||
|
||||
With radix disabled (ChunkCache) the flip needs nothing here: ChunkCache
|
||||
With radix disabled the flip needs nothing here: the disabled cache
|
||||
keeps no persistent prefixes and, since the instance is idle before the
|
||||
switch, the allocator is already empty. This is the historical
|
||||
``--disable-radix-cache`` path, left untouched by the guard below.
|
||||
|
||||
@@ -1220,7 +1220,8 @@ class Req(ReqDllmMixin):
|
||||
# stamp new tree nodes. Allocation itself must NOT read it back when
|
||||
# a tree node is available (the checkpoint dedup rebind
|
||||
# would make it stale); the only allocation-time reader is the
|
||||
# ChunkCache fallback, which has no tree nodes and no rebind.
|
||||
# radix-disabled fallback, which keeps no nodes past the root and has
|
||||
# no rebind.
|
||||
self.kv_rotation_base: Optional[int] = None
|
||||
|
||||
# Whether or not if it is chunked. It increments whenever
|
||||
|
||||
@@ -38,8 +38,7 @@ to keep. The layout is specified in
|
||||
Two groups sit outside that stack:
|
||||
|
||||
- **Radix cache** is its own axis. The per-model variants (`radix_cache.py`,
|
||||
`chunk_cache.py`)
|
||||
are converging onto the **Unified Radix Cache** (`unified_cache/`,
|
||||
`pure_swa_radix_cache.py`) are converging onto the **Unified Radix Cache** (`unified_cache/`,
|
||||
[#20415](https://github.com/sgl-project/sglang/issues/20415)), whose Full/SWA/Mamba
|
||||
component model is documented in
|
||||
[`unified_cache/components/README.md`](unified_cache/components/README.md).
|
||||
|
||||
@@ -262,9 +262,10 @@ def _kv_shard_rotation_bases(
|
||||
allocator draws from the least-full class at that request's turn (so
|
||||
the draw sees earlier requests' pops in the same batch) and resolves
|
||||
the entry in place.
|
||||
- ChunkCache has no tree nodes (``last_node`` is None); its chunked
|
||||
continuations fall back to the base recorded on the request at the
|
||||
previous chunk's alloc (no cross-request reuse, no rebind there).
|
||||
- A radix-disabled cache keeps no nodes past the root, which carries no
|
||||
base; its chunked continuations fall back to the base recorded on the
|
||||
request at the previous chunk's alloc (no cross-request reuse, no
|
||||
rebind there).
|
||||
"""
|
||||
assert batch is not None and len(batch.reqs) == len(prefix_lens_cpu)
|
||||
bases = []
|
||||
|
||||
@@ -322,7 +322,7 @@ def zero_match_result(
|
||||
tree_cache, match_result: MatchResult, extra_key: Optional[str] = None
|
||||
) -> MatchResult:
|
||||
if not tree_cache.supports_prefix_sharing():
|
||||
# match_prefix already returns a miss; no root_node to walk back to.
|
||||
# A non-sharing cache's match is already a miss; pass it through.
|
||||
return match_result
|
||||
root = tree_cache.root_node_handle(extra_key=extra_key)
|
||||
return match_result._replace(
|
||||
@@ -467,7 +467,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
"""Logical-page KV sharding: the rotation base stamped on ``node``.
|
||||
|
||||
``node`` is whatever this cache stores in ``req.last_node`` (a NodeId
|
||||
for the unified tree, None for caches without tree nodes). None means
|
||||
for the unified tree). None means
|
||||
"no base available here", which sends the alloc path to the base the
|
||||
request recorded at its previous alloc. Tree caches that keep the
|
||||
per-chain base override this. See UnifiedTreeNode.rotation_base.
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""Cache for chunked prefill, used when RadixCache is disabled."""
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.allocator.hisparse import (
|
||||
DeepSeekV4HiSparseTokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
BasePrefixCache,
|
||||
DecLockRefParams,
|
||||
DecLockRefResult,
|
||||
EvictParams,
|
||||
EvictResult,
|
||||
IncLockRefResult,
|
||||
InsertParams,
|
||||
InsertResult,
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ChunkCache(BasePrefixCache):
|
||||
"""
|
||||
ChunkCache is used when radix cache is disabled.
|
||||
|
||||
That includes standard chunked-prefill setups and the decode side of P/D
|
||||
disaggregation when decode radix cache is not enabled.
|
||||
"""
|
||||
|
||||
def __init__(self, params: CacheInitParams):
|
||||
self.req_to_token_pool = params.req_to_token_pool
|
||||
self.token_to_kv_pool_allocator = params.token_to_kv_pool_allocator
|
||||
self.page_size = params.page_size
|
||||
if self.token_to_kv_pool_allocator:
|
||||
self.device = self.token_to_kv_pool_allocator.device
|
||||
else:
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
self.protected_size_ = 0
|
||||
|
||||
def supports_prefix_sharing(self) -> bool:
|
||||
return False
|
||||
|
||||
# NOTE (csy): this is to determine if a cache has prefix matching feature.
|
||||
# Chunk cache always return True to indicate no prefix matching.
|
||||
# TODO (csy): Using a prefix cache trait to replace this
|
||||
@property
|
||||
def disable(self):
|
||||
return True
|
||||
|
||||
def reset(self):
|
||||
pass
|
||||
|
||||
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
|
||||
return MatchResult(
|
||||
device_prefix_len=0,
|
||||
last_device_node=None,
|
||||
last_host_node=None,
|
||||
best_match_node=None,
|
||||
)
|
||||
|
||||
def path_device_indices(self, node: Any) -> torch.Tensor:
|
||||
return torch.empty((0,), dtype=torch.int64)
|
||||
|
||||
def insert(self, params: InsertParams) -> InsertResult:
|
||||
# ChunkCache does not support prefix caching, so insert is a no-op
|
||||
return InsertResult(prefix_len=0)
|
||||
|
||||
def checkpoint(self, req: Req, *, up_to: int):
|
||||
pass
|
||||
|
||||
def evict(self, params: EvictParams) -> EvictResult:
|
||||
return EvictResult()
|
||||
|
||||
def inc_lock_ref(self, node: Any) -> IncLockRefResult:
|
||||
return IncLockRefResult(delta=0)
|
||||
|
||||
def dec_lock_ref(
|
||||
self, node: Any, params: Optional[DecLockRefParams] = None
|
||||
) -> DecLockRefResult:
|
||||
return DecLockRefResult(delta=0)
|
||||
|
||||
def protected_size(self):
|
||||
# NOTE: no protected size in chunk cache. Chunk cache's eviction is the same with request's lifecycle.
|
||||
return 0
|
||||
|
||||
def pretty_print(self):
|
||||
return ""
|
||||
|
||||
|
||||
class SWAChunkCache(ChunkCache):
|
||||
"""ChunkCache with support for sliding window attention."""
|
||||
|
||||
def __init__(self, params: CacheInitParams):
|
||||
# DeepSeek V4 HiSparse wraps SWATokenToKVPoolAllocator and exposes the same API.
|
||||
assert isinstance(
|
||||
params.token_to_kv_pool_allocator,
|
||||
(
|
||||
SWATokenToKVPoolAllocator,
|
||||
DeepSeekV4HiSparseTokenToKVPoolAllocator,
|
||||
),
|
||||
)
|
||||
super().__init__(params)
|
||||
|
||||
self.sliding_window_size = params.sliding_window_size
|
||||
self.chunked_prefill_size = params.chunked_prefill_size
|
||||
|
||||
def supports_swa(self) -> bool:
|
||||
assert self.sliding_window_size is not None, (
|
||||
"sliding_window_size must be set for SWAChunkCache"
|
||||
)
|
||||
return True
|
||||
|
||||
def evict(self, params: EvictParams) -> EvictResult:
|
||||
return EvictResult()
|
||||
|
||||
|
||||
class PureSWAChunkCache(SWAChunkCache):
|
||||
"""ChunkCache for all-SWA models (no full attention layers): no
|
||||
full_to_swa_index_mapping, so free_kv_row must skip the window-evicted span."""
|
||||
@@ -367,7 +367,6 @@ def build_kv_cache(
|
||||
is_dsa=is_dsa,
|
||||
enable_hierarchical_cache=enable_hierarchical_cache,
|
||||
disable_radix_cache=disable_radix_cache,
|
||||
effective_chunked_prefill_size=effective_chunked_prefill_size,
|
||||
tp_worker=tp_worker,
|
||||
model_config=model_config,
|
||||
tp_size=parallel.tp_size,
|
||||
|
||||
@@ -37,7 +37,6 @@ class TreeCacheBuildContext:
|
||||
is_hybrid_ssm: bool
|
||||
enable_hierarchical_cache: bool
|
||||
disable_radix_cache: bool
|
||||
effective_chunked_prefill_size: Optional[int]
|
||||
tp_worker: Any
|
||||
model_config: ModelConfig
|
||||
tp_size: int
|
||||
@@ -82,19 +81,18 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
params = ctx.params
|
||||
|
||||
is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0
|
||||
# The decode host-pool backup needs UnifiedRadixCache; otherwise its
|
||||
# disabled mode serves every layout but pure-SWA.
|
||||
if ctx.disable_radix_cache and (
|
||||
get_disagg().disaggregation_decode_retraction_backup == "host_pool"
|
||||
or not is_pure_swa
|
||||
):
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
# A disabled cache publishes nothing, so no storage backend applies.
|
||||
if ctx.disable_radix_cache and is_pure_swa:
|
||||
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
|
||||
# Host-pool retraction goes to UnifiedRadixCache, which validates it (and
|
||||
# rejects pure-SWA).
|
||||
if ctx.disable_radix_cache:
|
||||
if (
|
||||
is_pure_swa
|
||||
and get_disagg().disaggregation_decode_retraction_backup != "host_pool"
|
||||
):
|
||||
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
|
||||
|
||||
return PureSWARadixCache(params=params)
|
||||
return PureSWARadixCache(params=params)
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
if get_memory().enable_lmcache:
|
||||
from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import (
|
||||
@@ -103,7 +101,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
|
||||
tree_components = []
|
||||
if not (ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0):
|
||||
if not is_pure_swa:
|
||||
tree_components.append(ComponentType.FULL)
|
||||
if ctx.is_hybrid_swa:
|
||||
tree_components.append(ComponentType.SWA)
|
||||
@@ -122,7 +120,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
if get_memory().enable_unified_cache_external_linker:
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
if ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0:
|
||||
if is_pure_swa:
|
||||
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
|
||||
|
||||
return PureSWARadixCache(params=params)
|
||||
|
||||
@@ -943,7 +943,8 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
|
||||
|
||||
@staticmethod
|
||||
def is_applicable(kvc: KVCacheConfigurator) -> bool:
|
||||
"""True when SWAChunkCache can be sized from explicit max requests."""
|
||||
"""True when a radix-disabled hybrid SWA cache can be sized from
|
||||
explicit max requests."""
|
||||
if get_schedule().max_running_requests is None:
|
||||
return False
|
||||
if not get_memory().disable_radix_cache:
|
||||
|
||||
Reference in New Issue
Block a user