[mem_cache] Run mamba models on UnifiedRadixCache when the radix cache is disabled (#42354)
This commit is contained in:
@@ -1391,13 +1391,11 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
# Hybrid models (e.g. K3 with KDA): guard against prealloc
|
||||
# draining the mamba pool before the KV pool (would assert "Not
|
||||
# enough space for mamba cache"). Evict a cached mamba slot from
|
||||
# the radix tree first (only if it manages mamba states;
|
||||
# ChunkCache.evict is a no-op), else stop.
|
||||
# the radix tree first (a no-op with the radix cache disabled),
|
||||
# else stop.
|
||||
mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None)
|
||||
if mamba_allocator is not None and mamba_allocator.available_size() <= 0:
|
||||
supports_mamba = self.tree_cache.supports_mamba()
|
||||
if supports_mamba and hasattr(self.tree_cache, "evict"):
|
||||
self.tree_cache.evict(EvictParams(num_tokens=0, mamba_num=1))
|
||||
self.tree_cache.evict(EvictParams(num_tokens=0, mamba_num=1))
|
||||
if mamba_allocator.available_size() <= 0:
|
||||
break
|
||||
|
||||
|
||||
@@ -216,21 +216,8 @@ class MlxAuxiliaryStatePool:
|
||||
|
||||
|
||||
class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
|
||||
"""Req-to-token pool with MLX auxiliary-state slot bookkeeping.
|
||||
|
||||
Auxiliary-slot release has exactly one owner per configuration:
|
||||
|
||||
* Radix cache enabled: the ``MlxAuxiliaryStateComponent`` of the unified
|
||||
radix cache owns release — on finish it either frees the slot or
|
||||
transfers it to the tree, nulling ``req.kv.mamba_pool_idx`` before the
|
||||
request row is freed. The pool must NOT free auxiliary slots itself.
|
||||
* Radix cache disabled (``ChunkCache``): no tree component exists, and
|
||||
``release_kv_cache``'s ``free_mamba_cache`` fallback is gated on
|
||||
``HybridReqToTokenPool``, which this pool is not — so the pool itself
|
||||
owns release. Construct with ``owns_auxiliary_state_release=True`` and
|
||||
``free(req)`` returns the slot together with the request row; without
|
||||
this, every finished request leaks its slot until allocation asserts.
|
||||
"""
|
||||
"""Req-to-token pool with MLX auxiliary-state slot bookkeeping. It never
|
||||
frees auxiliary slots: ``MlxAuxiliaryStateComponent`` owns their release."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -240,7 +227,6 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
|
||||
device: str,
|
||||
enable_memory_saver: bool,
|
||||
auxiliary_state_size: int,
|
||||
owns_auxiliary_state_release: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
size=size,
|
||||
@@ -248,7 +234,6 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
|
||||
device=device,
|
||||
enable_memory_saver=enable_memory_saver,
|
||||
)
|
||||
self._owns_auxiliary_state_release = owns_auxiliary_state_release
|
||||
self.mamba_pool = MlxAuxiliaryStatePool(
|
||||
size=auxiliary_state_size,
|
||||
device=device,
|
||||
@@ -310,16 +295,6 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
|
||||
mamba_ping_pong_track_buffer_to_keep=track_buffer_to_keep,
|
||||
)
|
||||
|
||||
def free(self, req):
|
||||
if self._owns_auxiliary_state_release:
|
||||
# No-radix configuration: nothing else will ever release the
|
||||
# auxiliary slot, so return it with the request row. Keyed on
|
||||
# req.kv.mamba_pool_idx (None-safe, nulled by free_mamba_cache), NOT
|
||||
# on req_index_to_auxiliary_state_index_mapping, which may point
|
||||
# at a slot the radix tree owns.
|
||||
self.free_mamba_cache(req)
|
||||
super().free(req)
|
||||
|
||||
def clear(self):
|
||||
super().clear()
|
||||
self.auxiliary_state_pool.clear()
|
||||
|
||||
@@ -296,10 +296,6 @@ class MlxModelRunnerStub(ModelRunner):
|
||||
device="cpu",
|
||||
enable_memory_saver=False,
|
||||
auxiliary_state_size=auxiliary_state_size,
|
||||
# With the radix cache disabled no tree component exists to
|
||||
# release auxiliary slots, so the pool owns their release
|
||||
# (see MlxAuxiliaryStateReqToTokenPool docstring).
|
||||
owns_auxiliary_state_release=get_memory().disable_radix_cache,
|
||||
)
|
||||
else:
|
||||
self.req_to_token_pool = ReqToTokenPool(
|
||||
|
||||
@@ -652,7 +652,9 @@ class PrefillAdder:
|
||||
self.exact_chunk_fill = (
|
||||
_use_exact_chunk_fill()
|
||||
and dllm_config is None
|
||||
and not tree_cache.supports_mamba()
|
||||
and not (
|
||||
tree_cache.supports_mamba() and tree_cache.supports_prefix_sharing()
|
||||
)
|
||||
)
|
||||
|
||||
if self.dllm_config is not None:
|
||||
@@ -695,8 +697,7 @@ class PrefillAdder:
|
||||
self.is_hybrid_ssm_cache = self.tree_cache.supports_mamba()
|
||||
# A new state slot eats shared-gap bytes that `rem_total_tokens` counts
|
||||
# as free, so reserve per slot or admission over-commits. Gate on the
|
||||
# ALLOCATOR, not `is_hybrid_ssm_cache`: that is False for `ChunkCache`,
|
||||
# which would skip the reservation on the chunk-cache path.
|
||||
# ALLOCATOR: the reservation holds with the radix cache disabled too.
|
||||
self._mamba_slot_cost = 0
|
||||
if isinstance(
|
||||
self.token_to_kv_pool_allocator,
|
||||
|
||||
@@ -101,15 +101,9 @@ class SchedulerInvariantChecker:
|
||||
session_held = self.pool_stats_observer.session_held_full_tokens()
|
||||
total = ps.full_capacity
|
||||
elif self.is_hybrid_ssm:
|
||||
# Branch on cache type for the protected accessor (a mamba-capable
|
||||
# cache splits full/mamba; ChunkCache only has the single protected_size).
|
||||
# Use the allocator's `.size` for `total`: static max_total_num_tokens for
|
||||
# non-unified pools, the dynamic byte-coordinated cap (matching
|
||||
# `available_size`) for the unified pool.
|
||||
if self.tree_cache.supports_mamba():
|
||||
protected = self.tree_cache.full_protected_size()
|
||||
else:
|
||||
protected = self.tree_cache.protected_size()
|
||||
# `total` is the allocator's `.size`: static for non-unified pools,
|
||||
# the byte-coordinated cap (matching `available_size`) for the unified pool.
|
||||
protected = self.tree_cache.full_protected_size()
|
||||
session_held = self.pool_stats_observer.session_held_tokens()
|
||||
total = self.req_to_token_pool.schedulable_token_capacity(
|
||||
self.token_to_kv_pool_allocator.size
|
||||
|
||||
@@ -304,8 +304,8 @@ def alloc_req_slots(
|
||||
mamba_available_size = (
|
||||
req_to_token_pool.mamba_allocator.schedulable_available_size()
|
||||
)
|
||||
# Eviction headroom factor: 3x (or lazy variant) for radix COW, 1x for chunk.
|
||||
if tree_cache.supports_mamba():
|
||||
# Eviction headroom factor: 3x (or lazy variant) for radix COW, 1x without prefix sharing.
|
||||
if tree_cache.supports_mamba() and tree_cache.supports_prefix_sharing():
|
||||
factor = (
|
||||
MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY
|
||||
if req_to_token_pool.enable_mamba_extra_buffer_lazy
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.kernels.ops.memory.common import get_last_loc_kernel as get_last_loc
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolTransfer
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_serving, get_spec
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
@@ -334,14 +334,6 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
|
||||
)
|
||||
tree_cache.on_release(req, inserted=is_insert)
|
||||
|
||||
# If the prefix cache doesn't manage mamba states, we must free them here.
|
||||
if isinstance(tree_cache.req_to_token_pool, HybridReqToTokenPool) and (
|
||||
not tree_cache.supports_mamba()
|
||||
):
|
||||
assert req.kv.holds_mamba, (
|
||||
"mamba state is freed while the tree cache does not manage mamba states"
|
||||
)
|
||||
tree_cache.req_to_token_pool.free_mamba_cache(req)
|
||||
# The DSV4-NPU ReqToTokenPool subclass's free() additionally releases the
|
||||
# c4/c128 state pages; other ReqToTokenPool subclasses are a no-op here.
|
||||
tree_cache.req_to_token_pool.free(req)
|
||||
|
||||
@@ -84,9 +84,12 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0
|
||||
if ctx.disable_radix_cache and (
|
||||
get_disagg().disaggregation_decode_retraction_backup == "host_pool"
|
||||
# Streaming sessions live in UnifiedRadixCache; its disabled mode
|
||||
# stands in for the chunk caches. Pure-SWA has no unified layout.
|
||||
or (get_serving().enable_streaming_session and not is_pure_swa)
|
||||
# Streaming sessions and mamba states need UnifiedRadixCache, whose
|
||||
# disabled mode replaces the chunk caches; pure-SWA has no unified layout.
|
||||
or (
|
||||
not is_pure_swa
|
||||
and (get_serving().enable_streaming_session or ctx.is_hybrid_ssm)
|
||||
)
|
||||
):
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
@@ -292,6 +295,13 @@ def create_tree_cache(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
"a PR at https://github.com/sgl-project/sglang if you need this."
|
||||
)
|
||||
|
||||
if ctx.is_hybrid_ssm and not cache.supports_mamba():
|
||||
raise NotImplementedError(
|
||||
f"Models with mamba state are not verified with {type(cache).__name__}; "
|
||||
"mamba state lives in UnifiedRadixCache. Please open an issue or a PR "
|
||||
"at https://github.com/sgl-project/sglang if you need this."
|
||||
)
|
||||
|
||||
hicache_attached = cache.cache_controller is not None
|
||||
logger.info(
|
||||
"Tree cache initialized: source=%s impl=%s hybrid_swa=%s hybrid_ssm=%s "
|
||||
|
||||
@@ -67,7 +67,9 @@ class MambaComponent(TreeComponent):
|
||||
assert isinstance(params.req_to_token_pool, HybridReqToTokenPool), (
|
||||
f"MambaComponent requires HybridReqToTokenPool, got {type(params.req_to_token_pool)}"
|
||||
)
|
||||
if not params.enable_mamba_extra_buffer:
|
||||
# Without the extra buffer only the sequence-end state exists, so a cached
|
||||
# state needs page 1; a disabled tree caches none.
|
||||
if not params.enable_mamba_extra_buffer and not params.disable:
|
||||
assert params.page_size == 1, (
|
||||
f"MambaComponent requires page_size=1 when mamba_extra_buffer is disabled, got {params.page_size}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user