[PD] Fix crash for allocation when mamba + pd + decode radix cache (#41927)

This commit is contained in:
Byron Hsu
2026-10-06 09:36:31 -07:00
committed by GitHub
parent 209e190ee5
commit abed685de1
8 changed files with 368 additions and 65 deletions
+22 -14
View File
@@ -79,6 +79,8 @@ from sglang.srt.managers.schedule_batch import (
)
from sglang.srt.managers.schedule_policy import match_prefix_for_req
from sglang.srt.managers.utils import GenerationBatchResult
from sglang.srt.mem_cache.allocation import ensure_mamba_capacity
from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import (
@@ -291,14 +293,19 @@ class HybridMambaDecodeReqToTokenPool(HybridReqToTokenPool):
pre_alloc_size=pre_alloc_size,
)
self.mamba_ping_pong_track_buffer_size = 2 if enable_overlap_schedule else 1
self.mamba_ping_pong_track_buffer_size = get_mamba_tracking_slots(
extra_buffer=True, overlap=enable_overlap_schedule
)
self.mamba_initial_tracking_slots = get_mamba_tracking_slots(
extra_buffer=enable_mamba_extra_buffer,
overlap=enable_overlap_schedule,
lazy=False,
)
self.enable_mamba_extra_buffer = enable_mamba_extra_buffer
self.enable_memory_saver = enable_memory_saver
# Each request needs 1 main mamba slot + ping-pong slots when extra_buffer is enabled.
# Cap the pool at max concurrent requests * slots_per_req to avoid allocating failed.
slots_per_req = 1 + (
self.mamba_ping_pong_track_buffer_size if enable_mamba_extra_buffer else 0
)
slots_per_req = 1 + self.mamba_initial_tracking_slots
max_slots_needed = (size + pre_alloc_size) * slots_per_req
if mamba_size is not None:
effective_mamba_size = max(mamba_size, max_slots_needed)
@@ -965,6 +972,11 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if self.req_to_token_pool.available_size() <= 0:
break
if not ensure_mamba_capacity(
self.req_to_token_pool, [req], self.tree_cache
):
break
full_required, swa_required = self._prealloc_required_tokens(req)
if not self._prealloc_reservation_fits(
full_required,
@@ -1382,16 +1394,12 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if self.req_to_token_pool.available_size() <= 0:
break
# Hybrid models (e.g. K3 with KDA): guard against prealloc
# draining the mamba pool before the KV pool (would assert "Not
# enough space for mamba cache"). Evict a cached mamba slot from
# the radix tree first (a no-op with the radix cache disabled),
# else stop.
mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None)
if mamba_allocator is not None and mamba_allocator.available_size() <= 0:
self.tree_cache.evict(EvictParams(num_tokens=0, mamba_num=1))
if mamba_allocator.available_size() <= 0:
break
# Reserve enough Mamba capacity for radix COW and tracking buffers
# before prefix matching can bind state or lock cached entries.
if not ensure_mamba_capacity(
self.req_to_token_pool, [decode_req.req], self.tree_cache
):
break
if hisparse_req_budget <= 0:
break
@@ -855,7 +855,7 @@ class PrefillAdder:
radix COW headroom or locked-but-evictable bytes — that residual is
backstopped by the fail-loud RuntimeError in `alloc_req_slots`. FIXME: if
over-admission crashes under pressure, make this more conservative (e.g.
multiply by `MAMBA_STATE_PER_REQ_PREFIX_CACHE`)."""
also account for missing tracking buffers)."""
if self._mamba_slot_cost and not req.kv.holds_mamba:
return self._mamba_slot_cost
return 0
+32 -25
View File
@@ -20,9 +20,6 @@ from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import (
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams
from sglang.srt.mem_cache.common import (
MAMBA_STATE_PER_REQ_NO_CACHE,
MAMBA_STATE_PER_REQ_PREFIX_CACHE,
MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY,
available_and_evictable_str,
evict_from_tree_cache,
)
@@ -286,6 +283,35 @@ def _kv_shard_rotation_bases(
return bases
def ensure_mamba_capacity(
req_to_token_pool: ReqToTokenPool,
reqs: list[Req],
tree_cache: BasePrefixCache | None,
) -> bool:
"""Reclaim the Mamba slots needed before binding any request rows."""
if not isinstance(req_to_token_pool, HybridReqToTokenPool):
return True
# Count the slots HybridReqToTokenPool.alloc will allocate for each request.
needed = 0
for req in reqs:
if not req.kv.holds_mamba:
needed += 1
if req.kv.mamba_ping_pong_track_buffer is None:
needed += req_to_token_pool.mamba_initial_tracking_slots
# Byte-coordinated for the shared allocator; plain free slots otherwise.
allocator = req_to_token_pool.mamba_allocator
supports_mamba = tree_cache is not None and tree_cache.supports_mamba()
available = allocator.schedulable_available_size()
if available < needed and supports_mamba:
tree_cache.evict_for_alloc(
EvictParams(num_tokens=0, mamba_num=needed - available)
)
return allocator.schedulable_available_size() >= needed
def alloc_req_slots(
req_to_token_pool: ReqToTokenPool,
reqs: list[Req],
@@ -298,28 +324,9 @@ def alloc_req_slots(
and should surface rather than be masked.
"""
num_reqs = len(reqs)
if isinstance(req_to_token_pool, HybridReqToTokenPool):
# Byte-coordinated for the shared allocator (accounts for the peer full
# sub-pool's bytes); plain slot free count for the non-shared one.
mamba_available_size = (
req_to_token_pool.mamba_allocator.schedulable_available_size()
)
# Eviction headroom factor: 3x (or lazy variant) for radix COW, 1x without prefix sharing.
if tree_cache.supports_mamba() and tree_cache.supports_prefix_sharing():
factor = (
MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY
if req_to_token_pool.enable_mamba_extra_buffer_lazy
else MAMBA_STATE_PER_REQ_PREFIX_CACHE
)
else:
factor = MAMBA_STATE_PER_REQ_NO_CACHE
mamba_state_needed = num_reqs * factor
if mamba_available_size < mamba_state_needed:
if tree_cache is not None and tree_cache.supports_mamba():
mamba_num = max(0, mamba_state_needed - mamba_available_size)
tree_cache.evict_for_alloc(
EvictParams(num_tokens=0, mamba_num=mamba_num)
)
# Keep combined serving's existing behavior: try cache eviction, then let
# the pool allocate. A failed precheck need not mean allocation will fail.
ensure_mamba_capacity(req_to_token_pool, reqs, tree_cache)
req_pool_indices = req_to_token_pool.alloc(reqs)
if req_pool_indices is None:
raise RuntimeError(
@@ -8,6 +8,17 @@ from sglang.srt.runtime_context import (
)
def get_mamba_tracking_slots(
*, extra_buffer: bool, overlap: bool, lazy: bool = False
) -> int:
"""Tracking slots allocated initially; lazy mode defers the second slot."""
if not extra_buffer:
return 0
if lazy:
return 1
return 2 if overlap else 1
def get_alloc_page_size() -> int:
# Mirrors _build_token_to_kv_pool_allocator's DCP branch; the platform
# allocators that skip it page smaller, so this is an upper bound for them.
-6
View File
@@ -23,12 +23,6 @@ if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
# Needs 2 + 1 slots for mamba request with prefix cache. 2 for ping pong cache, 1 for running mamba state.
MAMBA_STATE_PER_REQ_PREFIX_CACHE = 3
# Lazy mode: 1 + 1 slots (1 ping-pong + 1 running), second ping-pong allocated on demand at boundary.
MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY = 2
MAMBA_STATE_PER_REQ_NO_CACHE = 1
logger = logging.getLogger(__name__)
@@ -34,7 +34,10 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
get_kv_cache_quant_method,
resolve_kv_cache_quant,
)
from sglang.srt.mem_cache.allocation_sizing import get_req_to_token_extra_context_len
from sglang.srt.mem_cache.allocation_sizing import (
get_mamba_tracking_slots,
get_req_to_token_extra_context_len,
)
from sglang.srt.mem_cache.allocator import (
BaseTokenToKVPoolAllocator,
PagedTokenToKVPoolAllocator,
@@ -179,9 +182,6 @@ def mm_runtime_reservation_gb(
# which the decode-time drop does not shrink.
MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO = 3
MAMBA_CACHE_BASE_RATIO_DROP_ON_SKIP = 1
MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP = 2
MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP_LAZY = 1
MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP = 1
MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_BUFFER = 1
@@ -2268,18 +2268,15 @@ class KVCacheConfigurator:
additional_ratio = 0
if get_exec().mamba.enable_mamba_extra_buffer:
# ping-pong buffer size is 2 when overlap schedule is on, 1 otherwise.
# Lazy mode saves 1 slot (2 → 1) for overlap; non-overlap already uses 1.
if not get_schedule().disable_overlap_schedule:
if get_exec().mamba.enable_mamba_extra_buffer_lazy:
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP_LAZY
else:
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP
else:
if get_schedule().disable_overlap_schedule:
assert not get_exec().mamba.enable_mamba_extra_buffer_lazy, (
"Lazy extra buffer requires overlap schedule (--disable-overlap-schedule is incompatible)"
)
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP
additional_ratio = get_mamba_tracking_slots(
extra_buffer=True,
overlap=not get_schedule().disable_overlap_schedule,
lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
)
elif skip_decode_lock:
# no_buffer under skip: add the base drop back so effective stays 3,
# the prefill->decode peak needs ~3 slots/req and this leaf-only mode
+10 -6
View File
@@ -59,6 +59,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
UnquantizedKVCacheMethod,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots
from sglang.srt.mem_cache.allocator.mamba import MambaSlotAllocator
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
@@ -1368,7 +1369,14 @@ class HybridReqToTokenPool(ReqToTokenPool):
enable_memory_saver=enable_memory_saver,
)
self.mamba_ping_pong_track_buffer_size = 2 if enable_overlap_schedule else 1
self.mamba_ping_pong_track_buffer_size = get_mamba_tracking_slots(
extra_buffer=True, overlap=enable_overlap_schedule
)
self.mamba_initial_tracking_slots = get_mamba_tracking_slots(
extra_buffer=enable_mamba_extra_buffer,
overlap=enable_overlap_schedule,
lazy=enable_mamba_extra_buffer_lazy,
)
self.enable_mamba_extra_buffer = enable_mamba_extra_buffer
self.enable_mamba_extra_buffer_lazy = enable_mamba_extra_buffer_lazy
self.enable_memory_saver = enable_memory_saver
@@ -1722,11 +1730,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
Lazy mode allocates 1 slot with the second set to -1 (allocated
on demand at track boundaries). Normal mode allocates all slots upfront.
"""
n = (
1
if self.enable_mamba_extra_buffer_lazy
else self.mamba_ping_pong_track_buffer_size
)
n = self.mamba_initial_tracking_slots
slots = self.mamba_allocator.alloc(n)
assert slots is not None, (
"Not enough space for mamba ping pong idx, "