[PD] Fix crash for allocation when mamba + pd + decode radix cache (#41927)
This commit is contained in:
@@ -79,6 +79,8 @@ from sglang.srt.managers.schedule_batch import (
|
||||
)
|
||||
from sglang.srt.managers.schedule_policy import match_prefix_for_req
|
||||
from sglang.srt.managers.utils import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.allocation import ensure_mamba_capacity
|
||||
from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
@@ -291,14 +293,19 @@ class HybridMambaDecodeReqToTokenPool(HybridReqToTokenPool):
|
||||
pre_alloc_size=pre_alloc_size,
|
||||
)
|
||||
|
||||
self.mamba_ping_pong_track_buffer_size = 2 if enable_overlap_schedule else 1
|
||||
self.mamba_ping_pong_track_buffer_size = get_mamba_tracking_slots(
|
||||
extra_buffer=True, overlap=enable_overlap_schedule
|
||||
)
|
||||
self.mamba_initial_tracking_slots = get_mamba_tracking_slots(
|
||||
extra_buffer=enable_mamba_extra_buffer,
|
||||
overlap=enable_overlap_schedule,
|
||||
lazy=False,
|
||||
)
|
||||
self.enable_mamba_extra_buffer = enable_mamba_extra_buffer
|
||||
self.enable_memory_saver = enable_memory_saver
|
||||
# Each request needs 1 main mamba slot + ping-pong slots when extra_buffer is enabled.
|
||||
# Cap the pool at max concurrent requests * slots_per_req to avoid allocating failed.
|
||||
slots_per_req = 1 + (
|
||||
self.mamba_ping_pong_track_buffer_size if enable_mamba_extra_buffer else 0
|
||||
)
|
||||
slots_per_req = 1 + self.mamba_initial_tracking_slots
|
||||
max_slots_needed = (size + pre_alloc_size) * slots_per_req
|
||||
if mamba_size is not None:
|
||||
effective_mamba_size = max(mamba_size, max_slots_needed)
|
||||
@@ -965,6 +972,11 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if self.req_to_token_pool.available_size() <= 0:
|
||||
break
|
||||
|
||||
if not ensure_mamba_capacity(
|
||||
self.req_to_token_pool, [req], self.tree_cache
|
||||
):
|
||||
break
|
||||
|
||||
full_required, swa_required = self._prealloc_required_tokens(req)
|
||||
if not self._prealloc_reservation_fits(
|
||||
full_required,
|
||||
@@ -1382,16 +1394,12 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if self.req_to_token_pool.available_size() <= 0:
|
||||
break
|
||||
|
||||
# Hybrid models (e.g. K3 with KDA): guard against prealloc
|
||||
# draining the mamba pool before the KV pool (would assert "Not
|
||||
# enough space for mamba cache"). Evict a cached mamba slot from
|
||||
# the radix tree first (a no-op with the radix cache disabled),
|
||||
# else stop.
|
||||
mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None)
|
||||
if mamba_allocator is not None and mamba_allocator.available_size() <= 0:
|
||||
self.tree_cache.evict(EvictParams(num_tokens=0, mamba_num=1))
|
||||
if mamba_allocator.available_size() <= 0:
|
||||
break
|
||||
# Reserve enough Mamba capacity for radix COW and tracking buffers
|
||||
# before prefix matching can bind state or lock cached entries.
|
||||
if not ensure_mamba_capacity(
|
||||
self.req_to_token_pool, [decode_req.req], self.tree_cache
|
||||
):
|
||||
break
|
||||
|
||||
if hisparse_req_budget <= 0:
|
||||
break
|
||||
|
||||
@@ -855,7 +855,7 @@ class PrefillAdder:
|
||||
radix COW headroom or locked-but-evictable bytes — that residual is
|
||||
backstopped by the fail-loud RuntimeError in `alloc_req_slots`. FIXME: if
|
||||
over-admission crashes under pressure, make this more conservative (e.g.
|
||||
multiply by `MAMBA_STATE_PER_REQ_PREFIX_CACHE`)."""
|
||||
also account for missing tracking buffers)."""
|
||||
if self._mamba_slot_cost and not req.kv.holds_mamba:
|
||||
return self._mamba_slot_cost
|
||||
return 0
|
||||
|
||||
@@ -20,9 +20,6 @@ from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import (
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams
|
||||
from sglang.srt.mem_cache.common import (
|
||||
MAMBA_STATE_PER_REQ_NO_CACHE,
|
||||
MAMBA_STATE_PER_REQ_PREFIX_CACHE,
|
||||
MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY,
|
||||
available_and_evictable_str,
|
||||
evict_from_tree_cache,
|
||||
)
|
||||
@@ -286,6 +283,35 @@ def _kv_shard_rotation_bases(
|
||||
return bases
|
||||
|
||||
|
||||
def ensure_mamba_capacity(
|
||||
req_to_token_pool: ReqToTokenPool,
|
||||
reqs: list[Req],
|
||||
tree_cache: BasePrefixCache | None,
|
||||
) -> bool:
|
||||
"""Reclaim the Mamba slots needed before binding any request rows."""
|
||||
if not isinstance(req_to_token_pool, HybridReqToTokenPool):
|
||||
return True
|
||||
|
||||
# Count the slots HybridReqToTokenPool.alloc will allocate for each request.
|
||||
needed = 0
|
||||
for req in reqs:
|
||||
if not req.kv.holds_mamba:
|
||||
needed += 1
|
||||
|
||||
if req.kv.mamba_ping_pong_track_buffer is None:
|
||||
needed += req_to_token_pool.mamba_initial_tracking_slots
|
||||
|
||||
# Byte-coordinated for the shared allocator; plain free slots otherwise.
|
||||
allocator = req_to_token_pool.mamba_allocator
|
||||
supports_mamba = tree_cache is not None and tree_cache.supports_mamba()
|
||||
available = allocator.schedulable_available_size()
|
||||
if available < needed and supports_mamba:
|
||||
tree_cache.evict_for_alloc(
|
||||
EvictParams(num_tokens=0, mamba_num=needed - available)
|
||||
)
|
||||
return allocator.schedulable_available_size() >= needed
|
||||
|
||||
|
||||
def alloc_req_slots(
|
||||
req_to_token_pool: ReqToTokenPool,
|
||||
reqs: list[Req],
|
||||
@@ -298,28 +324,9 @@ def alloc_req_slots(
|
||||
and should surface rather than be masked.
|
||||
"""
|
||||
num_reqs = len(reqs)
|
||||
if isinstance(req_to_token_pool, HybridReqToTokenPool):
|
||||
# Byte-coordinated for the shared allocator (accounts for the peer full
|
||||
# sub-pool's bytes); plain slot free count for the non-shared one.
|
||||
mamba_available_size = (
|
||||
req_to_token_pool.mamba_allocator.schedulable_available_size()
|
||||
)
|
||||
# Eviction headroom factor: 3x (or lazy variant) for radix COW, 1x without prefix sharing.
|
||||
if tree_cache.supports_mamba() and tree_cache.supports_prefix_sharing():
|
||||
factor = (
|
||||
MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY
|
||||
if req_to_token_pool.enable_mamba_extra_buffer_lazy
|
||||
else MAMBA_STATE_PER_REQ_PREFIX_CACHE
|
||||
)
|
||||
else:
|
||||
factor = MAMBA_STATE_PER_REQ_NO_CACHE
|
||||
mamba_state_needed = num_reqs * factor
|
||||
if mamba_available_size < mamba_state_needed:
|
||||
if tree_cache is not None and tree_cache.supports_mamba():
|
||||
mamba_num = max(0, mamba_state_needed - mamba_available_size)
|
||||
tree_cache.evict_for_alloc(
|
||||
EvictParams(num_tokens=0, mamba_num=mamba_num)
|
||||
)
|
||||
# Keep combined serving's existing behavior: try cache eviction, then let
|
||||
# the pool allocate. A failed precheck need not mean allocation will fail.
|
||||
ensure_mamba_capacity(req_to_token_pool, reqs, tree_cache)
|
||||
req_pool_indices = req_to_token_pool.alloc(reqs)
|
||||
if req_pool_indices is None:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -8,6 +8,17 @@ from sglang.srt.runtime_context import (
|
||||
)
|
||||
|
||||
|
||||
def get_mamba_tracking_slots(
|
||||
*, extra_buffer: bool, overlap: bool, lazy: bool = False
|
||||
) -> int:
|
||||
"""Tracking slots allocated initially; lazy mode defers the second slot."""
|
||||
if not extra_buffer:
|
||||
return 0
|
||||
if lazy:
|
||||
return 1
|
||||
return 2 if overlap else 1
|
||||
|
||||
|
||||
def get_alloc_page_size() -> int:
|
||||
# Mirrors _build_token_to_kv_pool_allocator's DCP branch; the platform
|
||||
# allocators that skip it page smaller, so this is an upper bound for them.
|
||||
|
||||
@@ -23,12 +23,6 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
|
||||
# Needs 2 + 1 slots for mamba request with prefix cache. 2 for ping pong cache, 1 for running mamba state.
|
||||
MAMBA_STATE_PER_REQ_PREFIX_CACHE = 3
|
||||
# Lazy mode: 1 + 1 slots (1 ping-pong + 1 running), second ping-pong allocated on demand at boundary.
|
||||
MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY = 2
|
||||
MAMBA_STATE_PER_REQ_NO_CACHE = 1
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -34,7 +34,10 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
|
||||
get_kv_cache_quant_method,
|
||||
resolve_kv_cache_quant,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocation_sizing import get_req_to_token_extra_context_len
|
||||
from sglang.srt.mem_cache.allocation_sizing import (
|
||||
get_mamba_tracking_slots,
|
||||
get_req_to_token_extra_context_len,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator import (
|
||||
BaseTokenToKVPoolAllocator,
|
||||
PagedTokenToKVPoolAllocator,
|
||||
@@ -179,9 +182,6 @@ def mm_runtime_reservation_gb(
|
||||
# which the decode-time drop does not shrink.
|
||||
MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO = 3
|
||||
MAMBA_CACHE_BASE_RATIO_DROP_ON_SKIP = 1
|
||||
MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP = 2
|
||||
MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP_LAZY = 1
|
||||
MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP = 1
|
||||
MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_BUFFER = 1
|
||||
|
||||
|
||||
@@ -2268,18 +2268,15 @@ class KVCacheConfigurator:
|
||||
|
||||
additional_ratio = 0
|
||||
if get_exec().mamba.enable_mamba_extra_buffer:
|
||||
# ping-pong buffer size is 2 when overlap schedule is on, 1 otherwise.
|
||||
# Lazy mode saves 1 slot (2 → 1) for overlap; non-overlap already uses 1.
|
||||
if not get_schedule().disable_overlap_schedule:
|
||||
if get_exec().mamba.enable_mamba_extra_buffer_lazy:
|
||||
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP_LAZY
|
||||
else:
|
||||
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP
|
||||
else:
|
||||
if get_schedule().disable_overlap_schedule:
|
||||
assert not get_exec().mamba.enable_mamba_extra_buffer_lazy, (
|
||||
"Lazy extra buffer requires overlap schedule (--disable-overlap-schedule is incompatible)"
|
||||
)
|
||||
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP
|
||||
additional_ratio = get_mamba_tracking_slots(
|
||||
extra_buffer=True,
|
||||
overlap=not get_schedule().disable_overlap_schedule,
|
||||
lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
|
||||
)
|
||||
elif skip_decode_lock:
|
||||
# no_buffer under skip: add the base drop back so effective stays 3,
|
||||
# the prefill->decode peak needs ~3 slots/req and this leaf-only mode
|
||||
|
||||
@@ -59,6 +59,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
|
||||
UnquantizedKVCacheMethod,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots
|
||||
from sglang.srt.mem_cache.allocator.mamba import MambaSlotAllocator
|
||||
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
|
||||
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
|
||||
@@ -1368,7 +1369,14 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
enable_memory_saver=enable_memory_saver,
|
||||
)
|
||||
|
||||
self.mamba_ping_pong_track_buffer_size = 2 if enable_overlap_schedule else 1
|
||||
self.mamba_ping_pong_track_buffer_size = get_mamba_tracking_slots(
|
||||
extra_buffer=True, overlap=enable_overlap_schedule
|
||||
)
|
||||
self.mamba_initial_tracking_slots = get_mamba_tracking_slots(
|
||||
extra_buffer=enable_mamba_extra_buffer,
|
||||
overlap=enable_overlap_schedule,
|
||||
lazy=enable_mamba_extra_buffer_lazy,
|
||||
)
|
||||
self.enable_mamba_extra_buffer = enable_mamba_extra_buffer
|
||||
self.enable_mamba_extra_buffer_lazy = enable_mamba_extra_buffer_lazy
|
||||
self.enable_memory_saver = enable_memory_saver
|
||||
@@ -1722,11 +1730,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
Lazy mode allocates 1 slot with the second set to -1 (allocated
|
||||
on demand at track boundaries). Normal mode allocates all slots upfront.
|
||||
"""
|
||||
n = (
|
||||
1
|
||||
if self.enable_mamba_extra_buffer_lazy
|
||||
else self.mamba_ping_pong_track_buffer_size
|
||||
)
|
||||
n = self.mamba_initial_tracking_slots
|
||||
slots = self.mamba_allocator.alloc(n)
|
||||
assert slots is not None, (
|
||||
"Not enough space for mamba ping pong idx, "
|
||||
|
||||
Reference in New Issue
Block a user