diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index bc732bca26..c4cf6ec4f5 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -79,6 +79,8 @@ from sglang.srt.managers.schedule_batch import ( ) from sglang.srt.managers.schedule_policy import match_prefix_for_req from sglang.srt.managers.utils import GenerationBatchResult +from sglang.srt.mem_cache.allocation import ensure_mamba_capacity +from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring from sglang.srt.mem_cache.base_prefix_cache import ( @@ -291,14 +293,19 @@ class HybridMambaDecodeReqToTokenPool(HybridReqToTokenPool): pre_alloc_size=pre_alloc_size, ) - self.mamba_ping_pong_track_buffer_size = 2 if enable_overlap_schedule else 1 + self.mamba_ping_pong_track_buffer_size = get_mamba_tracking_slots( + extra_buffer=True, overlap=enable_overlap_schedule + ) + self.mamba_initial_tracking_slots = get_mamba_tracking_slots( + extra_buffer=enable_mamba_extra_buffer, + overlap=enable_overlap_schedule, + lazy=False, + ) self.enable_mamba_extra_buffer = enable_mamba_extra_buffer self.enable_memory_saver = enable_memory_saver # Each request needs 1 main mamba slot + ping-pong slots when extra_buffer is enabled. # Cap the pool at max concurrent requests * slots_per_req to avoid allocating failed. - slots_per_req = 1 + ( - self.mamba_ping_pong_track_buffer_size if enable_mamba_extra_buffer else 0 - ) + slots_per_req = 1 + self.mamba_initial_tracking_slots max_slots_needed = (size + pre_alloc_size) * slots_per_req if mamba_size is not None: effective_mamba_size = max(mamba_size, max_slots_needed) @@ -965,6 +972,11 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): if self.req_to_token_pool.available_size() <= 0: break + if not ensure_mamba_capacity( + self.req_to_token_pool, [req], self.tree_cache + ): + break + full_required, swa_required = self._prealloc_required_tokens(req) if not self._prealloc_reservation_fits( full_required, @@ -1382,16 +1394,12 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): if self.req_to_token_pool.available_size() <= 0: break - # Hybrid models (e.g. K3 with KDA): guard against prealloc - # draining the mamba pool before the KV pool (would assert "Not - # enough space for mamba cache"). Evict a cached mamba slot from - # the radix tree first (a no-op with the radix cache disabled), - # else stop. - mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None) - if mamba_allocator is not None and mamba_allocator.available_size() <= 0: - self.tree_cache.evict(EvictParams(num_tokens=0, mamba_num=1)) - if mamba_allocator.available_size() <= 0: - break + # Reserve enough Mamba capacity for radix COW and tracking buffers + # before prefix matching can bind state or lock cached entries. + if not ensure_mamba_capacity( + self.req_to_token_pool, [decode_req.req], self.tree_cache + ): + break if hisparse_req_budget <= 0: break diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index db470c8335..ada0d210b7 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -855,7 +855,7 @@ class PrefillAdder: radix COW headroom or locked-but-evictable bytes — that residual is backstopped by the fail-loud RuntimeError in `alloc_req_slots`. FIXME: if over-admission crashes under pressure, make this more conservative (e.g. - multiply by `MAMBA_STATE_PER_REQ_PREFIX_CACHE`).""" + also account for missing tracking buffers).""" if self._mamba_slot_cost and not req.kv.holds_mamba: return self._mamba_slot_cost return 0 diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index a6db9227cd..b546b526f5 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -20,9 +20,6 @@ from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import ( from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams from sglang.srt.mem_cache.common import ( - MAMBA_STATE_PER_REQ_NO_CACHE, - MAMBA_STATE_PER_REQ_PREFIX_CACHE, - MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY, available_and_evictable_str, evict_from_tree_cache, ) @@ -286,6 +283,35 @@ def _kv_shard_rotation_bases( return bases +def ensure_mamba_capacity( + req_to_token_pool: ReqToTokenPool, + reqs: list[Req], + tree_cache: BasePrefixCache | None, +) -> bool: + """Reclaim the Mamba slots needed before binding any request rows.""" + if not isinstance(req_to_token_pool, HybridReqToTokenPool): + return True + + # Count the slots HybridReqToTokenPool.alloc will allocate for each request. + needed = 0 + for req in reqs: + if not req.kv.holds_mamba: + needed += 1 + + if req.kv.mamba_ping_pong_track_buffer is None: + needed += req_to_token_pool.mamba_initial_tracking_slots + + # Byte-coordinated for the shared allocator; plain free slots otherwise. + allocator = req_to_token_pool.mamba_allocator + supports_mamba = tree_cache is not None and tree_cache.supports_mamba() + available = allocator.schedulable_available_size() + if available < needed and supports_mamba: + tree_cache.evict_for_alloc( + EvictParams(num_tokens=0, mamba_num=needed - available) + ) + return allocator.schedulable_available_size() >= needed + + def alloc_req_slots( req_to_token_pool: ReqToTokenPool, reqs: list[Req], @@ -298,28 +324,9 @@ def alloc_req_slots( and should surface rather than be masked. """ num_reqs = len(reqs) - if isinstance(req_to_token_pool, HybridReqToTokenPool): - # Byte-coordinated for the shared allocator (accounts for the peer full - # sub-pool's bytes); plain slot free count for the non-shared one. - mamba_available_size = ( - req_to_token_pool.mamba_allocator.schedulable_available_size() - ) - # Eviction headroom factor: 3x (or lazy variant) for radix COW, 1x without prefix sharing. - if tree_cache.supports_mamba() and tree_cache.supports_prefix_sharing(): - factor = ( - MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY - if req_to_token_pool.enable_mamba_extra_buffer_lazy - else MAMBA_STATE_PER_REQ_PREFIX_CACHE - ) - else: - factor = MAMBA_STATE_PER_REQ_NO_CACHE - mamba_state_needed = num_reqs * factor - if mamba_available_size < mamba_state_needed: - if tree_cache is not None and tree_cache.supports_mamba(): - mamba_num = max(0, mamba_state_needed - mamba_available_size) - tree_cache.evict_for_alloc( - EvictParams(num_tokens=0, mamba_num=mamba_num) - ) + # Keep combined serving's existing behavior: try cache eviction, then let + # the pool allocate. A failed precheck need not mean allocation will fail. + ensure_mamba_capacity(req_to_token_pool, reqs, tree_cache) req_pool_indices = req_to_token_pool.alloc(reqs) if req_pool_indices is None: raise RuntimeError( diff --git a/python/sglang/srt/mem_cache/allocation_sizing.py b/python/sglang/srt/mem_cache/allocation_sizing.py index f41a0b1c46..e03a79f66d 100644 --- a/python/sglang/srt/mem_cache/allocation_sizing.py +++ b/python/sglang/srt/mem_cache/allocation_sizing.py @@ -8,6 +8,17 @@ from sglang.srt.runtime_context import ( ) +def get_mamba_tracking_slots( + *, extra_buffer: bool, overlap: bool, lazy: bool = False +) -> int: + """Tracking slots allocated initially; lazy mode defers the second slot.""" + if not extra_buffer: + return 0 + if lazy: + return 1 + return 2 if overlap else 1 + + def get_alloc_page_size() -> int: # Mirrors _build_token_to_kv_pool_allocator's DCP branch; the platform # allocators that skip it page smaller, so this is an upper bound for them. diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 6131f6f32b..55cdcd4028 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -23,12 +23,6 @@ if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache -# Needs 2 + 1 slots for mamba request with prefix cache. 2 for ping pong cache, 1 for running mamba state. -MAMBA_STATE_PER_REQ_PREFIX_CACHE = 3 -# Lazy mode: 1 + 1 slots (1 ping-pong + 1 running), second ping-pong allocated on demand at boundary. -MAMBA_STATE_PER_REQ_PREFIX_CACHE_LAZY = 2 -MAMBA_STATE_PER_REQ_NO_CACHE = 1 - logger = logging.getLogger(__name__) diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 0295f30a51..1ba3b17f37 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -34,7 +34,10 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( get_kv_cache_quant_method, resolve_kv_cache_quant, ) -from sglang.srt.mem_cache.allocation_sizing import get_req_to_token_extra_context_len +from sglang.srt.mem_cache.allocation_sizing import ( + get_mamba_tracking_slots, + get_req_to_token_extra_context_len, +) from sglang.srt.mem_cache.allocator import ( BaseTokenToKVPoolAllocator, PagedTokenToKVPoolAllocator, @@ -179,9 +182,6 @@ def mm_runtime_reservation_gb( # which the decode-time drop does not shrink. MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO = 3 MAMBA_CACHE_BASE_RATIO_DROP_ON_SKIP = 1 -MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP = 2 -MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP_LAZY = 1 -MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP = 1 MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_BUFFER = 1 @@ -2268,18 +2268,15 @@ class KVCacheConfigurator: additional_ratio = 0 if get_exec().mamba.enable_mamba_extra_buffer: - # ping-pong buffer size is 2 when overlap schedule is on, 1 otherwise. - # Lazy mode saves 1 slot (2 → 1) for overlap; non-overlap already uses 1. - if not get_schedule().disable_overlap_schedule: - if get_exec().mamba.enable_mamba_extra_buffer_lazy: - additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP_LAZY - else: - additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP - else: + if get_schedule().disable_overlap_schedule: assert not get_exec().mamba.enable_mamba_extra_buffer_lazy, ( "Lazy extra buffer requires overlap schedule (--disable-overlap-schedule is incompatible)" ) - additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP + additional_ratio = get_mamba_tracking_slots( + extra_buffer=True, + overlap=not get_schedule().disable_overlap_schedule, + lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy, + ) elif skip_decode_lock: # no_buffer under skip: add the base drop back so effective stays 3, # the prefill->decode peak needs ~3 slots/req and this leaf-only mode diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 953aa9d70c..1556eb64c3 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -59,6 +59,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( UnquantizedKVCacheMethod, ) from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots from sglang.srt.mem_cache.allocator.mamba import MambaSlotAllocator from sglang.srt.mem_cache.index_key_cache import IndexKeyCache from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner @@ -1368,7 +1369,14 @@ class HybridReqToTokenPool(ReqToTokenPool): enable_memory_saver=enable_memory_saver, ) - self.mamba_ping_pong_track_buffer_size = 2 if enable_overlap_schedule else 1 + self.mamba_ping_pong_track_buffer_size = get_mamba_tracking_slots( + extra_buffer=True, overlap=enable_overlap_schedule + ) + self.mamba_initial_tracking_slots = get_mamba_tracking_slots( + extra_buffer=enable_mamba_extra_buffer, + overlap=enable_overlap_schedule, + lazy=enable_mamba_extra_buffer_lazy, + ) self.enable_mamba_extra_buffer = enable_mamba_extra_buffer self.enable_mamba_extra_buffer_lazy = enable_mamba_extra_buffer_lazy self.enable_memory_saver = enable_memory_saver @@ -1722,11 +1730,7 @@ class HybridReqToTokenPool(ReqToTokenPool): Lazy mode allocates 1 slot with the second set to -1 (allocated on demand at track boundaries). Normal mode allocates all slots upfront. """ - n = ( - 1 - if self.enable_mamba_extra_buffer_lazy - else self.mamba_ping_pong_track_buffer_size - ) + n = self.mamba_initial_tracking_slots slots = self.mamba_allocator.alloc(n) assert slots is not None, ( "Not enough space for mamba ping pong idx, " diff --git a/test/registered/unit/mem_cache/test_mamba_capacity.py b/test/registered/unit/mem_cache/test_mamba_capacity.py new file mode 100644 index 0000000000..a2383b196e --- /dev/null +++ b/test/registered/unit/mem_cache/test_mamba_capacity.py @@ -0,0 +1,282 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from sglang.srt.disaggregation.decode import ( + DecodePreallocQueue, + DecodeReqToTokenPool, + HybridMambaDecodeReqToTokenPool, +) +from sglang.srt.mem_cache.allocation import alloc_req_slots, ensure_mamba_capacity +from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots +from sglang.srt.mem_cache.allocator.mamba import MambaSlotAllocator +from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +def make_pool(free, *, lazy=False, extra_buffer=True, track_buffer_size=2): + pool = object.__new__(HybridReqToTokenPool) + pool.mamba_allocator = MambaSlotAllocator(5, "cpu") + pool.enable_mamba_extra_buffer = extra_buffer + pool.enable_mamba_extra_buffer_lazy = lazy + pool.mamba_ping_pong_track_buffer_size = track_buffer_size + pool.mamba_initial_tracking_slots = get_mamba_tracking_slots( + extra_buffer=extra_buffer, overlap=track_buffer_size == 2, lazy=lazy + ) + held = pool.mamba_allocator.alloc(5 - free) + pool.available_size = lambda: 1 + pool.alloc = MagicMock(return_value=[1]) + return pool, held + + +def make_req(*, holds_mamba=False, has_buffer=False): + return SimpleNamespace( + kv=SimpleNamespace( + holds_mamba=holds_mamba, + mamba_ping_pong_track_buffer=object() if has_buffer else None, + req_pool_idx=None, + mamba_pool_idx=object() if holds_mamba else None, + ) + ) + + +def make_cache(pool, held, release): + reclaimed = False + + def evict(_): + nonlocal reclaimed + if release and not reclaimed: + pool.mamba_allocator.free(held[:release]) + reclaimed = True + + return SimpleNamespace( + supports_mamba=lambda: True, + evict_for_alloc=MagicMock(side_effect=evict), + ) + + +class TestMambaCapacity(unittest.TestCase): + def test_admission_matches_tracking_allocation(self): + for extra_buffer, overlap, lazy, expected in [ + (False, False, False, 0), + (False, True, False, 0), + (True, False, False, 1), + (True, True, False, 2), + (True, True, True, 1), + ]: + with self.subTest(extra_buffer=extra_buffer, overlap=overlap, lazy=lazy): + # Run the real constructor's sizing, without allocating model tensors. + with ( + patch.object(ReqToTokenPool, "__init__", return_value=None), + patch.object(HybridReqToTokenPool, "_init_mamba_pool"), + ): + pool = HybridReqToTokenPool( + size=1, + mamba_size=5, + mamba_spec_state_size=1, + max_context_len=16, + device="cpu", + enable_memory_saver=False, + cache_params=None, + mamba_layer_ids=[], + enable_mamba_extra_buffer=extra_buffer, + enable_overlap_schedule=overlap, + enable_mamba_extra_buffer_lazy=lazy, + ) + self.assertEqual(pool.mamba_initial_tracking_slots, expected) + pool.mamba_allocator = MambaSlotAllocator(1 + expected, "cpu") + req = make_req() + held = pool.mamba_allocator.alloc(1) + self.assertFalse(ensure_mamba_capacity(pool, [req], None)) + pool.mamba_allocator.free(held) + self.assertTrue(ensure_mamba_capacity(pool, [req], None)) + + # Consume the live slot, then allocate tracking buffers using production code. + pool.mamba_allocator.alloc(1) + if extra_buffer: + pool._alloc_ping_pong_buffer(req) + buf = req.kv.mamba_ping_pong_track_buffer + self.assertEqual(int((buf >= 0).sum()), expected) + if lazy: + self.assertEqual(buf.tolist()[1], -1) + self.assertEqual(pool.mamba_allocator.available_size(), 0) + + def test_pd_pool_sizing_includes_tracking_and_preallocated_requests(self): + for extra_buffer, overlap, expected_slots_per_req in [ + (False, False, 1), + (False, True, 1), + (True, False, 2), + (True, True, 3), + ]: + with self.subTest(extra_buffer=extra_buffer, overlap=overlap): + with ( + patch.object(DecodeReqToTokenPool, "__init__", return_value=None), + patch.object(HybridReqToTokenPool, "_init_mamba_pool") as init_pool, + ): + pool = HybridMambaDecodeReqToTokenPool( + size=2, + pre_alloc_size=1, + mamba_size=1, + max_context_len=16, + device="cpu", + enable_memory_saver=False, + cache_params=None, + mamba_layer_ids=[], + speculative_num_draft_tokens=None, + enable_mamba_extra_buffer=extra_buffer, + enable_overlap_schedule=overlap, + ) + self.assertEqual( + 1 + pool.mamba_initial_tracking_slots, expected_slots_per_req + ) + self.assertEqual( + init_pool.call_args.kwargs["mamba_size"], + 3 * expected_slots_per_req, + ) + + def test_reclaims_full_shortfall_before_allocation(self): + for free, expected_eviction in [(1, 2), (2, 1), (3, 0)]: + with self.subTest(free=free): + pool, held = make_pool(free) + cache = make_cache(pool, held, expected_eviction) + self.assertTrue(ensure_mamba_capacity(pool, [make_req()], cache)) + self.assertGreaterEqual(pool.mamba_allocator.available_size(), 3) + if expected_eviction: + self.assertEqual( + cache.evict_for_alloc.call_args.args[0].mamba_num, + expected_eviction, + ) + else: + cache.evict_for_alloc.assert_not_called() + + def test_partial_eviction_defers_before_mutating_request(self): + pool, held = make_pool(1) + cache = make_cache(pool, held, 1) + req = make_req() + self.assertFalse(ensure_mamba_capacity(pool, [req], cache)) + self.assertEqual(pool.mamba_allocator.available_size(), 2) + self.assertIsNone(req.kv.req_pool_idx) + self.assertIsNone(req.kv.mamba_pool_idx) + self.assertEqual(alloc_req_slots(pool, [req], cache), [1]) + pool.alloc.assert_called_once_with([req]) + + def test_uses_only_missing_slots_for_each_request(self): + pool, held = make_pool(2) + cache = make_cache(pool, held, 1) + reqs = [make_req(), make_req(holds_mamba=True, has_buffer=True)] + self.assertTrue(ensure_mamba_capacity(pool, reqs, cache)) + self.assertEqual(cache.evict_for_alloc.call_args.args[0].mamba_num, 1) + + for holds_mamba, has_buffer, free, release, expected_eviction in [ + (True, False, 1, 1, 1), + (False, True, 1, 0, 0), + (True, True, 0, 0, 0), + ]: + with self.subTest(holds_mamba=holds_mamba, has_buffer=has_buffer): + pool, held = make_pool(free) + cache = make_cache(pool, held, release) + req = make_req(holds_mamba=holds_mamba, has_buffer=has_buffer) + self.assertTrue(ensure_mamba_capacity(pool, [req], cache)) + if expected_eviction: + self.assertEqual( + cache.evict_for_alloc.call_args.args[0].mamba_num, + expected_eviction, + ) + else: + cache.evict_for_alloc.assert_not_called() + + def test_tracking_buffer_settings(self): + pool, held = make_pool(1, lazy=True) + cache = make_cache(pool, held, 1) + self.assertTrue(ensure_mamba_capacity(pool, [make_req()], cache)) + self.assertEqual(cache.evict_for_alloc.call_args.args[0].mamba_num, 1) + + pool, held = make_pool(1, track_buffer_size=1) + cache = make_cache(pool, held, 1) + self.assertTrue(ensure_mamba_capacity(pool, [make_req()], cache)) + self.assertEqual(cache.evict_for_alloc.call_args.args[0].mamba_num, 1) + + pool, held = make_pool(1, extra_buffer=False) + cache = make_cache(pool, held, 0) + cache.supports_mamba = lambda: False + self.assertTrue(ensure_mamba_capacity(pool, [make_req()], cache)) + cache.evict_for_alloc.assert_not_called() + + pool, held = make_pool(1) + cache = make_cache(pool, held, 0) + cache.supports_mamba = lambda: False + self.assertFalse(ensure_mamba_capacity(pool, [make_req()], cache)) + cache.evict_for_alloc.assert_not_called() + + def test_pd_preallocation_leaves_blocked_request_queued(self): + pool, held = make_pool(1) + cache = make_cache(pool, held, 0) + req = SimpleNamespace( + rid="blocked", + origin_input_ids=[1], + output_ids=[], + finished_reason=None, + is_retracted=True, + kv=make_req().kv, + ) + decode_req = SimpleNamespace( + req=req, waiting_for_input=True, is_rebootstrap=False + ) + queue = DecodePreallocQueue.__new__(DecodePreallocQueue) + queue.pp_size = 1 + queue.queue = [decode_req] + queue.pending_reqs = [] + queue.retracted_queue = [req] + queue.req_to_token_pool = pool + queue.req_to_metadata_buffer_idx_allocator = SimpleNamespace( + available_size=lambda: 1 + ) + queue.tree_cache = cache + queue.scheduler = SimpleNamespace( + running_batch=SimpleNamespace(reqs=[]), + enable_priority_scheduling=False, + enable_hisparse=False, + enable_lora=False, + ) + queue._resolve_pending_reqs = MagicMock() + queue._update_handshake_waiters = MagicMock() + queue._uses_swa_tail_prealloc = MagicMock(return_value=False) + queue._uses_swa_reservation = MagicMock(return_value=False) + queue._allocatable_token_budgets = MagicMock(return_value=4096) + queue._hicache_pending_restore_tokens = MagicMock(return_value=0) + queue._match_prefix_and_lock = MagicMock() + queue._pre_alloc = MagicMock() + + with patch( + "sglang.srt.disaggregation.decode.get_disagg", + return_value=SimpleNamespace( + disaggregation_decode_host_receive_threshold=0 + ), + ): + self.assertEqual(queue.pop_preallocated(), ([], [])) + self.assertEqual(queue.queue, [decode_req]) + queue._match_prefix_and_lock.assert_not_called() + queue._pre_alloc.assert_not_called() + self.assertIsNone(req.kv.req_pool_idx) + + queue._prealloc_required_tokens = MagicMock(return_value=(1, 0)) + queue._prealloc_reservation_fits = MagicMock(return_value=True) + queue.token_to_kv_pool_allocator = SimpleNamespace( + prealloc_fits_assumes_reclaim=lambda: False + ) + with patch( + "sglang.srt.disaggregation.decode.get_disagg", + return_value=SimpleNamespace( + disaggregation_decode_retraction_backup="cpu_tensor" + ), + ): + self.assertEqual(queue.resume_retracted_reqs(), []) + self.assertEqual(queue.retracted_queue, [req]) + self.assertTrue(req.is_retracted) + queue._pre_alloc.assert_not_called() + + +if __name__ == "__main__": + unittest.main()