From ba3fc9441dd1721b4fbf6ec6930e362fbdf61721 Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:16:31 +0800 Subject: [PATCH] [Fix] mamba: Release consumed lazy Mamba prefix locks before donation retry (#43559) --- .../unified_cache/components/mamba.py | 28 ++++++++++-- .../radix_cache/test_mamba_cache_kl.py | 13 ++++++ .../test_mamba_donated_alloc_ratio.py | 45 ++++++++++++++++++- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py index 04358a6cbb..762a5d45bc 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import defaultdict +from dataclasses import replace from typing import TYPE_CHECKING, Callable, Optional, Sequence import torch @@ -504,10 +505,29 @@ class MambaComponent(TreeComponent): if cd.lock_ref == 0: self.tree_core._update_evictable_leaf_sets(node) - def _alloc_mamba_slot(self) -> torch.Tensor: + def _alloc_mamba_slot(self, req: Optional[Req] = None) -> torch.Tensor: """Allocate one mamba pool slot, evicting if necessary.""" slot = self.cache.req_to_token_pool.mamba_allocator.alloc(1) if slot is None: + if req is not None and get_exec().mamba.enable_mamba_extra_buffer_lazy: + lock = req.lock + ct = self.component_type + if ( + lock is not None + and not lock.swa_released + and ct not in lock.receipt.skipped_lock_components + ): + # Prefill has consumed the COW source; retain only its KV locks. + self.cache.dec_lock_ref( + lock.node, + replace( + lock.receipt, + skipped_lock_components=tuple( + c for c in self.cache.tree_components if c != ct + ), + ), + ) + lock.receipt.skipped_lock_components += (ct,) self.cache.evict_for_alloc(EvictParams(num_tokens=0, mamba_num=1)) slot = self.cache.req_to_token_pool.mamba_allocator.alloc(1) assert slot is not None, "Can not alloc mamba cache" @@ -611,7 +631,7 @@ class MambaComponent(TreeComponent): # Donate the mamba index to the radix cache instead of copying. if self.int8_ckpt_pool is not None: if self.cache.enable_mamba_extra_buffer: - new_slot = self._alloc_mamba_slot() + new_slot = self._alloc_mamba_slot(req) src_active = ( self.cache.req_to_token_pool.donate_mamba_ping_pong_slot( req, new_slot @@ -624,14 +644,14 @@ class MambaComponent(TreeComponent): req.kv.mamba_pool_idx.view(-1) ) elif self.cache.enable_mamba_extra_buffer: - new_slot = self._alloc_mamba_slot() + new_slot = self._alloc_mamba_slot(req) mamba_value_donated = ( self.cache.req_to_token_pool.donate_mamba_ping_pong_slot( req, new_slot ) ) else: - mamba_value_donated = self._alloc_mamba_slot() + mamba_value_donated = self._alloc_mamba_slot(req) # mamba_pool is a pure PHYSICAL store; translate both slot ids # virtual->physical (identity for the non-unified memory pool) first. translate = self.cache.req_to_token_pool.translate_mamba_indices diff --git a/test/registered/radix_cache/test_mamba_cache_kl.py b/test/registered/radix_cache/test_mamba_cache_kl.py index b0543c14cf..97a448d0ac 100644 --- a/test/registered/radix_cache/test_mamba_cache_kl.py +++ b/test/registered/radix_cache/test_mamba_cache_kl.py @@ -110,6 +110,19 @@ class TestMambaLazyInplaceKL(MambaCacheKLMixin, DefaultServerBase): self._generate([44] * 128, 1) self._generate([42] * 128, 1) + def test_lazy_donation_with_four_slots_and_one_request(self): + """A prefix-hit request must finish without a fifth Mamba slot under overlap.""" + self._flush_cache() + self._generate([42] * 128, 1) + + # One cached prefix, live state, and keep checkpoint occupy three slots; + # the boundary decode takes the fourth before prefill donation runs. + result = self._generate([42] * 128 + [43] * 127, 32) + self.assertEqual(result["meta_info"]["cached_tokens"], 128) + logprobs = result["meta_info"]["output_token_logprobs"] + self.assertEqual(len(logprobs), 32) + self.assertTrue(np.isfinite([row[0] for row in logprobs]).all()) + def test_reuse_distribution_after_inplace_tracking(self): config = AutoConfig.from_pretrained(self.model) self.vocab_ids = list(range(config.get_text_config().vocab_size)) diff --git a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py index 53da60ce86..af4f71b86b 100644 --- a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py +++ b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py @@ -11,7 +11,7 @@ eviction -- which is why the peak, not the decode steady state, sets the floor. """ import unittest -from types import SimpleNamespace +from types import MethodType, SimpleNamespace import torch @@ -19,6 +19,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, EvictParams, IncLockRefResult, + TreeLock, ) from sglang.srt.mem_cache.unified_cache.components.base import ComponentType from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent @@ -160,6 +161,46 @@ class TestMambaRatioEnvGate(unittest.TestCase): r(extra_buffer=True, lazy=False, disable_overlap=False), 4 ) # overlap + def test_lazy_donation_releases_only_its_prefix_state(self): + from sglang.srt import runtime_context as rc + + for slots_per_req in (3, 4): + with self.subTest(slots_per_req=slots_per_req): + component, cache, _ = _build_peak(slots_per_req * N, lock_prefixes=True) + cache.allocator.alloc((slots_per_req - 2) * N) + nodes = {node.id: node for node in cache.prefix_nodes} + full = _RecordingComp(ComponentType.FULL, 2) + cache.components = (full, component) + cache.node_by_id = nodes.__getitem__ + cache._assert_receipt_anchor = UnifiedTreeCore._assert_receipt_anchor + cache._update_evictable_leaf_sets = lambda _: None + cache._release_components = MethodType( + UnifiedTreeCore._release_components, cache + ) + cache.dec_lock_ref = MethodType(UnifiedTreeCore.dec_lock_ref, cache) + with rc.get_context().override_server_args( + disable_radix_cache=False, + mamba_radix_cache_strategy="extra_buffer_lazy", + ): + for node in nodes.values(): + req = SimpleNamespace( + lock=TreeLock(node.id, DecLockRefParams(node_id=node.id)) + ) + released = len(full.released) + self.assertEqual(cache.allocator.free_ids, []) + self.assertIsNotNone(component._alloc_mamba_slot(req)) + self.assertEqual( + node.component_data[ComponentType.MAMBA].lock_ref, 0 + ) + self.assertEqual( + req.lock.receipt.skipped_lock_components, + (ComponentType.MAMBA,), + ) + self.assertEqual(len(full.released), released) + cache.dec_lock_ref(req.lock.node, req.lock.receipt) + self.assertEqual(len(full.released), released + 1) + self.assertEqual(len(full.released), N) + class _RecordingComp: """Fake tree component: records the dec params it is asked to release with.""" @@ -172,7 +213,7 @@ class _RecordingComp: def eviction_priority(self, is_leaf): return self._priority - def release_component_lock(self, node, params): + def release_component_lock(self, node, params, lock_host=False): self.released.append(params) def release_window_lock( # SWA only