fix(unified-memory): propagate unified-memory lazy checkpoint policy and handle unallocated session slots (#42072)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
SuperSong
2026-10-04 12:51:22 -07:00
committed by GitHub
co-authored by Cheng Wan
parent fea3c088a1
commit bab04cd791
5 changed files with 200 additions and 2 deletions
@@ -723,6 +723,7 @@ class KVCacheConfigurator:
max_num_reqs=max_num_reqs,
enable_memory_saver=get_exec().features.enable_memory_saver,
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens,
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
@@ -835,6 +836,7 @@ class KVCacheConfigurator:
max_num_reqs=max_num_reqs,
enable_memory_saver=get_exec().features.enable_memory_saver,
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens,
@@ -1148,6 +1148,7 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
cache_params,
mamba_layer_ids: List[int],
enable_mamba_extra_buffer: bool,
enable_mamba_extra_buffer_lazy: bool,
speculative_num_draft_tokens: Optional[int] = None,
enable_overlap_schedule: bool = True,
start_layer: Optional[int] = None,
@@ -1171,6 +1172,7 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
cache_params=cache_params,
mamba_layer_ids=mamba_layer_ids,
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=speculative_num_draft_tokens,
enable_overlap_schedule=enable_overlap_schedule,
start_layer=start_layer,
@@ -1367,6 +1369,7 @@ def init_unified_mamba_pools(
max_num_reqs: int,
enable_memory_saver: bool,
enable_mamba_extra_buffer: bool,
enable_mamba_extra_buffer_lazy: bool,
speculative_num_draft_tokens: Optional[int],
disable_overlap_schedule: bool,
need_sort: bool,
@@ -1466,6 +1469,7 @@ def init_unified_mamba_pools(
cache_params=mamba2_cache_params,
mamba_layer_ids=mamba_layer_ids,
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=speculative_num_draft_tokens,
enable_overlap_schedule=not disable_overlap_schedule,
start_layer=start_layer,
@@ -2062,6 +2066,7 @@ def init_unified_mamba_swa_pools(
max_num_reqs: int,
enable_memory_saver: bool,
enable_mamba_extra_buffer: bool,
enable_mamba_extra_buffer_lazy: bool,
disable_overlap_schedule: bool,
need_sort: bool,
speculative_num_draft_tokens: Optional[int] = None,
@@ -2194,6 +2199,7 @@ def init_unified_mamba_swa_pools(
cache_params=mamba2_cache_params,
mamba_layer_ids=mamba_layer_ids,
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=speculative_num_draft_tokens,
enable_overlap_schedule=not disable_overlap_schedule,
start_layer=start_layer,
@@ -323,7 +323,7 @@ class StreamingSession:
if slot.kv.holds_mamba:
total += slot.kv.mamba_pool_idx.numel()
if slot.kv.mamba_ping_pong_track_buffer is not None:
total += slot.kv.mamba_ping_pong_track_buffer.numel()
total += int((slot.kv.mamba_ping_pong_track_buffer != -1).sum().item())
return total
def _free_slot_mamba(self, slot: SessionSlot) -> None:
@@ -335,7 +335,8 @@ class StreamingSession:
mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0))
slot.kv.mamba_pool_idx = None
if slot.kv.mamba_ping_pong_track_buffer is not None:
mamba_allocator.free(slot.kv.mamba_ping_pong_track_buffer)
indices = slot.kv.mamba_ping_pong_track_buffer
mamba_allocator.free(indices[indices != -1])
slot.kv.mamba_ping_pong_track_buffer = None
# -- Internal helpers (streaming body bits) --
@@ -79,6 +79,7 @@ def build_tri_pool(*, page_size=1):
extra_max_context_len=1,
max_num_reqs=4,
enable_mamba_extra_buffer=False,
enable_mamba_extra_buffer_lazy=False,
disable_overlap_schedule=True,
sliding_window_size=32,
)
@@ -975,12 +975,200 @@ class TestTriFactorySizing(unittest.TestCase):
max_num_reqs=4,
enable_memory_saver=False,
enable_mamba_extra_buffer=False,
enable_mamba_extra_buffer_lazy=False,
disable_overlap_schedule=True,
need_sort=False,
)
kw.update(over)
return kw
def test_streaming_session_lazy_checkpoint_cleanup(self):
from types import SimpleNamespace
from sglang.srt.managers.schedule_batch import ReqKvInfo
from sglang.srt.mem_cache.allocator.mamba import MambaSlotAllocator
from sglang.srt.runtime_context import get_parallel
from sglang.srt.session.streaming_session import SessionSlot, StreamingSession
for lazy_compaction in (None, False, True):
for ids in ((0, -1), (-1, 0), (-1, -1), (0, 1)):
with self.subTest(compaction=lazy_compaction, ids=ids):
kw = self._factory_kwargs(
enable_mamba_extra_buffer=True,
enable_mamba_extra_buffer_lazy=True,
lazy_compaction=lazy_compaction,
)
with get_parallel().override(attn_dcp_size=1):
pool = init_unified_mamba_swa_pools(**kw).req_to_token_pool
if lazy_compaction is None:
pool = SimpleNamespace(
mamba_allocator=MambaSlotAllocator(16, "cpu")
)
allocator = pool.mamba_allocator
available = allocator.available_size()
count = sum(i != -1 for i in ids)
live = allocator.alloc(count + 1)
track = live.new_full((2,), -1)
for position, index in enumerate(ids):
if index != -1:
track[position] = live[index + 1]
slot = SessionSlot(
kv=ReqKvInfo(
req_pool_idx=1,
mamba_pool_idx=live[0],
mamba_ping_pong_track_buffer=track,
)
)
session = StreamingSession(SimpleNamespace(req_to_token_pool=pool))
session.slots = {"test": slot}
self.assertEqual(session.session_held_mamba_slots(), count + 1)
self.assertEqual(session.session_held_mamba_slots({1}), 0)
session._free_slot_mamba(slot)
self.assertEqual(session.session_held_mamba_slots(), 0)
self.assertEqual(allocator.available_size(), available)
session._free_slot_mamba(slot)
self.assertEqual(allocator.available_size(), available)
def test_mamba_lazy_checkpoint_cleanup(self):
from types import SimpleNamespace
from sglang.srt.runtime_context import get_parallel
for tri_pool in (False, True):
for lazy in (False, True):
for keep_checkpoint in (False, True):
with self.subTest(
tri_pool=tri_pool, lazy=lazy, keep=keep_checkpoint
):
from sglang.srt.mem_cache import kv_cache_configurator as cfg
kw = self._factory_kwargs()
model = SimpleNamespace(
get_num_kv_heads=lambda tp, dcp: 2,
head_dim=4,
context_len=16,
full_attention_layer_ids=[0],
swa_attention_layer_ids=[1],
sliding_window_size=8,
)
configurator = SimpleNamespace(
mambaish_config=SimpleNamespace(
mamba2_cache_params=kw["mamba2_cache_params"],
full_attention_layer_ids=[0],
),
model_config=model,
layer_info=SimpleNamespace(
swa_attention_layer_ids=[1],
full_attention_layer_ids=[0],
start_layer=0,
end_layer=2,
),
device=_DEV,
kv_cache_dtype=torch.float16,
page_size=1,
is_draft_worker=False,
use_mla_backend=False,
is_hybrid_swa=tri_pool,
is_hybrid_swa_compress=False,
forward_stream=None,
)
# Run the production configurator AND factory. Reverting
# either top-level flag forwarding must break cleanup.
with (
get_parallel().override(attn_dcp_size=1, attn_tp_size=1),
patch.object(
cfg,
"get_exec",
return_value=SimpleNamespace(
features=SimpleNamespace(enable_memory_saver=False),
mamba=SimpleNamespace(
enable_mamba_extra_buffer=True,
enable_mamba_extra_buffer_lazy=lazy,
),
),
),
patch.object(
cfg,
"get_spec",
return_value=SimpleNamespace(
speculative_num_draft_tokens=None,
),
),
patch.object(
cfg,
"get_schedule",
return_value=SimpleNamespace(
max_mamba_cache_size=4,
disable_overlap_schedule=False,
mamba_full_memory_ratio=None,
),
),
patch.object(
cfg,
"get_disagg",
return_value=SimpleNamespace(
disaggregation_mode="null",
),
),
patch.object(
cfg, "_should_enable_lazy_compaction", return_value=True
),
):
if tri_pool:
bundle = cfg.KVCacheConfigurator._init_unified_mamba_swa_pools(
configurator,
max_num_reqs=4,
full_max_total_num_tokens=64,
swa_max_total_num_tokens=32,
)
else:
bundle = (
cfg.KVCacheConfigurator._init_unified_mamba_pools(
configurator,
max_num_reqs=4,
max_total_num_tokens=64,
)
)
pool = bundle.req_to_token_pool
self.assertEqual(pool.enable_mamba_extra_buffer_lazy, lazy)
allocator = pool.mamba_allocator
initial_available = allocator.available_size()
req = SimpleNamespace(
kv=SimpleNamespace(
req_pool_idx=0, mamba_pool_idx=allocator.alloc(1)[0]
)
)
pool._alloc_ping_pong_buffer(req)
buf = req.kv.mamba_ping_pong_track_buffer
self.assertEqual(int((buf != -1).sum()), 1 if lazy else 2)
if lazy:
# A decode boundary replaces the old checkpoint and
# leaves its ping-pong entry unallocated (-1).
replacement = allocator.alloc(1)
allocator.free(buf[:1].clone())
pool.set_mamba_ping_pong_slot(req, 0, -1)
pool.set_mamba_ping_pong_slot(req, 1, replacement[0])
else:
pool.set_mamba_ping_pong_slot(req, 1, buf[1])
retained = buf[1:2].clone()
with patch.object(
allocator, "free", wraps=allocator.free
) as free:
pool.free_mamba_cache(req, 1 if keep_checkpoint else None)
for call in free.call_args_list:
self.assertTrue(bool((call.args[0] >= 0).all()))
self.assertEqual(
allocator.available_size(),
initial_available - int(keep_checkpoint),
)
self.assertIsNone(req.kv.mamba_ping_pong_track_buffer)
if keep_checkpoint:
self.assertTrue(
bool((allocator.translate(retained) >= 0).all())
)
allocator.free(retained)
self.assertEqual(allocator.available_size(), initial_available)
def test_budget_sizing_and_boot_signature(self):
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,