fix(unified-memory): propagate unified-memory lazy checkpoint policy and handle unallocated session slots (#42072)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
SuperSong
2026-10-04 12:51:22 -07:00
committed by GitHub
co-authored by Cheng Wan
parent fea3c088a1
commit bab04cd791
5 changed files with 200 additions and 2 deletions
@@ -723,6 +723,7 @@ class KVCacheConfigurator:
max_num_reqs=max_num_reqs,
enable_memory_saver=get_exec().features.enable_memory_saver,
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens,
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
@@ -835,6 +836,7 @@ class KVCacheConfigurator:
max_num_reqs=max_num_reqs,
enable_memory_saver=get_exec().features.enable_memory_saver,
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens,
@@ -1148,6 +1148,7 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
cache_params,
mamba_layer_ids: List[int],
enable_mamba_extra_buffer: bool,
enable_mamba_extra_buffer_lazy: bool,
speculative_num_draft_tokens: Optional[int] = None,
enable_overlap_schedule: bool = True,
start_layer: Optional[int] = None,
@@ -1171,6 +1172,7 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
cache_params=cache_params,
mamba_layer_ids=mamba_layer_ids,
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=speculative_num_draft_tokens,
enable_overlap_schedule=enable_overlap_schedule,
start_layer=start_layer,
@@ -1367,6 +1369,7 @@ def init_unified_mamba_pools(
max_num_reqs: int,
enable_memory_saver: bool,
enable_mamba_extra_buffer: bool,
enable_mamba_extra_buffer_lazy: bool,
speculative_num_draft_tokens: Optional[int],
disable_overlap_schedule: bool,
need_sort: bool,
@@ -1466,6 +1469,7 @@ def init_unified_mamba_pools(
cache_params=mamba2_cache_params,
mamba_layer_ids=mamba_layer_ids,
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=speculative_num_draft_tokens,
enable_overlap_schedule=not disable_overlap_schedule,
start_layer=start_layer,
@@ -2062,6 +2066,7 @@ def init_unified_mamba_swa_pools(
max_num_reqs: int,
enable_memory_saver: bool,
enable_mamba_extra_buffer: bool,
enable_mamba_extra_buffer_lazy: bool,
disable_overlap_schedule: bool,
need_sort: bool,
speculative_num_draft_tokens: Optional[int] = None,
@@ -2194,6 +2199,7 @@ def init_unified_mamba_swa_pools(
cache_params=mamba2_cache_params,
mamba_layer_ids=mamba_layer_ids,
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
speculative_num_draft_tokens=speculative_num_draft_tokens,
enable_overlap_schedule=not disable_overlap_schedule,
start_layer=start_layer,
@@ -323,7 +323,7 @@ class StreamingSession:
if slot.kv.holds_mamba:
total += slot.kv.mamba_pool_idx.numel()
if slot.kv.mamba_ping_pong_track_buffer is not None:
total += slot.kv.mamba_ping_pong_track_buffer.numel()
total += int((slot.kv.mamba_ping_pong_track_buffer != -1).sum().item())
return total
def _free_slot_mamba(self, slot: SessionSlot) -> None:
@@ -335,7 +335,8 @@ class StreamingSession:
mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0))
slot.kv.mamba_pool_idx = None
if slot.kv.mamba_ping_pong_track_buffer is not None:
mamba_allocator.free(slot.kv.mamba_ping_pong_track_buffer)
indices = slot.kv.mamba_ping_pong_track_buffer
mamba_allocator.free(indices[indices != -1])
slot.kv.mamba_ping_pong_track_buffer = None
# -- Internal helpers (streaming body bits) --
@@ -79,6 +79,7 @@ def build_tri_pool(*, page_size=1):
extra_max_context_len=1,
max_num_reqs=4,
enable_mamba_extra_buffer=False,
enable_mamba_extra_buffer_lazy=False,
disable_overlap_schedule=True,
sliding_window_size=32,
)