fix(unified-memory): propagate unified-memory lazy checkpoint policy and handle unallocated session slots (#42072)
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
@@ -723,6 +723,7 @@ class KVCacheConfigurator:
|
||||
max_num_reqs=max_num_reqs,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
|
||||
speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens,
|
||||
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
|
||||
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
|
||||
@@ -835,6 +836,7 @@ class KVCacheConfigurator:
|
||||
max_num_reqs=max_num_reqs,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
|
||||
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
|
||||
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
|
||||
speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens,
|
||||
|
||||
@@ -1148,6 +1148,7 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
|
||||
cache_params,
|
||||
mamba_layer_ids: List[int],
|
||||
enable_mamba_extra_buffer: bool,
|
||||
enable_mamba_extra_buffer_lazy: bool,
|
||||
speculative_num_draft_tokens: Optional[int] = None,
|
||||
enable_overlap_schedule: bool = True,
|
||||
start_layer: Optional[int] = None,
|
||||
@@ -1171,6 +1172,7 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
|
||||
cache_params=cache_params,
|
||||
mamba_layer_ids=mamba_layer_ids,
|
||||
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
|
||||
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
||||
enable_overlap_schedule=enable_overlap_schedule,
|
||||
start_layer=start_layer,
|
||||
@@ -1367,6 +1369,7 @@ def init_unified_mamba_pools(
|
||||
max_num_reqs: int,
|
||||
enable_memory_saver: bool,
|
||||
enable_mamba_extra_buffer: bool,
|
||||
enable_mamba_extra_buffer_lazy: bool,
|
||||
speculative_num_draft_tokens: Optional[int],
|
||||
disable_overlap_schedule: bool,
|
||||
need_sort: bool,
|
||||
@@ -1466,6 +1469,7 @@ def init_unified_mamba_pools(
|
||||
cache_params=mamba2_cache_params,
|
||||
mamba_layer_ids=mamba_layer_ids,
|
||||
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
|
||||
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
||||
enable_overlap_schedule=not disable_overlap_schedule,
|
||||
start_layer=start_layer,
|
||||
@@ -2062,6 +2066,7 @@ def init_unified_mamba_swa_pools(
|
||||
max_num_reqs: int,
|
||||
enable_memory_saver: bool,
|
||||
enable_mamba_extra_buffer: bool,
|
||||
enable_mamba_extra_buffer_lazy: bool,
|
||||
disable_overlap_schedule: bool,
|
||||
need_sort: bool,
|
||||
speculative_num_draft_tokens: Optional[int] = None,
|
||||
@@ -2194,6 +2199,7 @@ def init_unified_mamba_swa_pools(
|
||||
cache_params=mamba2_cache_params,
|
||||
mamba_layer_ids=mamba_layer_ids,
|
||||
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
|
||||
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
||||
enable_overlap_schedule=not disable_overlap_schedule,
|
||||
start_layer=start_layer,
|
||||
|
||||
@@ -323,7 +323,7 @@ class StreamingSession:
|
||||
if slot.kv.holds_mamba:
|
||||
total += slot.kv.mamba_pool_idx.numel()
|
||||
if slot.kv.mamba_ping_pong_track_buffer is not None:
|
||||
total += slot.kv.mamba_ping_pong_track_buffer.numel()
|
||||
total += int((slot.kv.mamba_ping_pong_track_buffer != -1).sum().item())
|
||||
return total
|
||||
|
||||
def _free_slot_mamba(self, slot: SessionSlot) -> None:
|
||||
@@ -335,7 +335,8 @@ class StreamingSession:
|
||||
mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0))
|
||||
slot.kv.mamba_pool_idx = None
|
||||
if slot.kv.mamba_ping_pong_track_buffer is not None:
|
||||
mamba_allocator.free(slot.kv.mamba_ping_pong_track_buffer)
|
||||
indices = slot.kv.mamba_ping_pong_track_buffer
|
||||
mamba_allocator.free(indices[indices != -1])
|
||||
slot.kv.mamba_ping_pong_track_buffer = None
|
||||
|
||||
# -- Internal helpers (streaming body bits) --
|
||||
|
||||
@@ -79,6 +79,7 @@ def build_tri_pool(*, page_size=1):
|
||||
extra_max_context_len=1,
|
||||
max_num_reqs=4,
|
||||
enable_mamba_extra_buffer=False,
|
||||
enable_mamba_extra_buffer_lazy=False,
|
||||
disable_overlap_schedule=True,
|
||||
sliding_window_size=32,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user