[MLX] Sync decode KV to the pool once at release, clamped to the owned prefix (#43292)
This commit is contained in:
@@ -839,14 +839,17 @@ class MlxModelRunner:
|
||||
)
|
||||
self._attention_kv_pool.set_kv_all_layers(slot_ids_mx, k_all, v_all)
|
||||
|
||||
def _sync_decode_kv_to_pool(self, req_id: str) -> None:
|
||||
"""Sync un-flushed decode KV for *req_id* to the shared pool."""
|
||||
def _sync_decode_kv_to_pool(self, req_id: str, end: int | None = None) -> None:
|
||||
"""Sync un-flushed decode KV for *req_id* to the shared pool, up to
|
||||
position *end* when given."""
|
||||
if self._attention_kv_pool is None or self._req_to_token_pool is None:
|
||||
return
|
||||
cache = self._req_caches.get(req_id)
|
||||
if cache is None:
|
||||
return
|
||||
current_offset = self._first_attention_cache(cache).offset
|
||||
if end is not None:
|
||||
current_offset = min(current_offset, end)
|
||||
synced_offset = self._req_synced_offset.get(req_id, 0)
|
||||
if current_offset <= synced_offset:
|
||||
return
|
||||
@@ -864,12 +867,13 @@ class MlxModelRunner:
|
||||
self._sync_new_kv_to_pool(cache, synced_offset, slot_ids)
|
||||
self._req_synced_offset[req_id] = current_offset
|
||||
|
||||
def flush_all_decode_kv(self) -> None:
|
||||
"""Sync all active requests' un-flushed decode KV to the pool."""
|
||||
if self.disable_radix_cache or self._attention_kv_pool is None:
|
||||
return
|
||||
for req_id in list(self._req_caches.keys()):
|
||||
self._sync_decode_kv_to_pool(req_id)
|
||||
def release_request_row(self, req_id: str, owned_len: int) -> None:
|
||||
"""Sync decode KV only within the first *owned_len* positions (chained
|
||||
decode steps own no slots past it), then forget the row, which the
|
||||
scheduler may reuse once released."""
|
||||
if not self.disable_radix_cache:
|
||||
self._sync_decode_kv_to_pool(req_id, end=owned_len)
|
||||
self._req_pool_idx.pop(req_id, None)
|
||||
|
||||
def decode_batch(self, req_ids: list[str]) -> list[int]:
|
||||
"""Decode one token per request.
|
||||
@@ -1678,12 +1682,9 @@ class MlxModelRunner:
|
||||
"""Check if a request has active state."""
|
||||
return req_id in self._req_caches
|
||||
|
||||
def remove_request(self, req_id: str, sync_kv: bool = True):
|
||||
"""Release request state, first syncing its remaining decode KV to the
|
||||
pool unless *sync_kv* is False (the caller discards that KV)."""
|
||||
if sync_kv and not self.disable_radix_cache:
|
||||
self._sync_decode_kv_to_pool(req_id)
|
||||
|
||||
def remove_request(self, req_id: str):
|
||||
"""Release request state without syncing: decode KV reaches the pool
|
||||
only through release_request_row, while the request owns its row."""
|
||||
self._req_token_ids.pop(req_id, None)
|
||||
self._req_sampling.pop(req_id, None)
|
||||
cache = self._req_caches.pop(req_id, None)
|
||||
|
||||
@@ -169,12 +169,14 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
self._mlx_active_rids |= current_rids
|
||||
|
||||
def prepare_for_kv_cache_release(self, req) -> None:
|
||||
"""Snapshot MLX auxiliary state at the scheduler's radix insert point."""
|
||||
"""Write the request's MLX state to the pool at the scheduler's radix
|
||||
insert point, while the request still owns its req_to_token row."""
|
||||
if self._mlx_runner.has_request(req.rid):
|
||||
self._mlx_runner.store_auxiliary_state_for_request(req.rid)
|
||||
# Prefer the just-snapshotted live auxiliary state for the final
|
||||
# insert. Any older tracked slot is released during component cleanup.
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
self._mlx_runner.release_request_row(req.rid, req.owned_kv_len())
|
||||
|
||||
def _drop_state_if_retracted(self, req: Req) -> None:
|
||||
"""Drop the MLX state of a request retracted since its state was made.
|
||||
@@ -186,7 +188,7 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
made_at = self._req_retraction_count.get(req.rid)
|
||||
if made_at is not None and made_at != req.retraction_count:
|
||||
del self._req_retraction_count[req.rid]
|
||||
self._mlx_runner.remove_request(req.rid, sync_kv=False)
|
||||
self._mlx_runner.remove_request(req.rid)
|
||||
self._mlx_active_rids.discard(req.rid)
|
||||
|
||||
def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str:
|
||||
@@ -458,10 +460,6 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
)
|
||||
|
||||
if forward_mode.is_extend():
|
||||
# TODO (changminbark): Implement per-batch flushing using prefix_slot_ids
|
||||
# Ensure the pool is up-to-date before pool-backed attention
|
||||
# reads it for prefix-cached prefills. Mirror the sync path.
|
||||
self._mlx_runner.flush_all_decode_kv()
|
||||
return self._async_extend_batch(batch)
|
||||
|
||||
raise ValueError(
|
||||
|
||||
@@ -374,6 +374,8 @@ class SchedulerBatchResultProcessor:
|
||||
if sampling_mask_finish_reason is None:
|
||||
self._maybe_collect_routed_experts(req)
|
||||
self._maybe_collect_indexer_topk(req)
|
||||
if batch.decoding_reqs and req in batch.decoding_reqs:
|
||||
self._prepare_kv_cache_release(req)
|
||||
release_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
@@ -1323,6 +1325,14 @@ class SchedulerBatchResultProcessor:
|
||||
"InternalServerError",
|
||||
)
|
||||
|
||||
def _prepare_kv_cache_release(self, req: Req) -> None:
|
||||
# Only workers that keep KV outside the pool (MLX) define the hook.
|
||||
prepare_release = getattr(
|
||||
self.model_worker, "prepare_for_kv_cache_release", None
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
|
||||
def _handle_sampling_mask_abort(self, req: Req) -> None:
|
||||
"""Release a request whose sampled token must not be committed."""
|
||||
if req.multimodal_inputs is not None and req.session is None:
|
||||
@@ -1332,11 +1342,7 @@ class SchedulerBatchResultProcessor:
|
||||
else:
|
||||
if get_memory().enable_hisparse:
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
prepare_release = getattr(
|
||||
self.model_worker, "prepare_for_kv_cache_release", None
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
self._prepare_kv_cache_release(req)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
req.time_stats.set_completion_time()
|
||||
|
||||
@@ -1419,11 +1425,7 @@ class SchedulerBatchResultProcessor:
|
||||
else:
|
||||
if get_memory().enable_hisparse:
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
prepare_release = getattr(
|
||||
self.model_worker, "prepare_for_kv_cache_release", None
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
self._prepare_kv_cache_release(req)
|
||||
checkpoint = (
|
||||
req.mamba_lazy_checkpoint
|
||||
if get_exec().mamba.enable_mamba_extra_buffer_lazy
|
||||
|
||||
Reference in New Issue
Block a user