[MLX] Sync decode KV to the pool once at release, clamped to the owned prefix (#43292)

This commit is contained in:
Liangsheng Yin
2026-10-09 16:20:45 -07:00
committed by GitHub
parent 114cb44920
commit 46498bc8c5
6 changed files with 234 additions and 39 deletions
@@ -839,14 +839,17 @@ class MlxModelRunner:
)
self._attention_kv_pool.set_kv_all_layers(slot_ids_mx, k_all, v_all)
def _sync_decode_kv_to_pool(self, req_id: str) -> None:
"""Sync un-flushed decode KV for *req_id* to the shared pool."""
def _sync_decode_kv_to_pool(self, req_id: str, end: int | None = None) -> None:
"""Sync un-flushed decode KV for *req_id* to the shared pool, up to
position *end* when given."""
if self._attention_kv_pool is None or self._req_to_token_pool is None:
return
cache = self._req_caches.get(req_id)
if cache is None:
return
current_offset = self._first_attention_cache(cache).offset
if end is not None:
current_offset = min(current_offset, end)
synced_offset = self._req_synced_offset.get(req_id, 0)
if current_offset <= synced_offset:
return
@@ -864,12 +867,13 @@ class MlxModelRunner:
self._sync_new_kv_to_pool(cache, synced_offset, slot_ids)
self._req_synced_offset[req_id] = current_offset
def flush_all_decode_kv(self) -> None:
"""Sync all active requests' un-flushed decode KV to the pool."""
if self.disable_radix_cache or self._attention_kv_pool is None:
return
for req_id in list(self._req_caches.keys()):
self._sync_decode_kv_to_pool(req_id)
def release_request_row(self, req_id: str, owned_len: int) -> None:
"""Sync decode KV only within the first *owned_len* positions (chained
decode steps own no slots past it), then forget the row, which the
scheduler may reuse once released."""
if not self.disable_radix_cache:
self._sync_decode_kv_to_pool(req_id, end=owned_len)
self._req_pool_idx.pop(req_id, None)
def decode_batch(self, req_ids: list[str]) -> list[int]:
"""Decode one token per request.
@@ -1678,12 +1682,9 @@ class MlxModelRunner:
"""Check if a request has active state."""
return req_id in self._req_caches
def remove_request(self, req_id: str, sync_kv: bool = True):
"""Release request state, first syncing its remaining decode KV to the
pool unless *sync_kv* is False (the caller discards that KV)."""
if sync_kv and not self.disable_radix_cache:
self._sync_decode_kv_to_pool(req_id)
def remove_request(self, req_id: str):
"""Release request state without syncing: decode KV reaches the pool
only through release_request_row, while the request owns its row."""
self._req_token_ids.pop(req_id, None)
self._req_sampling.pop(req_id, None)
cache = self._req_caches.pop(req_id, None)
@@ -169,12 +169,14 @@ class MlxTpModelWorker(TpModelWorker):
self._mlx_active_rids |= current_rids
def prepare_for_kv_cache_release(self, req) -> None:
"""Snapshot MLX auxiliary state at the scheduler's radix insert point."""
"""Write the request's MLX state to the pool at the scheduler's radix
insert point, while the request still owns its req_to_token row."""
if self._mlx_runner.has_request(req.rid):
self._mlx_runner.store_auxiliary_state_for_request(req.rid)
# Prefer the just-snapshotted live auxiliary state for the final
# insert. Any older tracked slot is released during component cleanup.
req.kv.mamba_last_track_seqlen = None
self._mlx_runner.release_request_row(req.rid, req.owned_kv_len())
def _drop_state_if_retracted(self, req: Req) -> None:
"""Drop the MLX state of a request retracted since its state was made.
@@ -186,7 +188,7 @@ class MlxTpModelWorker(TpModelWorker):
made_at = self._req_retraction_count.get(req.rid)
if made_at is not None and made_at != req.retraction_count:
del self._req_retraction_count[req.rid]
self._mlx_runner.remove_request(req.rid, sync_kv=False)
self._mlx_runner.remove_request(req.rid)
self._mlx_active_rids.discard(req.rid)
def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str:
@@ -458,10 +460,6 @@ class MlxTpModelWorker(TpModelWorker):
)
if forward_mode.is_extend():
# TODO (changminbark): Implement per-batch flushing using prefix_slot_ids
# Ensure the pool is up-to-date before pool-backed attention
# reads it for prefix-cached prefills. Mirror the sync path.
self._mlx_runner.flush_all_decode_kv()
return self._async_extend_batch(batch)
raise ValueError(
@@ -374,6 +374,8 @@ class SchedulerBatchResultProcessor:
if sampling_mask_finish_reason is None:
self._maybe_collect_routed_experts(req)
self._maybe_collect_indexer_topk(req)
if batch.decoding_reqs and req in batch.decoding_reqs:
self._prepare_kv_cache_release(req)
release_kv_cache(
req,
self.tree_cache,
@@ -1323,6 +1325,14 @@ class SchedulerBatchResultProcessor:
"InternalServerError",
)
def _prepare_kv_cache_release(self, req: Req) -> None:
# Only workers that keep KV outside the pool (MLX) define the hook.
prepare_release = getattr(
self.model_worker, "prepare_for_kv_cache_release", None
)
if callable(prepare_release):
prepare_release(req)
def _handle_sampling_mask_abort(self, req: Req) -> None:
"""Release a request whose sampled token must not be committed."""
if req.multimodal_inputs is not None and req.session is None:
@@ -1332,11 +1342,7 @@ class SchedulerBatchResultProcessor:
else:
if get_memory().enable_hisparse:
self.hisparse_coordinator.request_finished(req)
prepare_release = getattr(
self.model_worker, "prepare_for_kv_cache_release", None
)
if callable(prepare_release):
prepare_release(req)
self._prepare_kv_cache_release(req)
release_kv_cache(req, self.tree_cache, checkpoint=False)
req.time_stats.set_completion_time()
@@ -1419,11 +1425,7 @@ class SchedulerBatchResultProcessor:
else:
if get_memory().enable_hisparse:
self.hisparse_coordinator.request_finished(req)
prepare_release = getattr(
self.model_worker, "prepare_for_kv_cache_release", None
)
if callable(prepare_release):
prepare_release(req)
self._prepare_kv_cache_release(req)
checkpoint = (
req.mamba_lazy_checkpoint
if get_exec().mamba.enable_mamba_extra_buffer_lazy