[MLX] Sync decode KV to the pool once at release, clamped to the owned prefix (#43292)

This commit is contained in:
Liangsheng Yin
2026-10-09 16:20:45 -07:00
committed by GitHub
parent 114cb44920
commit 46498bc8c5
6 changed files with 234 additions and 39 deletions
@@ -839,14 +839,17 @@ class MlxModelRunner:
)
self._attention_kv_pool.set_kv_all_layers(slot_ids_mx, k_all, v_all)
def _sync_decode_kv_to_pool(self, req_id: str) -> None:
"""Sync un-flushed decode KV for *req_id* to the shared pool."""
def _sync_decode_kv_to_pool(self, req_id: str, end: int | None = None) -> None:
"""Sync un-flushed decode KV for *req_id* to the shared pool, up to
position *end* when given."""
if self._attention_kv_pool is None or self._req_to_token_pool is None:
return
cache = self._req_caches.get(req_id)
if cache is None:
return
current_offset = self._first_attention_cache(cache).offset
if end is not None:
current_offset = min(current_offset, end)
synced_offset = self._req_synced_offset.get(req_id, 0)
if current_offset <= synced_offset:
return
@@ -864,12 +867,13 @@ class MlxModelRunner:
self._sync_new_kv_to_pool(cache, synced_offset, slot_ids)
self._req_synced_offset[req_id] = current_offset
def flush_all_decode_kv(self) -> None:
"""Sync all active requests' un-flushed decode KV to the pool."""
if self.disable_radix_cache or self._attention_kv_pool is None:
return
for req_id in list(self._req_caches.keys()):
self._sync_decode_kv_to_pool(req_id)
def release_request_row(self, req_id: str, owned_len: int) -> None:
"""Sync decode KV only within the first *owned_len* positions (chained
decode steps own no slots past it), then forget the row, which the
scheduler may reuse once released."""
if not self.disable_radix_cache:
self._sync_decode_kv_to_pool(req_id, end=owned_len)
self._req_pool_idx.pop(req_id, None)
def decode_batch(self, req_ids: list[str]) -> list[int]:
"""Decode one token per request.
@@ -1678,12 +1682,9 @@ class MlxModelRunner:
"""Check if a request has active state."""
return req_id in self._req_caches
def remove_request(self, req_id: str, sync_kv: bool = True):
"""Release request state, first syncing its remaining decode KV to the
pool unless *sync_kv* is False (the caller discards that KV)."""
if sync_kv and not self.disable_radix_cache:
self._sync_decode_kv_to_pool(req_id)
def remove_request(self, req_id: str):
"""Release request state without syncing: decode KV reaches the pool
only through release_request_row, while the request owns its row."""
self._req_token_ids.pop(req_id, None)
self._req_sampling.pop(req_id, None)
cache = self._req_caches.pop(req_id, None)
@@ -169,12 +169,14 @@ class MlxTpModelWorker(TpModelWorker):
self._mlx_active_rids |= current_rids
def prepare_for_kv_cache_release(self, req) -> None:
"""Snapshot MLX auxiliary state at the scheduler's radix insert point."""
"""Write the request's MLX state to the pool at the scheduler's radix
insert point, while the request still owns its req_to_token row."""
if self._mlx_runner.has_request(req.rid):
self._mlx_runner.store_auxiliary_state_for_request(req.rid)
# Prefer the just-snapshotted live auxiliary state for the final
# insert. Any older tracked slot is released during component cleanup.
req.kv.mamba_last_track_seqlen = None
self._mlx_runner.release_request_row(req.rid, req.owned_kv_len())
def _drop_state_if_retracted(self, req: Req) -> None:
"""Drop the MLX state of a request retracted since its state was made.
@@ -186,7 +188,7 @@ class MlxTpModelWorker(TpModelWorker):
made_at = self._req_retraction_count.get(req.rid)
if made_at is not None and made_at != req.retraction_count:
del self._req_retraction_count[req.rid]
self._mlx_runner.remove_request(req.rid, sync_kv=False)
self._mlx_runner.remove_request(req.rid)
self._mlx_active_rids.discard(req.rid)
def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str:
@@ -458,10 +460,6 @@ class MlxTpModelWorker(TpModelWorker):
)
if forward_mode.is_extend():
# TODO (changminbark): Implement per-batch flushing using prefix_slot_ids
# Ensure the pool is up-to-date before pool-backed attention
# reads it for prefix-cached prefills. Mirror the sync path.
self._mlx_runner.flush_all_decode_kv()
return self._async_extend_batch(batch)
raise ValueError(
@@ -374,6 +374,8 @@ class SchedulerBatchResultProcessor:
if sampling_mask_finish_reason is None:
self._maybe_collect_routed_experts(req)
self._maybe_collect_indexer_topk(req)
if batch.decoding_reqs and req in batch.decoding_reqs:
self._prepare_kv_cache_release(req)
release_kv_cache(
req,
self.tree_cache,
@@ -1323,6 +1325,14 @@ class SchedulerBatchResultProcessor:
"InternalServerError",
)
def _prepare_kv_cache_release(self, req: Req) -> None:
# Only workers that keep KV outside the pool (MLX) define the hook.
prepare_release = getattr(
self.model_worker, "prepare_for_kv_cache_release", None
)
if callable(prepare_release):
prepare_release(req)
def _handle_sampling_mask_abort(self, req: Req) -> None:
"""Release a request whose sampled token must not be committed."""
if req.multimodal_inputs is not None and req.session is None:
@@ -1332,11 +1342,7 @@ class SchedulerBatchResultProcessor:
else:
if get_memory().enable_hisparse:
self.hisparse_coordinator.request_finished(req)
prepare_release = getattr(
self.model_worker, "prepare_for_kv_cache_release", None
)
if callable(prepare_release):
prepare_release(req)
self._prepare_kv_cache_release(req)
release_kv_cache(req, self.tree_cache, checkpoint=False)
req.time_stats.set_completion_time()
@@ -1419,11 +1425,7 @@ class SchedulerBatchResultProcessor:
else:
if get_memory().enable_hisparse:
self.hisparse_coordinator.request_finished(req)
prepare_release = getattr(
self.model_worker, "prepare_for_kv_cache_release", None
)
if callable(prepare_release):
prepare_release(req)
self._prepare_kv_cache_release(req)
checkpoint = (
req.mamba_lazy_checkpoint
if get_exec().mamba.enable_mamba_extra_buffer_lazy
@@ -0,0 +1,140 @@
"""Decode KV must never be written through req_to_token entries past
``owned_kv_len()``, nor after the request's row is released."""
from __future__ import annotations
import importlib.util
import unittest
from types import SimpleNamespace
from sglang.test.ci.ci_register import register_mlx_ci
from sglang.test.test_utils import CustomTestCase
register_mlx_ci(est_time=5, suite="stage-a-unit-test-mlx")
_HAS_MLX = (
importlib.util.find_spec("mlx") is not None
and importlib.util.find_spec("mlx_lm") is not None
)
if _HAS_MLX:
import mlx.core as mx
import torch
from mlx_lm.models import llama
from sglang.srt.hardware_backend.mlx.aot import MlxAOTKernelSet
from sglang.srt.hardware_backend.mlx.kv_cache import (
find_attention_layers,
get_layer_window_sizes,
patch_model_attention,
)
from sglang.srt.hardware_backend.mlx.kv_cache.layout import MlxModelCacheLayout
from sglang.srt.hardware_backend.mlx.model_runner import MlxModelRunner
from sglang.srt.hardware_backend.mlx.tp_worker import MlxTpModelWorker
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
PROMPT_LEN = 5
DECODE_STEPS = 5
OWN_SLOTS = [1, 2, 3, 4, 5, 6] # prompt slots + the one allocated decode slot
FOREIGN_SLOTS = [20, 21, 22, 23] # stale row entries left by a previous owner
def _runner(req_to_token_pool):
model = llama.Model(
llama.ModelArgs(
model_type="llama",
hidden_size=32,
num_hidden_layers=2,
intermediate_size=64,
num_attention_heads=2,
rms_norm_eps=1e-5,
vocab_size=64,
head_dim=16,
num_key_value_heads=1,
)
)
patch_model_attention(model)
layers, attrs = find_attention_layers(model)
runner = MlxModelRunner.__new__(MlxModelRunner)
runner.model = model
runner.disable_radix_cache = False
runner._cache_layout = MlxModelCacheLayout.from_attention_discovery(
layers, attrs, layer_window_sizes=get_layer_window_sizes(model)
)
runner._max_seq_len = 64
runner._cache_pool = []
runner._req_caches = {}
runner._req_token_ids = {}
runner._req_sampling = {}
runner._req_pool_idx = {}
runner._req_synced_offset = {}
runner._attention_kv_pool = None
runner._decode_step_ct = 0
runner._clear_steps = 0
runner._aot_kernels = MlxAOTKernelSet()
runner._pool_size = 32
runner.init_cache_pools(req_to_token_pool)
return runner
@unittest.skipUnless(_HAS_MLX, "requires mlx + mlx_lm")
class TestDecodeKvRelease(CustomTestCase):
def _decoded_runner(self):
req_to_token_pool = ReqToTokenPool(
size=2, max_context_len=64, device="cpu", enable_memory_saver=False
)
row = req_to_token_pool.req_to_token[0]
row[: len(OWN_SLOTS)] = torch.tensor(OWN_SLOTS)
row[len(OWN_SLOTS) : len(OWN_SLOTS) + len(FOREIGN_SLOTS)] = torch.tensor(
FOREIGN_SLOTS
)
runner = _runner(req_to_token_pool)
prompt = [3, 14, 15, 9, 2]
runner.prefill(
req_id="r",
new_token_ids=prompt,
full_token_ids=prompt,
prefix_slot_ids=[],
new_slot_ids=OWN_SLOTS[:PROMPT_LEN],
req_pool_idx=0,
)
for _ in range(DECODE_STEPS):
runner.decode_batch(["r"])
return runner
def _assert_foreign_slots_untouched(self, runner):
pool = runner._attention_kv_pool
foreign_k, foreign_v = pool.get_kv(0, mx.array(FOREIGN_SLOTS, dtype=mx.int32))
self.assertEqual(mx.abs(foreign_k).max().item(), 0.0)
self.assertEqual(mx.abs(foreign_v).max().item(), 0.0)
def test_release_syncs_owned_prefix_and_nothing_after(self):
runner = self._decoded_runner()
worker = MlxTpModelWorker.__new__(MlxTpModelWorker)
worker._mlx_runner = runner
req = SimpleNamespace(
rid="r",
kv=SimpleNamespace(mamba_last_track_seqlen=None),
owned_kv_len=lambda: len(OWN_SLOTS),
)
worker.prepare_for_kv_cache_release(req)
# The row goes back to the scheduler; the request's MLX state is
# dropped later, when it leaves the running batch.
runner.remove_request("r")
own_k, _ = runner._attention_kv_pool.get_kv(
0, mx.array([OWN_SLOTS[-1]], dtype=mx.int32)
)
self.assertGreater(mx.abs(own_k).max().item(), 0.0)
self._assert_foreign_slots_untouched(runner)
def test_retracted_request_writes_nothing(self):
# A retraction frees the row with no release hook; dropping the
# state afterwards must not write through the (reusable) row.
runner = self._decoded_runner()
runner.remove_request("r")
self._assert_foreign_slots_untouched(runner)
if __name__ == "__main__":
unittest.main()
@@ -65,7 +65,6 @@ class _FakeRunner:
# chunk-finality derivation reaching the runner intact.
self.logits_flags: dict[tuple[str, str], bool] = {}
self.prefix_slot_ids: dict[str, list[int]] = {}
self.removed_sync_kv: dict[str, bool] = {}
self._req_caches: dict[str, list] = {}
self._counter = 0
@@ -73,12 +72,8 @@ class _FakeRunner:
def has_request(self, rid):
return rid in self._known
def flush_all_decode_kv(self):
pass
def remove_request(self, rid, sync_kv=True):
def remove_request(self, rid):
self.calls.append(("remove_request", rid))
self.removed_sync_kv[rid] = sync_kv
self._known.discard(rid)
def ops_for(self, rid):
@@ -267,8 +262,8 @@ class TestMlxExtendRouting(CustomTestCase):
# ---------- retraction ----------
def test_retracted_request_reprefills_from_scratch(self):
"""A request retracted after its MLX state was made drops that state
without a pool sync, so its re-prefill routes as a fresh prefill."""
"""A request retracted after its MLX state was made drops that state,
so its re-prefill routes as a fresh prefill."""
worker = self._worker(known_rids=set())
req = _FakeReq("r1")
worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4]))
@@ -276,7 +271,6 @@ class TestMlxExtendRouting(CustomTestCase):
req.retraction_count += 1
worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4]))
self.assertEqual(worker._mlx_runner.removed_sync_kv, {"r1": False})
self.assertEqual(
worker._mlx_runner.ops_for("r1"),
["prefill_start", "remove_request", "prefill_start"],
@@ -318,6 +318,66 @@ class TestPrefillSkippedOutput(CustomTestCase):
)
class TestMixedBatchKvRelease(CustomTestCase):
def test_decode_req_finishing_in_mixed_batch_gets_release_hook(self):
"""A decode req that finishes inside a mixed batch goes through the
worker's pre-release hook, as it would in a decode batch; a prefill
req finishing on its first token does not."""
prefill_req = _PrefillReq(
rid="prefill", inflight_middle_chunks=0, return_hidden_states=False
)
decode_req = _PrefillReq(
rid="decode", inflight_middle_chunks=0, return_hidden_states=False
)
for req in (prefill_req, decode_req):
req.finished = lambda req=req: bool(req.output_ids)
req.kv = SimpleNamespace(kv_committed_len=0)
batch = SimpleNamespace(
reqs=[prefill_req, decode_req],
decoding_reqs=[decode_req],
return_logprob=False,
return_hidden_states=False,
return_hidden_states_mode=CaptureHiddenMode.NULL,
spec_info=None,
spec_algorithm=SimpleNamespace(is_none=lambda: True),
prefill_stats=None,
dp_cooperation_info=None,
)
result = SimpleNamespace(
copy_done=None,
auxiliary_host_output=None,
routed_experts_output=None,
indexer_topk_output=None,
logits_output=SimpleNamespace(
hidden_states=None, customized_info=None, sampling_mask_output=None
),
next_token_ids=torch.tensor([5, 6]),
extend_input_len_per_req=None,
extend_logprob_start_len_per_req=None,
grammar_advanced=False,
can_run_cuda_graph=False,
skipped_output_comm=False,
)
processor = _make_processor(self)
with (
patch(
"sglang.srt.managers.scheduler_components."
"batch_result_processor.release_kv_cache"
) as release,
patch.object(
SchedulerBatchResultProcessor, "_maybe_collect_routed_experts"
),
patch.object(SchedulerBatchResultProcessor, "_maybe_collect_indexer_topk"),
):
processor.process_batch_result_prefill(batch, result)
self.assertEqual(release.call_count, 2)
processor.model_worker.prepare_for_kv_cache_release.assert_called_once_with(
decode_req
)
class TestDecodeWithoutLogits(CustomTestCase):
def test_pipeline_result_commits_token_without_sampling_metadata(self):
processor = _make_processor(self)