From 46498bc8c5cd9dc48147dffe10f98a770ce84b2c Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 9 Oct 2026 16:20:45 -0700 Subject: [PATCH] [MLX] Sync decode KV to the pool once at release, clamped to the owned prefix (#43292) --- .../srt/hardware_backend/mlx/model_runner.py | 29 ++-- .../srt/hardware_backend/mlx/tp_worker.py | 10 +- .../batch_result_processor.py | 22 +-- .../mlx/test_decode_kv_release.py | 140 ++++++++++++++++++ .../mlx/test_tp_worker_routing.py | 12 +- ...st_batch_result_processor_hidden_states.py | 60 ++++++++ 6 files changed, 234 insertions(+), 39 deletions(-) create mode 100644 test/registered/unit/hardware_backend/mlx/test_decode_kv_release.py diff --git a/python/sglang/srt/hardware_backend/mlx/model_runner.py b/python/sglang/srt/hardware_backend/mlx/model_runner.py index 3c7f931a4d..703e6b70be 100644 --- a/python/sglang/srt/hardware_backend/mlx/model_runner.py +++ b/python/sglang/srt/hardware_backend/mlx/model_runner.py @@ -839,14 +839,17 @@ class MlxModelRunner: ) self._attention_kv_pool.set_kv_all_layers(slot_ids_mx, k_all, v_all) - def _sync_decode_kv_to_pool(self, req_id: str) -> None: - """Sync un-flushed decode KV for *req_id* to the shared pool.""" + def _sync_decode_kv_to_pool(self, req_id: str, end: int | None = None) -> None: + """Sync un-flushed decode KV for *req_id* to the shared pool, up to + position *end* when given.""" if self._attention_kv_pool is None or self._req_to_token_pool is None: return cache = self._req_caches.get(req_id) if cache is None: return current_offset = self._first_attention_cache(cache).offset + if end is not None: + current_offset = min(current_offset, end) synced_offset = self._req_synced_offset.get(req_id, 0) if current_offset <= synced_offset: return @@ -864,12 +867,13 @@ class MlxModelRunner: self._sync_new_kv_to_pool(cache, synced_offset, slot_ids) self._req_synced_offset[req_id] = current_offset - def flush_all_decode_kv(self) -> None: - """Sync all active requests' un-flushed decode KV to the pool.""" - if self.disable_radix_cache or self._attention_kv_pool is None: - return - for req_id in list(self._req_caches.keys()): - self._sync_decode_kv_to_pool(req_id) + def release_request_row(self, req_id: str, owned_len: int) -> None: + """Sync decode KV only within the first *owned_len* positions (chained + decode steps own no slots past it), then forget the row, which the + scheduler may reuse once released.""" + if not self.disable_radix_cache: + self._sync_decode_kv_to_pool(req_id, end=owned_len) + self._req_pool_idx.pop(req_id, None) def decode_batch(self, req_ids: list[str]) -> list[int]: """Decode one token per request. @@ -1678,12 +1682,9 @@ class MlxModelRunner: """Check if a request has active state.""" return req_id in self._req_caches - def remove_request(self, req_id: str, sync_kv: bool = True): - """Release request state, first syncing its remaining decode KV to the - pool unless *sync_kv* is False (the caller discards that KV).""" - if sync_kv and not self.disable_radix_cache: - self._sync_decode_kv_to_pool(req_id) - + def remove_request(self, req_id: str): + """Release request state without syncing: decode KV reaches the pool + only through release_request_row, while the request owns its row.""" self._req_token_ids.pop(req_id, None) self._req_sampling.pop(req_id, None) cache = self._req_caches.pop(req_id, None) diff --git a/python/sglang/srt/hardware_backend/mlx/tp_worker.py b/python/sglang/srt/hardware_backend/mlx/tp_worker.py index f81bfc0b1a..fad0aaac56 100644 --- a/python/sglang/srt/hardware_backend/mlx/tp_worker.py +++ b/python/sglang/srt/hardware_backend/mlx/tp_worker.py @@ -169,12 +169,14 @@ class MlxTpModelWorker(TpModelWorker): self._mlx_active_rids |= current_rids def prepare_for_kv_cache_release(self, req) -> None: - """Snapshot MLX auxiliary state at the scheduler's radix insert point.""" + """Write the request's MLX state to the pool at the scheduler's radix + insert point, while the request still owns its req_to_token row.""" if self._mlx_runner.has_request(req.rid): self._mlx_runner.store_auxiliary_state_for_request(req.rid) # Prefer the just-snapshotted live auxiliary state for the final # insert. Any older tracked slot is released during component cleanup. req.kv.mamba_last_track_seqlen = None + self._mlx_runner.release_request_row(req.rid, req.owned_kv_len()) def _drop_state_if_retracted(self, req: Req) -> None: """Drop the MLX state of a request retracted since its state was made. @@ -186,7 +188,7 @@ class MlxTpModelWorker(TpModelWorker): made_at = self._req_retraction_count.get(req.rid) if made_at is not None and made_at != req.retraction_count: del self._req_retraction_count[req.rid] - self._mlx_runner.remove_request(req.rid, sync_kv=False) + self._mlx_runner.remove_request(req.rid) self._mlx_active_rids.discard(req.rid) def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str: @@ -458,10 +460,6 @@ class MlxTpModelWorker(TpModelWorker): ) if forward_mode.is_extend(): - # TODO (changminbark): Implement per-batch flushing using prefix_slot_ids - # Ensure the pool is up-to-date before pool-backed attention - # reads it for prefix-cached prefills. Mirror the sync path. - self._mlx_runner.flush_all_decode_kv() return self._async_extend_batch(batch) raise ValueError( diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index c0fbc5e80c..f404cb0d83 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -374,6 +374,8 @@ class SchedulerBatchResultProcessor: if sampling_mask_finish_reason is None: self._maybe_collect_routed_experts(req) self._maybe_collect_indexer_topk(req) + if batch.decoding_reqs and req in batch.decoding_reqs: + self._prepare_kv_cache_release(req) release_kv_cache( req, self.tree_cache, @@ -1323,6 +1325,14 @@ class SchedulerBatchResultProcessor: "InternalServerError", ) + def _prepare_kv_cache_release(self, req: Req) -> None: + # Only workers that keep KV outside the pool (MLX) define the hook. + prepare_release = getattr( + self.model_worker, "prepare_for_kv_cache_release", None + ) + if callable(prepare_release): + prepare_release(req) + def _handle_sampling_mask_abort(self, req: Req) -> None: """Release a request whose sampled token must not be committed.""" if req.multimodal_inputs is not None and req.session is None: @@ -1332,11 +1342,7 @@ class SchedulerBatchResultProcessor: else: if get_memory().enable_hisparse: self.hisparse_coordinator.request_finished(req) - prepare_release = getattr( - self.model_worker, "prepare_for_kv_cache_release", None - ) - if callable(prepare_release): - prepare_release(req) + self._prepare_kv_cache_release(req) release_kv_cache(req, self.tree_cache, checkpoint=False) req.time_stats.set_completion_time() @@ -1419,11 +1425,7 @@ class SchedulerBatchResultProcessor: else: if get_memory().enable_hisparse: self.hisparse_coordinator.request_finished(req) - prepare_release = getattr( - self.model_worker, "prepare_for_kv_cache_release", None - ) - if callable(prepare_release): - prepare_release(req) + self._prepare_kv_cache_release(req) checkpoint = ( req.mamba_lazy_checkpoint if get_exec().mamba.enable_mamba_extra_buffer_lazy diff --git a/test/registered/unit/hardware_backend/mlx/test_decode_kv_release.py b/test/registered/unit/hardware_backend/mlx/test_decode_kv_release.py new file mode 100644 index 0000000000..e7a30032f5 --- /dev/null +++ b/test/registered/unit/hardware_backend/mlx/test_decode_kv_release.py @@ -0,0 +1,140 @@ +"""Decode KV must never be written through req_to_token entries past +``owned_kv_len()``, nor after the request's row is released.""" + +from __future__ import annotations + +import importlib.util +import unittest +from types import SimpleNamespace + +from sglang.test.ci.ci_register import register_mlx_ci +from sglang.test.test_utils import CustomTestCase + +register_mlx_ci(est_time=5, suite="stage-a-unit-test-mlx") + +_HAS_MLX = ( + importlib.util.find_spec("mlx") is not None + and importlib.util.find_spec("mlx_lm") is not None +) + +if _HAS_MLX: + import mlx.core as mx + import torch + from mlx_lm.models import llama + + from sglang.srt.hardware_backend.mlx.aot import MlxAOTKernelSet + from sglang.srt.hardware_backend.mlx.kv_cache import ( + find_attention_layers, + get_layer_window_sizes, + patch_model_attention, + ) + from sglang.srt.hardware_backend.mlx.kv_cache.layout import MlxModelCacheLayout + from sglang.srt.hardware_backend.mlx.model_runner import MlxModelRunner + from sglang.srt.hardware_backend.mlx.tp_worker import MlxTpModelWorker + from sglang.srt.mem_cache.memory_pool import ReqToTokenPool + +PROMPT_LEN = 5 +DECODE_STEPS = 5 +OWN_SLOTS = [1, 2, 3, 4, 5, 6] # prompt slots + the one allocated decode slot +FOREIGN_SLOTS = [20, 21, 22, 23] # stale row entries left by a previous owner + + +def _runner(req_to_token_pool): + model = llama.Model( + llama.ModelArgs( + model_type="llama", + hidden_size=32, + num_hidden_layers=2, + intermediate_size=64, + num_attention_heads=2, + rms_norm_eps=1e-5, + vocab_size=64, + head_dim=16, + num_key_value_heads=1, + ) + ) + patch_model_attention(model) + layers, attrs = find_attention_layers(model) + runner = MlxModelRunner.__new__(MlxModelRunner) + runner.model = model + runner.disable_radix_cache = False + runner._cache_layout = MlxModelCacheLayout.from_attention_discovery( + layers, attrs, layer_window_sizes=get_layer_window_sizes(model) + ) + runner._max_seq_len = 64 + runner._cache_pool = [] + runner._req_caches = {} + runner._req_token_ids = {} + runner._req_sampling = {} + runner._req_pool_idx = {} + runner._req_synced_offset = {} + runner._attention_kv_pool = None + runner._decode_step_ct = 0 + runner._clear_steps = 0 + runner._aot_kernels = MlxAOTKernelSet() + runner._pool_size = 32 + runner.init_cache_pools(req_to_token_pool) + return runner + + +@unittest.skipUnless(_HAS_MLX, "requires mlx + mlx_lm") +class TestDecodeKvRelease(CustomTestCase): + def _decoded_runner(self): + req_to_token_pool = ReqToTokenPool( + size=2, max_context_len=64, device="cpu", enable_memory_saver=False + ) + row = req_to_token_pool.req_to_token[0] + row[: len(OWN_SLOTS)] = torch.tensor(OWN_SLOTS) + row[len(OWN_SLOTS) : len(OWN_SLOTS) + len(FOREIGN_SLOTS)] = torch.tensor( + FOREIGN_SLOTS + ) + runner = _runner(req_to_token_pool) + prompt = [3, 14, 15, 9, 2] + runner.prefill( + req_id="r", + new_token_ids=prompt, + full_token_ids=prompt, + prefix_slot_ids=[], + new_slot_ids=OWN_SLOTS[:PROMPT_LEN], + req_pool_idx=0, + ) + for _ in range(DECODE_STEPS): + runner.decode_batch(["r"]) + return runner + + def _assert_foreign_slots_untouched(self, runner): + pool = runner._attention_kv_pool + foreign_k, foreign_v = pool.get_kv(0, mx.array(FOREIGN_SLOTS, dtype=mx.int32)) + self.assertEqual(mx.abs(foreign_k).max().item(), 0.0) + self.assertEqual(mx.abs(foreign_v).max().item(), 0.0) + + def test_release_syncs_owned_prefix_and_nothing_after(self): + runner = self._decoded_runner() + worker = MlxTpModelWorker.__new__(MlxTpModelWorker) + worker._mlx_runner = runner + req = SimpleNamespace( + rid="r", + kv=SimpleNamespace(mamba_last_track_seqlen=None), + owned_kv_len=lambda: len(OWN_SLOTS), + ) + worker.prepare_for_kv_cache_release(req) + # The row goes back to the scheduler; the request's MLX state is + # dropped later, when it leaves the running batch. + runner.remove_request("r") + + own_k, _ = runner._attention_kv_pool.get_kv( + 0, mx.array([OWN_SLOTS[-1]], dtype=mx.int32) + ) + self.assertGreater(mx.abs(own_k).max().item(), 0.0) + self._assert_foreign_slots_untouched(runner) + + def test_retracted_request_writes_nothing(self): + # A retraction frees the row with no release hook; dropping the + # state afterwards must not write through the (reusable) row. + runner = self._decoded_runner() + runner.remove_request("r") + self._assert_foreign_slots_untouched(runner) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py index 7a802eebb5..bc8621c508 100644 --- a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py +++ b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py @@ -65,7 +65,6 @@ class _FakeRunner: # chunk-finality derivation reaching the runner intact. self.logits_flags: dict[tuple[str, str], bool] = {} self.prefix_slot_ids: dict[str, list[int]] = {} - self.removed_sync_kv: dict[str, bool] = {} self._req_caches: dict[str, list] = {} self._counter = 0 @@ -73,12 +72,8 @@ class _FakeRunner: def has_request(self, rid): return rid in self._known - def flush_all_decode_kv(self): - pass - - def remove_request(self, rid, sync_kv=True): + def remove_request(self, rid): self.calls.append(("remove_request", rid)) - self.removed_sync_kv[rid] = sync_kv self._known.discard(rid) def ops_for(self, rid): @@ -267,8 +262,8 @@ class TestMlxExtendRouting(CustomTestCase): # ---------- retraction ---------- def test_retracted_request_reprefills_from_scratch(self): - """A request retracted after its MLX state was made drops that state - without a pool sync, so its re-prefill routes as a fresh prefill.""" + """A request retracted after its MLX state was made drops that state, + so its re-prefill routes as a fresh prefill.""" worker = self._worker(known_rids=set()) req = _FakeReq("r1") worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4])) @@ -276,7 +271,6 @@ class TestMlxExtendRouting(CustomTestCase): req.retraction_count += 1 worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4])) - self.assertEqual(worker._mlx_runner.removed_sync_kv, {"r1": False}) self.assertEqual( worker._mlx_runner.ops_for("r1"), ["prefill_start", "remove_request", "prefill_start"], diff --git a/test/registered/unit/managers/test_batch_result_processor_hidden_states.py b/test/registered/unit/managers/test_batch_result_processor_hidden_states.py index 6e62181374..4e33bef9fc 100644 --- a/test/registered/unit/managers/test_batch_result_processor_hidden_states.py +++ b/test/registered/unit/managers/test_batch_result_processor_hidden_states.py @@ -318,6 +318,66 @@ class TestPrefillSkippedOutput(CustomTestCase): ) +class TestMixedBatchKvRelease(CustomTestCase): + def test_decode_req_finishing_in_mixed_batch_gets_release_hook(self): + """A decode req that finishes inside a mixed batch goes through the + worker's pre-release hook, as it would in a decode batch; a prefill + req finishing on its first token does not.""" + prefill_req = _PrefillReq( + rid="prefill", inflight_middle_chunks=0, return_hidden_states=False + ) + decode_req = _PrefillReq( + rid="decode", inflight_middle_chunks=0, return_hidden_states=False + ) + for req in (prefill_req, decode_req): + req.finished = lambda req=req: bool(req.output_ids) + req.kv = SimpleNamespace(kv_committed_len=0) + batch = SimpleNamespace( + reqs=[prefill_req, decode_req], + decoding_reqs=[decode_req], + return_logprob=False, + return_hidden_states=False, + return_hidden_states_mode=CaptureHiddenMode.NULL, + spec_info=None, + spec_algorithm=SimpleNamespace(is_none=lambda: True), + prefill_stats=None, + dp_cooperation_info=None, + ) + result = SimpleNamespace( + copy_done=None, + auxiliary_host_output=None, + routed_experts_output=None, + indexer_topk_output=None, + logits_output=SimpleNamespace( + hidden_states=None, customized_info=None, sampling_mask_output=None + ), + next_token_ids=torch.tensor([5, 6]), + extend_input_len_per_req=None, + extend_logprob_start_len_per_req=None, + grammar_advanced=False, + can_run_cuda_graph=False, + skipped_output_comm=False, + ) + processor = _make_processor(self) + + with ( + patch( + "sglang.srt.managers.scheduler_components." + "batch_result_processor.release_kv_cache" + ) as release, + patch.object( + SchedulerBatchResultProcessor, "_maybe_collect_routed_experts" + ), + patch.object(SchedulerBatchResultProcessor, "_maybe_collect_indexer_topk"), + ): + processor.process_batch_result_prefill(batch, result) + + self.assertEqual(release.call_count, 2) + processor.model_worker.prepare_for_kv_cache_release.assert_called_once_with( + decode_req + ) + + class TestDecodeWithoutLogits(CustomTestCase): def test_pipeline_result_commits_token_without_sampling_metadata(self): processor = _make_processor(self)