[MLX] Sync decode KV to the pool once at release, clamped to the owned prefix (#43292)
This commit is contained in:
@@ -839,14 +839,17 @@ class MlxModelRunner:
|
||||
)
|
||||
self._attention_kv_pool.set_kv_all_layers(slot_ids_mx, k_all, v_all)
|
||||
|
||||
def _sync_decode_kv_to_pool(self, req_id: str) -> None:
|
||||
"""Sync un-flushed decode KV for *req_id* to the shared pool."""
|
||||
def _sync_decode_kv_to_pool(self, req_id: str, end: int | None = None) -> None:
|
||||
"""Sync un-flushed decode KV for *req_id* to the shared pool, up to
|
||||
position *end* when given."""
|
||||
if self._attention_kv_pool is None or self._req_to_token_pool is None:
|
||||
return
|
||||
cache = self._req_caches.get(req_id)
|
||||
if cache is None:
|
||||
return
|
||||
current_offset = self._first_attention_cache(cache).offset
|
||||
if end is not None:
|
||||
current_offset = min(current_offset, end)
|
||||
synced_offset = self._req_synced_offset.get(req_id, 0)
|
||||
if current_offset <= synced_offset:
|
||||
return
|
||||
@@ -864,12 +867,13 @@ class MlxModelRunner:
|
||||
self._sync_new_kv_to_pool(cache, synced_offset, slot_ids)
|
||||
self._req_synced_offset[req_id] = current_offset
|
||||
|
||||
def flush_all_decode_kv(self) -> None:
|
||||
"""Sync all active requests' un-flushed decode KV to the pool."""
|
||||
if self.disable_radix_cache or self._attention_kv_pool is None:
|
||||
return
|
||||
for req_id in list(self._req_caches.keys()):
|
||||
self._sync_decode_kv_to_pool(req_id)
|
||||
def release_request_row(self, req_id: str, owned_len: int) -> None:
|
||||
"""Sync decode KV only within the first *owned_len* positions (chained
|
||||
decode steps own no slots past it), then forget the row, which the
|
||||
scheduler may reuse once released."""
|
||||
if not self.disable_radix_cache:
|
||||
self._sync_decode_kv_to_pool(req_id, end=owned_len)
|
||||
self._req_pool_idx.pop(req_id, None)
|
||||
|
||||
def decode_batch(self, req_ids: list[str]) -> list[int]:
|
||||
"""Decode one token per request.
|
||||
@@ -1678,12 +1682,9 @@ class MlxModelRunner:
|
||||
"""Check if a request has active state."""
|
||||
return req_id in self._req_caches
|
||||
|
||||
def remove_request(self, req_id: str, sync_kv: bool = True):
|
||||
"""Release request state, first syncing its remaining decode KV to the
|
||||
pool unless *sync_kv* is False (the caller discards that KV)."""
|
||||
if sync_kv and not self.disable_radix_cache:
|
||||
self._sync_decode_kv_to_pool(req_id)
|
||||
|
||||
def remove_request(self, req_id: str):
|
||||
"""Release request state without syncing: decode KV reaches the pool
|
||||
only through release_request_row, while the request owns its row."""
|
||||
self._req_token_ids.pop(req_id, None)
|
||||
self._req_sampling.pop(req_id, None)
|
||||
cache = self._req_caches.pop(req_id, None)
|
||||
|
||||
@@ -169,12 +169,14 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
self._mlx_active_rids |= current_rids
|
||||
|
||||
def prepare_for_kv_cache_release(self, req) -> None:
|
||||
"""Snapshot MLX auxiliary state at the scheduler's radix insert point."""
|
||||
"""Write the request's MLX state to the pool at the scheduler's radix
|
||||
insert point, while the request still owns its req_to_token row."""
|
||||
if self._mlx_runner.has_request(req.rid):
|
||||
self._mlx_runner.store_auxiliary_state_for_request(req.rid)
|
||||
# Prefer the just-snapshotted live auxiliary state for the final
|
||||
# insert. Any older tracked slot is released during component cleanup.
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
self._mlx_runner.release_request_row(req.rid, req.owned_kv_len())
|
||||
|
||||
def _drop_state_if_retracted(self, req: Req) -> None:
|
||||
"""Drop the MLX state of a request retracted since its state was made.
|
||||
@@ -186,7 +188,7 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
made_at = self._req_retraction_count.get(req.rid)
|
||||
if made_at is not None and made_at != req.retraction_count:
|
||||
del self._req_retraction_count[req.rid]
|
||||
self._mlx_runner.remove_request(req.rid, sync_kv=False)
|
||||
self._mlx_runner.remove_request(req.rid)
|
||||
self._mlx_active_rids.discard(req.rid)
|
||||
|
||||
def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str:
|
||||
@@ -458,10 +460,6 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
)
|
||||
|
||||
if forward_mode.is_extend():
|
||||
# TODO (changminbark): Implement per-batch flushing using prefix_slot_ids
|
||||
# Ensure the pool is up-to-date before pool-backed attention
|
||||
# reads it for prefix-cached prefills. Mirror the sync path.
|
||||
self._mlx_runner.flush_all_decode_kv()
|
||||
return self._async_extend_batch(batch)
|
||||
|
||||
raise ValueError(
|
||||
|
||||
@@ -374,6 +374,8 @@ class SchedulerBatchResultProcessor:
|
||||
if sampling_mask_finish_reason is None:
|
||||
self._maybe_collect_routed_experts(req)
|
||||
self._maybe_collect_indexer_topk(req)
|
||||
if batch.decoding_reqs and req in batch.decoding_reqs:
|
||||
self._prepare_kv_cache_release(req)
|
||||
release_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
@@ -1323,6 +1325,14 @@ class SchedulerBatchResultProcessor:
|
||||
"InternalServerError",
|
||||
)
|
||||
|
||||
def _prepare_kv_cache_release(self, req: Req) -> None:
|
||||
# Only workers that keep KV outside the pool (MLX) define the hook.
|
||||
prepare_release = getattr(
|
||||
self.model_worker, "prepare_for_kv_cache_release", None
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
|
||||
def _handle_sampling_mask_abort(self, req: Req) -> None:
|
||||
"""Release a request whose sampled token must not be committed."""
|
||||
if req.multimodal_inputs is not None and req.session is None:
|
||||
@@ -1332,11 +1342,7 @@ class SchedulerBatchResultProcessor:
|
||||
else:
|
||||
if get_memory().enable_hisparse:
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
prepare_release = getattr(
|
||||
self.model_worker, "prepare_for_kv_cache_release", None
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
self._prepare_kv_cache_release(req)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
req.time_stats.set_completion_time()
|
||||
|
||||
@@ -1419,11 +1425,7 @@ class SchedulerBatchResultProcessor:
|
||||
else:
|
||||
if get_memory().enable_hisparse:
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
prepare_release = getattr(
|
||||
self.model_worker, "prepare_for_kv_cache_release", None
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
self._prepare_kv_cache_release(req)
|
||||
checkpoint = (
|
||||
req.mamba_lazy_checkpoint
|
||||
if get_exec().mamba.enable_mamba_extra_buffer_lazy
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Decode KV must never be written through req_to_token entries past
|
||||
``owned_kv_len()``, nor after the request's row is released."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.ci.ci_register import register_mlx_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_mlx_ci(est_time=5, suite="stage-a-unit-test-mlx")
|
||||
|
||||
_HAS_MLX = (
|
||||
importlib.util.find_spec("mlx") is not None
|
||||
and importlib.util.find_spec("mlx_lm") is not None
|
||||
)
|
||||
|
||||
if _HAS_MLX:
|
||||
import mlx.core as mx
|
||||
import torch
|
||||
from mlx_lm.models import llama
|
||||
|
||||
from sglang.srt.hardware_backend.mlx.aot import MlxAOTKernelSet
|
||||
from sglang.srt.hardware_backend.mlx.kv_cache import (
|
||||
find_attention_layers,
|
||||
get_layer_window_sizes,
|
||||
patch_model_attention,
|
||||
)
|
||||
from sglang.srt.hardware_backend.mlx.kv_cache.layout import MlxModelCacheLayout
|
||||
from sglang.srt.hardware_backend.mlx.model_runner import MlxModelRunner
|
||||
from sglang.srt.hardware_backend.mlx.tp_worker import MlxTpModelWorker
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
|
||||
PROMPT_LEN = 5
|
||||
DECODE_STEPS = 5
|
||||
OWN_SLOTS = [1, 2, 3, 4, 5, 6] # prompt slots + the one allocated decode slot
|
||||
FOREIGN_SLOTS = [20, 21, 22, 23] # stale row entries left by a previous owner
|
||||
|
||||
|
||||
def _runner(req_to_token_pool):
|
||||
model = llama.Model(
|
||||
llama.ModelArgs(
|
||||
model_type="llama",
|
||||
hidden_size=32,
|
||||
num_hidden_layers=2,
|
||||
intermediate_size=64,
|
||||
num_attention_heads=2,
|
||||
rms_norm_eps=1e-5,
|
||||
vocab_size=64,
|
||||
head_dim=16,
|
||||
num_key_value_heads=1,
|
||||
)
|
||||
)
|
||||
patch_model_attention(model)
|
||||
layers, attrs = find_attention_layers(model)
|
||||
runner = MlxModelRunner.__new__(MlxModelRunner)
|
||||
runner.model = model
|
||||
runner.disable_radix_cache = False
|
||||
runner._cache_layout = MlxModelCacheLayout.from_attention_discovery(
|
||||
layers, attrs, layer_window_sizes=get_layer_window_sizes(model)
|
||||
)
|
||||
runner._max_seq_len = 64
|
||||
runner._cache_pool = []
|
||||
runner._req_caches = {}
|
||||
runner._req_token_ids = {}
|
||||
runner._req_sampling = {}
|
||||
runner._req_pool_idx = {}
|
||||
runner._req_synced_offset = {}
|
||||
runner._attention_kv_pool = None
|
||||
runner._decode_step_ct = 0
|
||||
runner._clear_steps = 0
|
||||
runner._aot_kernels = MlxAOTKernelSet()
|
||||
runner._pool_size = 32
|
||||
runner.init_cache_pools(req_to_token_pool)
|
||||
return runner
|
||||
|
||||
|
||||
@unittest.skipUnless(_HAS_MLX, "requires mlx + mlx_lm")
|
||||
class TestDecodeKvRelease(CustomTestCase):
|
||||
def _decoded_runner(self):
|
||||
req_to_token_pool = ReqToTokenPool(
|
||||
size=2, max_context_len=64, device="cpu", enable_memory_saver=False
|
||||
)
|
||||
row = req_to_token_pool.req_to_token[0]
|
||||
row[: len(OWN_SLOTS)] = torch.tensor(OWN_SLOTS)
|
||||
row[len(OWN_SLOTS) : len(OWN_SLOTS) + len(FOREIGN_SLOTS)] = torch.tensor(
|
||||
FOREIGN_SLOTS
|
||||
)
|
||||
runner = _runner(req_to_token_pool)
|
||||
prompt = [3, 14, 15, 9, 2]
|
||||
runner.prefill(
|
||||
req_id="r",
|
||||
new_token_ids=prompt,
|
||||
full_token_ids=prompt,
|
||||
prefix_slot_ids=[],
|
||||
new_slot_ids=OWN_SLOTS[:PROMPT_LEN],
|
||||
req_pool_idx=0,
|
||||
)
|
||||
for _ in range(DECODE_STEPS):
|
||||
runner.decode_batch(["r"])
|
||||
return runner
|
||||
|
||||
def _assert_foreign_slots_untouched(self, runner):
|
||||
pool = runner._attention_kv_pool
|
||||
foreign_k, foreign_v = pool.get_kv(0, mx.array(FOREIGN_SLOTS, dtype=mx.int32))
|
||||
self.assertEqual(mx.abs(foreign_k).max().item(), 0.0)
|
||||
self.assertEqual(mx.abs(foreign_v).max().item(), 0.0)
|
||||
|
||||
def test_release_syncs_owned_prefix_and_nothing_after(self):
|
||||
runner = self._decoded_runner()
|
||||
worker = MlxTpModelWorker.__new__(MlxTpModelWorker)
|
||||
worker._mlx_runner = runner
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
kv=SimpleNamespace(mamba_last_track_seqlen=None),
|
||||
owned_kv_len=lambda: len(OWN_SLOTS),
|
||||
)
|
||||
worker.prepare_for_kv_cache_release(req)
|
||||
# The row goes back to the scheduler; the request's MLX state is
|
||||
# dropped later, when it leaves the running batch.
|
||||
runner.remove_request("r")
|
||||
|
||||
own_k, _ = runner._attention_kv_pool.get_kv(
|
||||
0, mx.array([OWN_SLOTS[-1]], dtype=mx.int32)
|
||||
)
|
||||
self.assertGreater(mx.abs(own_k).max().item(), 0.0)
|
||||
self._assert_foreign_slots_untouched(runner)
|
||||
|
||||
def test_retracted_request_writes_nothing(self):
|
||||
# A retraction frees the row with no release hook; dropping the
|
||||
# state afterwards must not write through the (reusable) row.
|
||||
runner = self._decoded_runner()
|
||||
runner.remove_request("r")
|
||||
self._assert_foreign_slots_untouched(runner)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -65,7 +65,6 @@ class _FakeRunner:
|
||||
# chunk-finality derivation reaching the runner intact.
|
||||
self.logits_flags: dict[tuple[str, str], bool] = {}
|
||||
self.prefix_slot_ids: dict[str, list[int]] = {}
|
||||
self.removed_sync_kv: dict[str, bool] = {}
|
||||
self._req_caches: dict[str, list] = {}
|
||||
self._counter = 0
|
||||
|
||||
@@ -73,12 +72,8 @@ class _FakeRunner:
|
||||
def has_request(self, rid):
|
||||
return rid in self._known
|
||||
|
||||
def flush_all_decode_kv(self):
|
||||
pass
|
||||
|
||||
def remove_request(self, rid, sync_kv=True):
|
||||
def remove_request(self, rid):
|
||||
self.calls.append(("remove_request", rid))
|
||||
self.removed_sync_kv[rid] = sync_kv
|
||||
self._known.discard(rid)
|
||||
|
||||
def ops_for(self, rid):
|
||||
@@ -267,8 +262,8 @@ class TestMlxExtendRouting(CustomTestCase):
|
||||
# ---------- retraction ----------
|
||||
|
||||
def test_retracted_request_reprefills_from_scratch(self):
|
||||
"""A request retracted after its MLX state was made drops that state
|
||||
without a pool sync, so its re-prefill routes as a fresh prefill."""
|
||||
"""A request retracted after its MLX state was made drops that state,
|
||||
so its re-prefill routes as a fresh prefill."""
|
||||
worker = self._worker(known_rids=set())
|
||||
req = _FakeReq("r1")
|
||||
worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4]))
|
||||
@@ -276,7 +271,6 @@ class TestMlxExtendRouting(CustomTestCase):
|
||||
|
||||
req.retraction_count += 1
|
||||
worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4]))
|
||||
self.assertEqual(worker._mlx_runner.removed_sync_kv, {"r1": False})
|
||||
self.assertEqual(
|
||||
worker._mlx_runner.ops_for("r1"),
|
||||
["prefill_start", "remove_request", "prefill_start"],
|
||||
|
||||
@@ -318,6 +318,66 @@ class TestPrefillSkippedOutput(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestMixedBatchKvRelease(CustomTestCase):
|
||||
def test_decode_req_finishing_in_mixed_batch_gets_release_hook(self):
|
||||
"""A decode req that finishes inside a mixed batch goes through the
|
||||
worker's pre-release hook, as it would in a decode batch; a prefill
|
||||
req finishing on its first token does not."""
|
||||
prefill_req = _PrefillReq(
|
||||
rid="prefill", inflight_middle_chunks=0, return_hidden_states=False
|
||||
)
|
||||
decode_req = _PrefillReq(
|
||||
rid="decode", inflight_middle_chunks=0, return_hidden_states=False
|
||||
)
|
||||
for req in (prefill_req, decode_req):
|
||||
req.finished = lambda req=req: bool(req.output_ids)
|
||||
req.kv = SimpleNamespace(kv_committed_len=0)
|
||||
batch = SimpleNamespace(
|
||||
reqs=[prefill_req, decode_req],
|
||||
decoding_reqs=[decode_req],
|
||||
return_logprob=False,
|
||||
return_hidden_states=False,
|
||||
return_hidden_states_mode=CaptureHiddenMode.NULL,
|
||||
spec_info=None,
|
||||
spec_algorithm=SimpleNamespace(is_none=lambda: True),
|
||||
prefill_stats=None,
|
||||
dp_cooperation_info=None,
|
||||
)
|
||||
result = SimpleNamespace(
|
||||
copy_done=None,
|
||||
auxiliary_host_output=None,
|
||||
routed_experts_output=None,
|
||||
indexer_topk_output=None,
|
||||
logits_output=SimpleNamespace(
|
||||
hidden_states=None, customized_info=None, sampling_mask_output=None
|
||||
),
|
||||
next_token_ids=torch.tensor([5, 6]),
|
||||
extend_input_len_per_req=None,
|
||||
extend_logprob_start_len_per_req=None,
|
||||
grammar_advanced=False,
|
||||
can_run_cuda_graph=False,
|
||||
skipped_output_comm=False,
|
||||
)
|
||||
processor = _make_processor(self)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components."
|
||||
"batch_result_processor.release_kv_cache"
|
||||
) as release,
|
||||
patch.object(
|
||||
SchedulerBatchResultProcessor, "_maybe_collect_routed_experts"
|
||||
),
|
||||
patch.object(SchedulerBatchResultProcessor, "_maybe_collect_indexer_topk"),
|
||||
):
|
||||
processor.process_batch_result_prefill(batch, result)
|
||||
|
||||
self.assertEqual(release.call_count, 2)
|
||||
processor.model_worker.prepare_for_kv_cache_release.assert_called_once_with(
|
||||
decode_req
|
||||
)
|
||||
|
||||
|
||||
class TestDecodeWithoutLogits(CustomTestCase):
|
||||
def test_pipeline_result_commits_token_without_sampling_metadata(self):
|
||||
processor = _make_processor(self)
|
||||
|
||||
Reference in New Issue
Block a user