From 8020ea2ee959bff10938eefcaea966192822a763 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 9 Oct 2026 16:16:59 -0700 Subject: [PATCH] [MLX] Drop a retracted request's runner state at its re-prefill (#43284) --- .../srt/hardware_backend/mlx/model_runner.py | 7 +++-- .../srt/hardware_backend/mlx/tp_worker.py | 21 ++++++++++++- .../mlx/test_tp_worker_routing.py | 31 ++++++++++++++++--- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/hardware_backend/mlx/model_runner.py b/python/sglang/srt/hardware_backend/mlx/model_runner.py index 6ab8992f2d..3c7f931a4d 100644 --- a/python/sglang/srt/hardware_backend/mlx/model_runner.py +++ b/python/sglang/srt/hardware_backend/mlx/model_runner.py @@ -1678,9 +1678,10 @@ class MlxModelRunner: """Check if a request has active state.""" return req_id in self._req_caches - def remove_request(self, req_id: str): - """Sync remaining decode KV to pool, then release request state.""" - if not self.disable_radix_cache: + def remove_request(self, req_id: str, sync_kv: bool = True): + """Release request state, first syncing its remaining decode KV to the + pool unless *sync_kv* is False (the caller discards that KV).""" + if sync_kv and not self.disable_radix_cache: self._sync_decode_kv_to_pool(req_id) self._req_token_ids.pop(req_id, None) diff --git a/python/sglang/srt/hardware_backend/mlx/tp_worker.py b/python/sglang/srt/hardware_backend/mlx/tp_worker.py index be7e6f3133..bd166db15b 100644 --- a/python/sglang/srt/hardware_backend/mlx/tp_worker.py +++ b/python/sglang/srt/hardware_backend/mlx/tp_worker.py @@ -29,7 +29,7 @@ from sglang.srt.hardware_backend.mlx.sampling import ( MlxStepLogprobs, lazy_logprob_arrays, ) -from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.managers.schedule_batch import Req, ScheduleBatch from sglang.srt.managers.tp_worker import TpModelWorker from sglang.srt.managers.utils import GenerationBatchResult from sglang.srt.model_executor.forward_batch_info import ( @@ -118,6 +118,8 @@ class MlxTpModelWorker(TpModelWorker): ) self._mlx_active_rids: set[str] = set() + # rid -> req.retraction_count when its MLX state was created. + self._req_retraction_count: dict[str, int] = {} self._mlx_pool_initialized = False def get_pad_input_ids_func(self): @@ -161,6 +163,7 @@ class MlxTpModelWorker(TpModelWorker): stale_rids = self._mlx_active_rids - current_rids for rid in stale_rids: self._mlx_runner.remove_request(rid) + self._req_retraction_count.pop(rid, None) self._mlx_active_rids = current_rids else: self._mlx_active_rids |= current_rids @@ -173,6 +176,19 @@ class MlxTpModelWorker(TpModelWorker): # insert. Any older tracked slot is released during component cleanup. req.kv.mamba_last_track_seqlen = None + def _drop_state_if_retracted(self, req: Req) -> None: + """Drop the MLX state of a request retracted since its state was made. + + The scheduler freed that KV without a tree insert, so nothing is synced + to the pool. Kept state would make the re-prefill route as a + continuation and extend the old KV a second time. + """ + made_at = self._req_retraction_count.get(req.rid) + if made_at is not None and made_at != req.retraction_count: + del self._req_retraction_count[req.rid] + self._mlx_runner.remove_request(req.rid, sync_kv=False) + self._mlx_active_rids.discard(req.rid) + def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str: """Classify a request within an extend / mixed batch. @@ -477,7 +493,10 @@ class MlxTpModelWorker(TpModelWorker): offset += seq_len slot_offset += seq_len + self._drop_state_if_retracted(req) route = self._route_extend_request(req.rid, decoding_rids) + if route == "prefill": + self._req_retraction_count[req.rid] = req.retraction_count if route == "continuation": pending_extends.append( self._mlx_runner.extend_start( diff --git a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py index bb2a2bbcdb..81e91d6bd1 100644 --- a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py +++ b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py @@ -64,6 +64,7 @@ class _FakeRunner: # (op, rid) -> needs_logits as received; guards the worker's # chunk-finality derivation reaching the runner intact. self.logits_flags: dict[tuple[str, str], bool] = {} + self.removed_sync_kv: dict[str, bool] = {} self._req_caches: dict[str, list] = {} self._counter = 0 @@ -74,6 +75,11 @@ class _FakeRunner: def flush_all_decode_kv(self): pass + def remove_request(self, rid, sync_kv=True): + self.calls.append(("remove_request", rid)) + self.removed_sync_kv[rid] = sync_kv + self._known.discard(rid) + def ops_for(self, rid): return [op for op, r in self.calls if r == rid] @@ -167,6 +173,7 @@ class _FakeRunner: class _FakeReq: def __init__(self, rid, req_pool_idx=0): self.rid = rid + self.retraction_count = 0 self.prefix_len = 0 self.fill_ids = [0] self.kv = ReqKvInfo(req_pool_idx=req_pool_idx) @@ -214,6 +221,7 @@ class TestMlxExtendRouting(CustomTestCase): worker = MlxTpModelWorker.__new__(MlxTpModelWorker) worker._mlx_runner = _FakeRunner(known_rids) worker._mlx_active_rids = set() + worker._req_retraction_count = {} # The sync entry point delegates to the async launch, which guards # pool creation behind this flag; forward_batch_generation has # already run it for real by the time either path is reached. @@ -254,6 +262,24 @@ class TestMlxExtendRouting(CustomTestCase): worker = self._worker(known_rids={"r1"}) self.assertEqual(worker._route_extend_request("r1", {"r1"}), "decode") + # ---------- retraction ---------- + + def test_retracted_request_reprefills_from_scratch(self): + """A request retracted after its MLX state was made drops that state + without a pool sync, so its re-prefill routes as a fresh prefill.""" + worker = self._worker(known_rids=set()) + req = _FakeReq("r1") + worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4])) + worker._mlx_runner._known.add("r1") + + req.retraction_count += 1 + worker._async_extend_batch(_FakeBatch(ForwardMode.EXTEND, [req], [4])) + self.assertEqual(worker._mlx_runner.removed_sync_kv, {"r1": False}) + self.assertEqual( + worker._mlx_runner.ops_for("r1"), + ["prefill_start", "remove_request", "prefill_start"], + ) + # ---------- sync path: _forward_batch_generation_mlx ---------- def _run_sync(self, reqs, extend_lens, known_rids, decoding_reqs, forward_mode): @@ -292,10 +318,7 @@ class TestMlxExtendRouting(CustomTestCase): # ---------- async path: _async_extend_batch ---------- def _run_async(self, reqs, extend_lens, known_rids, decoding_reqs, forward_mode): - from sglang.srt.hardware_backend.mlx.tp_worker import MlxTpModelWorker - - worker = MlxTpModelWorker.__new__(MlxTpModelWorker) - worker._mlx_runner = _FakeRunner(known_rids) + worker = self._worker(known_rids) batch = _FakeBatch(forward_mode, reqs, extend_lens, decoding_reqs) launch = worker._async_extend_batch(batch) return worker._mlx_runner, launch