[MLX] Drop a retracted request's runner state at its re-prefill (#43284)

This commit is contained in:
Liangsheng Yin
2026-10-09 16:16:59 -07:00
committed by GitHub
parent f7e601b936
commit 8020ea2ee9
3 changed files with 51 additions and 8 deletions
@@ -1678,9 +1678,10 @@ class MlxModelRunner:
"""Check if a request has active state."""
return req_id in self._req_caches
def remove_request(self, req_id: str):
"""Sync remaining decode KV to pool, then release request state."""
if not self.disable_radix_cache:
def remove_request(self, req_id: str, sync_kv: bool = True):
"""Release request state, first syncing its remaining decode KV to the
pool unless *sync_kv* is False (the caller discards that KV)."""
if sync_kv and not self.disable_radix_cache:
self._sync_decode_kv_to_pool(req_id)
self._req_token_ids.pop(req_id, None)
@@ -29,7 +29,7 @@ from sglang.srt.hardware_backend.mlx.sampling import (
MlxStepLogprobs,
lazy_logprob_arrays,
)
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.managers.utils import GenerationBatchResult
from sglang.srt.model_executor.forward_batch_info import (
@@ -118,6 +118,8 @@ class MlxTpModelWorker(TpModelWorker):
)
self._mlx_active_rids: set[str] = set()
# rid -> req.retraction_count when its MLX state was created.
self._req_retraction_count: dict[str, int] = {}
self._mlx_pool_initialized = False
def get_pad_input_ids_func(self):
@@ -161,6 +163,7 @@ class MlxTpModelWorker(TpModelWorker):
stale_rids = self._mlx_active_rids - current_rids
for rid in stale_rids:
self._mlx_runner.remove_request(rid)
self._req_retraction_count.pop(rid, None)
self._mlx_active_rids = current_rids
else:
self._mlx_active_rids |= current_rids
@@ -173,6 +176,19 @@ class MlxTpModelWorker(TpModelWorker):
# insert. Any older tracked slot is released during component cleanup.
req.kv.mamba_last_track_seqlen = None
def _drop_state_if_retracted(self, req: Req) -> None:
"""Drop the MLX state of a request retracted since its state was made.
The scheduler freed that KV without a tree insert, so nothing is synced
to the pool. Kept state would make the re-prefill route as a
continuation and extend the old KV a second time.
"""
made_at = self._req_retraction_count.get(req.rid)
if made_at is not None and made_at != req.retraction_count:
del self._req_retraction_count[req.rid]
self._mlx_runner.remove_request(req.rid, sync_kv=False)
self._mlx_active_rids.discard(req.rid)
def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str:
"""Classify a request within an extend / mixed batch.
@@ -477,7 +493,10 @@ class MlxTpModelWorker(TpModelWorker):
offset += seq_len
slot_offset += seq_len
self._drop_state_if_retracted(req)
route = self._route_extend_request(req.rid, decoding_rids)
if route == "prefill":
self._req_retraction_count[req.rid] = req.retraction_count
if route == "continuation":
pending_extends.append(
self._mlx_runner.extend_start(