[MLX] Drop a retracted request's runner state at its re-prefill (#43284)
This commit is contained in:
@@ -1678,9 +1678,10 @@ class MlxModelRunner:
|
||||
"""Check if a request has active state."""
|
||||
return req_id in self._req_caches
|
||||
|
||||
def remove_request(self, req_id: str):
|
||||
"""Sync remaining decode KV to pool, then release request state."""
|
||||
if not self.disable_radix_cache:
|
||||
def remove_request(self, req_id: str, sync_kv: bool = True):
|
||||
"""Release request state, first syncing its remaining decode KV to the
|
||||
pool unless *sync_kv* is False (the caller discards that KV)."""
|
||||
if sync_kv and not self.disable_radix_cache:
|
||||
self._sync_decode_kv_to_pool(req_id)
|
||||
|
||||
self._req_token_ids.pop(req_id, None)
|
||||
|
||||
@@ -29,7 +29,7 @@ from sglang.srt.hardware_backend.mlx.sampling import (
|
||||
MlxStepLogprobs,
|
||||
lazy_logprob_arrays,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.managers.utils import GenerationBatchResult
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -118,6 +118,8 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
)
|
||||
|
||||
self._mlx_active_rids: set[str] = set()
|
||||
# rid -> req.retraction_count when its MLX state was created.
|
||||
self._req_retraction_count: dict[str, int] = {}
|
||||
self._mlx_pool_initialized = False
|
||||
|
||||
def get_pad_input_ids_func(self):
|
||||
@@ -161,6 +163,7 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
stale_rids = self._mlx_active_rids - current_rids
|
||||
for rid in stale_rids:
|
||||
self._mlx_runner.remove_request(rid)
|
||||
self._req_retraction_count.pop(rid, None)
|
||||
self._mlx_active_rids = current_rids
|
||||
else:
|
||||
self._mlx_active_rids |= current_rids
|
||||
@@ -173,6 +176,19 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
# insert. Any older tracked slot is released during component cleanup.
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
|
||||
def _drop_state_if_retracted(self, req: Req) -> None:
|
||||
"""Drop the MLX state of a request retracted since its state was made.
|
||||
|
||||
The scheduler freed that KV without a tree insert, so nothing is synced
|
||||
to the pool. Kept state would make the re-prefill route as a
|
||||
continuation and extend the old KV a second time.
|
||||
"""
|
||||
made_at = self._req_retraction_count.get(req.rid)
|
||||
if made_at is not None and made_at != req.retraction_count:
|
||||
del self._req_retraction_count[req.rid]
|
||||
self._mlx_runner.remove_request(req.rid, sync_kv=False)
|
||||
self._mlx_active_rids.discard(req.rid)
|
||||
|
||||
def _route_extend_request(self, rid: str, decoding_rids: set[str]) -> str:
|
||||
"""Classify a request within an extend / mixed batch.
|
||||
|
||||
@@ -477,7 +493,10 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
offset += seq_len
|
||||
slot_offset += seq_len
|
||||
|
||||
self._drop_state_if_retracted(req)
|
||||
route = self._route_extend_request(req.rid, decoding_rids)
|
||||
if route == "prefill":
|
||||
self._req_retraction_count[req.rid] = req.retraction_count
|
||||
if route == "continuation":
|
||||
pending_extends.append(
|
||||
self._mlx_runner.extend_start(
|
||||
|
||||
Reference in New Issue
Block a user