[MLX] Drain in-flight jobs before a retract-mode pause_generation (#43282)

This commit is contained in:
Liangsheng Yin
2026-10-09 16:14:23 -07:00
committed by GitHub
parent f6c848c882
commit f7e601b936
5 changed files with 165 additions and 12 deletions
@@ -114,6 +114,11 @@ class SchedulerMlxOverlapMixin:
self.last_batch = pending.schedule_batch
self.process_batch_result(pending.batch_copy, result)
def _drain_mlx_pending_jobs(self: Scheduler):
"""Finalize and process every launched job, oldest first."""
while self.result_queue:
self._finalize_mlx_pending_job(self.result_queue.popleft())
@DynamicGradMode()
def event_loop_overlap_mlx(self: Scheduler):
"""MLX-specific overlap loop modelled on ``mlx_lm.generate.generate_step``.
@@ -154,8 +159,6 @@ class SchedulerMlxOverlapMixin:
are the source of truth and are never merged into a shared
batched buffer.
"""
pending_curr: Optional[MlxPendingJob] = None
pending_next: Optional[MlxPendingJob] = None
def _launch_fresh(batch: ScheduleBatch) -> MlxPendingJob:
self._prepare_mlx_launch(batch)
@@ -216,6 +219,12 @@ class SchedulerMlxOverlapMixin:
self._record_scheduler_state_for_paused_engine()
continue
# In-flight jobs live in self.result_queue across iterations, so
# a retract-mode pause_generation can drain them. At most one is
# left here.
pending_curr = self.result_queue[0] if self.result_queue else None
pending_next = None
# 1. If pending_curr is a pure decode AND no new prefill is waiting,
# build pending_next on top of it NOW — before we block on curr.
can_chain = (
@@ -225,7 +234,7 @@ class SchedulerMlxOverlapMixin:
and pending_curr.chain_safe
and not self.waiting_queue
)
if can_chain and pending_next is None:
if can_chain:
# Build + launch the chained step BEFORE we block on
# pending_curr — this is the "no idle gap" trick.
# GPU now has 2 steps queued.
@@ -237,10 +246,9 @@ class SchedulerMlxOverlapMixin:
if pending_curr is not None:
self._finalize_mlx_pending_job(pending_curr)
self.result_queue.popleft()
pending_curr = None
# 3. Decide whether pending_next is still valid (if no reqs finished)
# and promote it.
# and promote it: it stays queued as the next pending_curr.
finished_any = any(
req.finished() for req in (pending_next.reqs if pending_next else [])
)
@@ -250,10 +258,8 @@ class SchedulerMlxOverlapMixin:
and not finished_any
and not new_prefill_waiting
):
pending_curr = pending_next
pending_next = None
self.cur_batch_for_debug = pending_curr.schedule_batch
self.last_batch = pending_curr.schedule_batch
self.cur_batch_for_debug = pending_next.schedule_batch
self.last_batch = pending_next.schedule_batch
if envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.get():
self.invariant_checker.self_check_during_busy()
continue
@@ -263,7 +269,6 @@ class SchedulerMlxOverlapMixin:
if pending_next is not None:
self._finalize_mlx_pending_job(pending_next)
self.result_queue.popleft()
pending_next = None
plan = self.get_next_batch_to_run(
running_batch=self.running_batch, last_batch=self.last_batch
)
@@ -271,8 +276,7 @@ class SchedulerMlxOverlapMixin:
next_batch = plan.batch_to_run
self.cur_batch_for_debug = next_batch
if next_batch:
pending_curr = _launch_fresh(next_batch)
self.result_queue.append(pending_curr)
self.result_queue.append(_launch_fresh(next_batch))
else:
self._sched_idled = True
self.on_idle()
+2
View File
@@ -5503,6 +5503,8 @@ class Scheduler(
req.tmp_end_idx = min(req.extend_end, len(req.origin_input_ids))
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result(tmp_batch, tmp_result)
elif self.enable_overlap_mlx:
self._drain_mlx_pending_jobs()
retract_reqs = [r for r in self.running_batch.reqs if not r.finished()]
if (