[MLX] Drain in-flight jobs before a retract-mode pause_generation (#43282)
This commit is contained in:
@@ -114,6 +114,11 @@ class SchedulerMlxOverlapMixin:
|
||||
self.last_batch = pending.schedule_batch
|
||||
self.process_batch_result(pending.batch_copy, result)
|
||||
|
||||
def _drain_mlx_pending_jobs(self: Scheduler):
|
||||
"""Finalize and process every launched job, oldest first."""
|
||||
while self.result_queue:
|
||||
self._finalize_mlx_pending_job(self.result_queue.popleft())
|
||||
|
||||
@DynamicGradMode()
|
||||
def event_loop_overlap_mlx(self: Scheduler):
|
||||
"""MLX-specific overlap loop modelled on ``mlx_lm.generate.generate_step``.
|
||||
@@ -154,8 +159,6 @@ class SchedulerMlxOverlapMixin:
|
||||
are the source of truth and are never merged into a shared
|
||||
batched buffer.
|
||||
"""
|
||||
pending_curr: Optional[MlxPendingJob] = None
|
||||
pending_next: Optional[MlxPendingJob] = None
|
||||
|
||||
def _launch_fresh(batch: ScheduleBatch) -> MlxPendingJob:
|
||||
self._prepare_mlx_launch(batch)
|
||||
@@ -216,6 +219,12 @@ class SchedulerMlxOverlapMixin:
|
||||
self._record_scheduler_state_for_paused_engine()
|
||||
continue
|
||||
|
||||
# In-flight jobs live in self.result_queue across iterations, so
|
||||
# a retract-mode pause_generation can drain them. At most one is
|
||||
# left here.
|
||||
pending_curr = self.result_queue[0] if self.result_queue else None
|
||||
pending_next = None
|
||||
|
||||
# 1. If pending_curr is a pure decode AND no new prefill is waiting,
|
||||
# build pending_next on top of it NOW — before we block on curr.
|
||||
can_chain = (
|
||||
@@ -225,7 +234,7 @@ class SchedulerMlxOverlapMixin:
|
||||
and pending_curr.chain_safe
|
||||
and not self.waiting_queue
|
||||
)
|
||||
if can_chain and pending_next is None:
|
||||
if can_chain:
|
||||
# Build + launch the chained step BEFORE we block on
|
||||
# pending_curr — this is the "no idle gap" trick.
|
||||
# GPU now has 2 steps queued.
|
||||
@@ -237,10 +246,9 @@ class SchedulerMlxOverlapMixin:
|
||||
if pending_curr is not None:
|
||||
self._finalize_mlx_pending_job(pending_curr)
|
||||
self.result_queue.popleft()
|
||||
pending_curr = None
|
||||
|
||||
# 3. Decide whether pending_next is still valid (if no reqs finished)
|
||||
# and promote it.
|
||||
# and promote it: it stays queued as the next pending_curr.
|
||||
finished_any = any(
|
||||
req.finished() for req in (pending_next.reqs if pending_next else [])
|
||||
)
|
||||
@@ -250,10 +258,8 @@ class SchedulerMlxOverlapMixin:
|
||||
and not finished_any
|
||||
and not new_prefill_waiting
|
||||
):
|
||||
pending_curr = pending_next
|
||||
pending_next = None
|
||||
self.cur_batch_for_debug = pending_curr.schedule_batch
|
||||
self.last_batch = pending_curr.schedule_batch
|
||||
self.cur_batch_for_debug = pending_next.schedule_batch
|
||||
self.last_batch = pending_next.schedule_batch
|
||||
if envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.get():
|
||||
self.invariant_checker.self_check_during_busy()
|
||||
continue
|
||||
@@ -263,7 +269,6 @@ class SchedulerMlxOverlapMixin:
|
||||
if pending_next is not None:
|
||||
self._finalize_mlx_pending_job(pending_next)
|
||||
self.result_queue.popleft()
|
||||
pending_next = None
|
||||
plan = self.get_next_batch_to_run(
|
||||
running_batch=self.running_batch, last_batch=self.last_batch
|
||||
)
|
||||
@@ -271,8 +276,7 @@ class SchedulerMlxOverlapMixin:
|
||||
next_batch = plan.batch_to_run
|
||||
self.cur_batch_for_debug = next_batch
|
||||
if next_batch:
|
||||
pending_curr = _launch_fresh(next_batch)
|
||||
self.result_queue.append(pending_curr)
|
||||
self.result_queue.append(_launch_fresh(next_batch))
|
||||
else:
|
||||
self._sched_idled = True
|
||||
self.on_idle()
|
||||
|
||||
@@ -5503,6 +5503,8 @@ class Scheduler(
|
||||
req.tmp_end_idx = min(req.extend_end, len(req.origin_input_ids))
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
elif self.enable_overlap_mlx:
|
||||
self._drain_mlx_pending_jobs()
|
||||
|
||||
retract_reqs = [r for r in self.running_batch.reqs if not r.finished()]
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user