From f7e601b93664b78f003c037f58d4cb01a9cec809 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 9 Oct 2026 16:14:23 -0700 Subject: [PATCH] [MLX] Drain in-flight jobs before a retract-mode `pause_generation` (#43282) --- .../hardware_backend/mlx/scheduler_mixin.py | 28 ++-- python/sglang/srt/managers/scheduler.py | 2 + .../test_prefill_result_polling.py | 1 + .../mlx/test_scheduler_mixin.py | 145 ++++++++++++++++++ .../test_scheduler_pause_generation.py | 1 + 5 files changed, 165 insertions(+), 12 deletions(-) diff --git a/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py b/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py index 0e754870df..187f140a3f 100644 --- a/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py +++ b/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py @@ -114,6 +114,11 @@ class SchedulerMlxOverlapMixin: self.last_batch = pending.schedule_batch self.process_batch_result(pending.batch_copy, result) + def _drain_mlx_pending_jobs(self: Scheduler): + """Finalize and process every launched job, oldest first.""" + while self.result_queue: + self._finalize_mlx_pending_job(self.result_queue.popleft()) + @DynamicGradMode() def event_loop_overlap_mlx(self: Scheduler): """MLX-specific overlap loop modelled on ``mlx_lm.generate.generate_step``. @@ -154,8 +159,6 @@ class SchedulerMlxOverlapMixin: are the source of truth and are never merged into a shared batched buffer. """ - pending_curr: Optional[MlxPendingJob] = None - pending_next: Optional[MlxPendingJob] = None def _launch_fresh(batch: ScheduleBatch) -> MlxPendingJob: self._prepare_mlx_launch(batch) @@ -216,6 +219,12 @@ class SchedulerMlxOverlapMixin: self._record_scheduler_state_for_paused_engine() continue + # In-flight jobs live in self.result_queue across iterations, so + # a retract-mode pause_generation can drain them. At most one is + # left here. + pending_curr = self.result_queue[0] if self.result_queue else None + pending_next = None + # 1. If pending_curr is a pure decode AND no new prefill is waiting, # build pending_next on top of it NOW — before we block on curr. can_chain = ( @@ -225,7 +234,7 @@ class SchedulerMlxOverlapMixin: and pending_curr.chain_safe and not self.waiting_queue ) - if can_chain and pending_next is None: + if can_chain: # Build + launch the chained step BEFORE we block on # pending_curr — this is the "no idle gap" trick. # GPU now has 2 steps queued. @@ -237,10 +246,9 @@ class SchedulerMlxOverlapMixin: if pending_curr is not None: self._finalize_mlx_pending_job(pending_curr) self.result_queue.popleft() - pending_curr = None # 3. Decide whether pending_next is still valid (if no reqs finished) - # and promote it. + # and promote it: it stays queued as the next pending_curr. finished_any = any( req.finished() for req in (pending_next.reqs if pending_next else []) ) @@ -250,10 +258,8 @@ class SchedulerMlxOverlapMixin: and not finished_any and not new_prefill_waiting ): - pending_curr = pending_next - pending_next = None - self.cur_batch_for_debug = pending_curr.schedule_batch - self.last_batch = pending_curr.schedule_batch + self.cur_batch_for_debug = pending_next.schedule_batch + self.last_batch = pending_next.schedule_batch if envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.get(): self.invariant_checker.self_check_during_busy() continue @@ -263,7 +269,6 @@ class SchedulerMlxOverlapMixin: if pending_next is not None: self._finalize_mlx_pending_job(pending_next) self.result_queue.popleft() - pending_next = None plan = self.get_next_batch_to_run( running_batch=self.running_batch, last_batch=self.last_batch ) @@ -271,8 +276,7 @@ class SchedulerMlxOverlapMixin: next_batch = plan.batch_to_run self.cur_batch_for_debug = next_batch if next_batch: - pending_curr = _launch_fresh(next_batch) - self.result_queue.append(pending_curr) + self.result_queue.append(_launch_fresh(next_batch)) else: self._sched_idled = True self.on_idle() diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 5875904fa0..427ffb6864 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -5503,6 +5503,8 @@ class Scheduler( req.tmp_end_idx = min(req.extend_end, len(req.origin_input_ids)) tmp_batch, tmp_result = self.result_queue.popleft() self.process_batch_result(tmp_batch, tmp_result) + elif self.enable_overlap_mlx: + self._drain_mlx_pending_jobs() retract_reqs = [r for r in self.running_batch.reqs if not r.finished()] if ( diff --git a/test/registered/unit/disaggregation/test_prefill_result_polling.py b/test/registered/unit/disaggregation/test_prefill_result_polling.py index 0f005a4625..c2577a9e34 100644 --- a/test/registered/unit/disaggregation/test_prefill_result_polling.py +++ b/test/registered/unit/disaggregation/test_prefill_result_polling.py @@ -169,6 +169,7 @@ class _Scheduler(SchedulerDisaggregationPrefillMixin): self.gracefully_exit = False self.enable_staging = False self.enable_overlap = True + self.enable_overlap_mlx = False self.disagg_prefill_bootstrap_queue = SimpleNamespace( pop_bootstrapped=lambda: [] ) diff --git a/test/registered/unit/hardware_backend/mlx/test_scheduler_mixin.py b/test/registered/unit/hardware_backend/mlx/test_scheduler_mixin.py index 4a1ec2ea3f..234fa0bb4d 100644 --- a/test/registered/unit/hardware_backend/mlx/test_scheduler_mixin.py +++ b/test/registered/unit/hardware_backend/mlx/test_scheduler_mixin.py @@ -4,6 +4,7 @@ Covers: - Every MLX launch advances forward_ct and stamps forward_iter/launch_ts. - The profiler predicate runs before the async forward is enqueued, matching Scheduler.run_batch() so step-bounded profiling stops on the right step. + - A retract-mode pause processes the in-flight MLX job before retracting. Skips on non-Apple-Silicon platforms and when ``mlx`` is missing (importing scheduler_mixin requires ``mlx.core``). @@ -327,5 +328,149 @@ class TestOverlapLoopGracefulExit(unittest.TestCase): synchronize.assert_called_once_with() +@unittest.skipUnless(_IS_APPLE_SILICON and _HAS_MLX, _SKIP_REASON) +class TestRetractPauseDrainsInflightJob(unittest.TestCase): + """``pause_generation(mode="retract")`` must process the in-flight MLX job first. + + The MLX overlap loop leaves a launched job queued in ``result_queue`` at + the top of each iteration. A retract-mode pause that does not drain it + retracts the job's requests and frees their KV, and on resume the loop + processes the stale job for requests that are back in the waiting queue. + """ + + def _make_scheduler(self, *, ingest_steps, events): + from collections import deque + + from sglang.srt.disaggregation.utils import DisaggregationMode + from sglang.srt.hardware_backend.mlx.scheduler_mixin import ( + SchedulerMlxOverlapMixin, + ) + from sglang.srt.hardware_backend.mlx.tp_worker import MlxLaunch + from sglang.srt.managers.scheduler import Scheduler + + scheduler = MagicMock() + scheduler.forward_ct = 0 + scheduler._sched_idled = False + scheduler.gracefully_exit = False + scheduler._engine_paused = False + scheduler.enable_overlap = False + scheduler.enable_overlap_mlx = True + scheduler.disaggregation_mode = DisaggregationMode.NULL + scheduler.chunked_req = None + scheduler.decode_offload_manager = None + scheduler.last_batch = None + scheduler.waiting_queue = [] + scheduler.result_queue = deque() + scheduler.metrics_reporter.current_scheduler_metrics_enabled = False + for name in ( + "_prepare_mlx_launch", + "_finalize_mlx_pending_job", + "_drain_mlx_pending_jobs", + ): + getattr(scheduler, name).side_effect = getattr( + SchedulerMlxOverlapMixin, name + ).__get__(scheduler) + scheduler.pause_generation.side_effect = Scheduler.pause_generation.__get__( + scheduler + ) + + req = MagicMock() + req.rid = "r1" + req.finished.return_value = False + batch = MagicMock() + batch.reqs = [req] + batch.return_logprob = False + batch.forward_mode.is_extend.return_value = False + batch.copy.side_effect = lambda: MagicMock(reqs=list(batch.reqs)) + scheduler.running_batch = batch + + def get_next_batch_to_run(running_batch, last_batch): + plan = MagicMock() + plan.running_batch = running_batch + plan.batch_to_run = running_batch if running_batch.reqs else None + return plan + + scheduler.get_next_batch_to_run.side_effect = get_next_batch_to_run + # A decode launch without a chainable handle: one job in flight. + scheduler.tp_worker.async_forward_batch_generation_mlx.side_effect = lambda b: ( + events.append(("launch", [r.rid for r in b.reqs])), + MlxLaunch( + lazy_tokens=None, + prefills=[], + extends=[], + decode=None, + mode="decode", + ), + )[1] + result = MagicMock() + result.next_token_ids = None + scheduler.tp_worker.finalize_mlx_result.return_value = result + scheduler.process_batch_result.side_effect = lambda b, _r: events.append( + ("process", [r.rid for r in b.reqs]) + ) + + steps = iter(ingest_steps) + + def ingest_requests(): + step = next(steps, None) + if step is None: + raise _StopLoop() + step(scheduler) + + scheduler.ingest_requests.side_effect = ingest_requests + return scheduler + + def _run_loop(self, scheduler, events): + from sglang.srt.hardware_backend.mlx.scheduler_mixin import ( + SchedulerMlxOverlapMixin, + ) + + def fake_retract_all(reqs, **_kwargs): + events.append(("retract", [r.rid for r in reqs])) + + with ( + patch("sglang.srt.managers.scheduler.retract_all", fake_retract_all), + patch( + "sglang.srt.hardware_backend.mlx.scheduler_mixin.resolve_forward_inputs" + ), + self.assertRaises(_StopLoop), + ): + SchedulerMlxOverlapMixin.event_loop_overlap_mlx(scheduler) + + def _pause(self, mode): + from sglang.srt.managers.io_struct import PauseGenerationReqInput + + return lambda s: s.pause_generation(PauseGenerationReqInput(mode=mode)) + + @staticmethod + def _resume(scheduler): + scheduler._engine_paused = False + + def test_retract_pause_processes_inflight_job_before_retract(self): + # Iteration 1 launches a decode job. Iteration 2 pauses with retract + # while that job is in flight. Iteration 3 resumes; the retracted + # request is no longer running, so the loop idles. + events = [] + queue_after_pause = [] + pause = self._pause("retract") + scheduler = self._make_scheduler( + ingest_steps=[ + lambda s: None, + lambda s: (pause(s), queue_after_pause.append(len(s.result_queue))), + self._resume, + ], + events=events, + ) + + self._run_loop(scheduler, events) + + self.assertEqual( + events, + [("launch", ["r1"]), ("process", ["r1"]), ("retract", ["r1"])], + ) + self.assertEqual(queue_after_pause, [0]) + self.assertEqual(len(scheduler.result_queue), 0) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_scheduler_pause_generation.py b/test/registered/unit/managers/test_scheduler_pause_generation.py index c6ec667e37..1955063882 100644 --- a/test/registered/unit/managers/test_scheduler_pause_generation.py +++ b/test/registered/unit/managers/test_scheduler_pause_generation.py @@ -72,6 +72,7 @@ class TestSchedulerPauseGeneration(CustomTestCase): scheduler._engine_paused = False scheduler.enable_overlap = False scheduler.enable_skip_finishing_decode = False + scheduler.enable_overlap_mlx = False scheduler.last_batch = None scheduler.cur_batch_for_debug = None scheduler.chunked_req = None