From 5a86d6ee17e1f2fd34cfa6d703ea38c581ab23fa Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 23 Oct 2026 09:04:54 -0700 Subject: [PATCH] [Fix] rust-server: Deliver scheduler-side aborts to Rust frontend clients (#45899) Co-authored-by: metamergebot Co-authored-by: shodoco --- python/sglang/srt/managers/scheduler.py | 29 ++++++++++-------- python/sglang/srt/rust_server/server.py | 30 +++++++++++++++++-- .../dllm/test_gemma4_uniform_lifecycle.py | 1 + .../managers/test_multimodal_abort_cleanup.py | 1 + ...test_priority_scheduling_disaggregation.py | 2 ++ .../test_scheduler_chunked_abort_race.py | 1 + .../test_scheduler_pause_generation.py | 1 + .../unit/managers/test_scheduler_timeouts.py | 1 + .../rust/shared/test_rust_server_extension.py | 30 +++++++++++++++++++ .../unit/utils/test_weight_versions.py | 6 +--- 10 files changed, 83 insertions(+), 19 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 1831160646..e20dc87f42 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2228,6 +2228,15 @@ class Scheduler( self._event_loop_should_restart = False raise role_switch.PdRoleSwitchRestart() + def send_abort_output(self, abort_req: AbortReq, req: Req) -> None: + """Report a request the scheduler ends outside the output stream + (aborted while queued, preempted, retracted or rejected).""" + if self.rust_server is not None: + # The zmq tokenizer socket is not consumed in Rust mode. + self.rust_server.push_abort(abort_req, req) + else: + self.ipc_channels.send_to_tokenizer.send_output(abort_req, req) + def send_control_output(self, output, recv_req) -> None: """Answer a control request, at dispatch or later (a deferred flush).""" if self.rust_server is not None: @@ -3083,7 +3092,7 @@ class Scheduler( }, ) req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason) - self.ipc_channels.send_to_tokenizer.send_output(abort_req, req) + self.send_abort_output(abort_req, req) return if self.spec_algorithm.is_dflash_family(): @@ -3483,7 +3492,7 @@ class Scheduler( }, ) req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason) - self.ipc_channels.send_to_tokenizer.send_output(abort_req, req) + self.send_abort_output(abort_req, req) return False return True @@ -3526,7 +3535,7 @@ class Scheduler( req_to_abort = candidate_req message = "The request is aborted by a higher priority request." - self.ipc_channels.send_to_tokenizer.send_output( + self.send_abort_output( _make_abort_req( req_to_abort, finished_reason={ @@ -3767,7 +3776,7 @@ class Scheduler( self._pending_chunked_abort_req = None # A client abort leaves the message to the tokenizer. finished_reason = req.discard_output_reason and reason.to_json() - self.ipc_channels.send_to_tokenizer.send_output( + self.send_abort_output( _make_abort_req(req, finished_reason=finished_reason), req ) logger.debug(f"Abort chunked prefill request. {req.rid=}") @@ -4458,7 +4467,7 @@ class Scheduler( self.new_token_ratio_tracker.current = new_token_ratio for req in reqs_to_abort: abort_reason: FINISH_ABORT = req.to_finish - self.ipc_channels.send_to_tokenizer.send_output( + self.send_abort_output( _make_abort_req(req, finished_reason=abort_reason.to_json()), req, ) @@ -5584,7 +5593,7 @@ class Scheduler( self.beam_coordinator.retire_group(req) # Without the initiator's reason the tokenizer falls back to a # generic abort message. - self.ipc_channels.send_to_tokenizer.send_output( + self.send_abort_output( _make_abort_req(req, finished_reason=recv_req.finished_reason), req ) # For disaggregation decode mode, the request in the waiting queue has KV cache allocated. @@ -5612,9 +5621,7 @@ class Scheduler( recv_req.abort_all, recv_req.rid ): self._release_aborted_request(req) - self.ipc_channels.send_to_tokenizer.send_output( - _make_abort_req(req), req - ) + self.send_abort_output(_make_abort_req(req), req) if req.kv.holds_kv or req.kv.holds_mamba: release_kv_cache(req, self.tree_cache, checkpoint=False) logger.debug(f"Abort dLLM queued request. {req.rid=}") @@ -5677,9 +5684,7 @@ class Scheduler( self.tree_cache, get_disagg().disaggregation_decode_retraction_backup, ) - self.ipc_channels.send_to_tokenizer.send_output( - _make_abort_req(decode_req), decode_req - ) + self.send_abort_output(_make_abort_req(decode_req), decode_req) else: remaining_retracted.append(decode_req) self.disagg_decode_prealloc_queue.retracted_queue = remaining_retracted diff --git a/python/sglang/srt/rust_server/server.py b/python/sglang/srt/rust_server/server.py index 6bac0f753b..24392e60e8 100644 --- a/python/sglang/srt/rust_server/server.py +++ b/python/sglang/srt/rust_server/server.py @@ -44,8 +44,8 @@ from sglang.srt.utils.flatten import ( from sglang.srt.utils.network import NetworkAddress if TYPE_CHECKING: - from sglang.srt.managers.io_struct import BatchTokenIDOutput - from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput + from sglang.srt.managers.io_struct import AbortReq, BatchTokenIDOutput + from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput, Req from sglang.srt.managers.scheduler import Scheduler from sglang.srt.rust_extensions._server import MmSpec, Server @@ -383,6 +383,32 @@ class RustServer: self.server.push_control_result(recv_req.rid, encoded) + def push_abort(self, abort: AbortReq, req: Req) -> None: + """The terminal frame of a request the scheduler ended outside the output + stream (aborted while queued, preempted, retracted or rejected), which + Python's ``TokenizerManager._handle_abort_req`` builds from the same + ``AbortReq``: a one-request batch carrying no new tokens and the abort + finish reason. The counts are the request's own, so the frame does not + reset the ones a stream already reported.""" + finish_reason = abort.finished_reason or { + "type": "abort", + "message": abort.abort_message or "Abort in waiting queue", + } + header = msgspec.msgpack.encode( + [ + [abort.rid], + [finish_reason], + [len(req.origin_input_ids)], + [0], + [req.cached_tokens], + [req.reasoning_tokens], + ] + ) + if not self.server.push_decode_result_batch(header, [b""]): + logger.warning( + "Rust egress closed; dropped abort of %s during shutdown", abort.rid + ) + def push_health_ack(self) -> None: """Acknowledge the health probes skipped while the scheduler was busy.""" self.server.push_health_ack() diff --git a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py index b0a896192c..0dfed6cca5 100644 --- a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py +++ b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py @@ -387,6 +387,7 @@ class TestGemma4RequestValidation(unittest.TestCase): scheduler.output_streamer = Mock() sender = Mock() scheduler.ipc_channels = SimpleNamespace(send_to_tokenizer=sender) + scheduler.rust_server = None recv_req = MagicMock( session_params=None, session_id=None, diff --git a/test/registered/unit/managers/test_multimodal_abort_cleanup.py b/test/registered/unit/managers/test_multimodal_abort_cleanup.py index 45479f9e3b..23f87edf03 100644 --- a/test/registered/unit/managers/test_multimodal_abort_cleanup.py +++ b/test/registered/unit/managers/test_multimodal_abort_cleanup.py @@ -96,6 +96,7 @@ def _queued_scheduler(req): scheduler.tree_cache = MagicMock() scheduler.beam_coordinator = MagicMock() scheduler.ipc_channels = MagicMock() + scheduler.rust_server = None scheduler.grammar_manager = MagicMock() scheduler.collect_inflight_reqs = lambda: set() return scheduler diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index f77cb53bb6..41fd73dfb1 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -67,6 +67,7 @@ class TestDisaggregationPriorityQueueing(unittest.TestCase): scheduler.disagg_prefill_bootstrap_queue = MagicMock() scheduler.disagg_decode_prealloc_queue = MagicMock() scheduler.ipc_channels = MagicMock() + scheduler.rust_server = None return scheduler def _new_req(self, priority=None): @@ -435,6 +436,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): scheduler.beam_coordinator = MagicMock() scheduler.enable_hicache_storage = False scheduler.ipc_channels = MagicMock() + scheduler.rust_server = None scheduler.disaggregation_mode = DisaggregationMode.PREFILL scheduler.release_aborted_prefill_waiting_req = MagicMock() scheduler.dllm_config = None diff --git a/test/registered/unit/managers/test_scheduler_chunked_abort_race.py b/test/registered/unit/managers/test_scheduler_chunked_abort_race.py index cf49739f92..01a7313c3f 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_abort_race.py +++ b/test/registered/unit/managers/test_scheduler_chunked_abort_race.py @@ -52,6 +52,7 @@ def _make_scheduler(pending_req, *, chunked_req, running_reqs) -> Scheduler: sched.disaggregation_mode = None sched.enable_hicache_storage = False sched.mm_receiver = None + sched.rust_server = None sched.running_batch = SimpleNamespace(reqs=running_reqs) sched.last_batch = None return sched diff --git a/test/registered/unit/managers/test_scheduler_pause_generation.py b/test/registered/unit/managers/test_scheduler_pause_generation.py index 4017fa3796..2b5a526acb 100644 --- a/test/registered/unit/managers/test_scheduler_pause_generation.py +++ b/test/registered/unit/managers/test_scheduler_pause_generation.py @@ -240,6 +240,7 @@ class TestSchedulerPauseGeneration(CustomTestCase): scheduler.tree_cache.req_to_token_pool.mamba_allocator = None scheduler.new_token_ratio_tracker = SimpleNamespace(current=1.0) scheduler.ipc_channels = MagicMock() + scheduler.rust_server = None scheduler.beam_coordinator = MagicMock() scheduler.update_running_batch(batch) diff --git a/test/registered/unit/managers/test_scheduler_timeouts.py b/test/registered/unit/managers/test_scheduler_timeouts.py index b84591f987..85da34dfee 100644 --- a/test/registered/unit/managers/test_scheduler_timeouts.py +++ b/test/registered/unit/managers/test_scheduler_timeouts.py @@ -76,6 +76,7 @@ def _scheduler(waiting_queue, running_reqs=(), last_batch_reqs=()): s.enable_hicache_storage = False s.enable_unified_cache_external_linker = False s.ipc_channels = SimpleNamespace(send_to_tokenizer=MagicMock()) + s.rust_server = None s.beam_coordinator = MagicMock() s.running_batch = _batch(list(running_reqs)) s.last_batch = _batch(list(last_batch_reqs)) if last_batch_reqs else None diff --git a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py index 82444b095a..c55a60ca7a 100644 --- a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py +++ b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py @@ -120,6 +120,36 @@ class TestRustServerExtension(CustomTestCase): self.assertEqual(startup_time["load_weight"], 1.0) self.assertEqual(startup_time["cuda_graph"]["decode"], 2.0) + def test_scheduler_aborts_reach_the_rust_egress(self): + """An abort the scheduler sends outside the output stream becomes a + one-request terminal batch in Rust mode, keeping the request's counts; + without the Rust server it still goes to the tokenizer socket.""" + import msgspec + + from sglang.srt.managers.io_struct import AbortReq + from sglang.srt.managers.scheduler import Scheduler + + req = SimpleNamespace( + origin_input_ids=[1, 2, 3], cached_tokens=2, reasoning_tokens=1 + ) + abort = AbortReq(rid="r#1", abort_message="gone") + server = RustServer(Mock(), http_port=30000) + scheduler = SimpleNamespace(rust_server=server, ipc_channels=Mock()) + Scheduler.send_abort_output(scheduler, abort, req) + header, cols = server.server.push_decode_result_batch.call_args.args + self.assertEqual( + msgspec.msgpack.decode(header), + [["r#1"], [{"type": "abort", "message": "gone"}], [3], [0], [2], [1]], + ) + self.assertEqual(cols, [b""]) + scheduler.ipc_channels.send_to_tokenizer.send_output.assert_not_called() + + scheduler.rust_server = None + Scheduler.send_abort_output(scheduler, abort, req) + scheduler.ipc_channels.send_to_tokenizer.send_output.assert_called_once_with( + abort, req + ) + def test_launch_resolves_the_mm_pipeline_before_the_server_starts(self): """The listener answers once the server is constructed, so a model without a Rust MM pipeline must be refused before that; a supported diff --git a/test/registered/unit/utils/test_weight_versions.py b/test/registered/unit/utils/test_weight_versions.py index b15302fcd1..7841897283 100644 --- a/test/registered/unit/utils/test_weight_versions.py +++ b/test/registered/unit/utils/test_weight_versions.py @@ -478,11 +478,7 @@ class TestSpanlessAbortPaths(CustomTestCase): scheduler = SimpleNamespace( enable_priority_scheduling=False, abort_on_priority_when_disabled=True, - ipc_channels=SimpleNamespace( - send_to_tokenizer=SimpleNamespace( - send_output=lambda obj, req_arg: sent.append(obj) - ) - ), + send_abort_output=lambda obj, req_arg: sent.append(obj), ) with patch(