[Fix] rust-server: Deliver scheduler-side aborts to Rust frontend clients (#45899)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-23 09:04:54 -07:00
committed by GitHub
co-authored by metamergebot shodoco
parent eb8ef80842
commit 5a86d6ee17
10 changed files with 83 additions and 19 deletions
+17 -12
View File
@@ -2228,6 +2228,15 @@ class Scheduler(
self._event_loop_should_restart = False
raise role_switch.PdRoleSwitchRestart()
def send_abort_output(self, abort_req: AbortReq, req: Req) -> None:
"""Report a request the scheduler ends outside the output stream
(aborted while queued, preempted, retracted or rejected)."""
if self.rust_server is not None:
# The zmq tokenizer socket is not consumed in Rust mode.
self.rust_server.push_abort(abort_req, req)
else:
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
def send_control_output(self, output, recv_req) -> None:
"""Answer a control request, at dispatch or later (a deferred flush)."""
if self.rust_server is not None:
@@ -3083,7 +3092,7 @@ class Scheduler(
},
)
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
self.send_abort_output(abort_req, req)
return
if self.spec_algorithm.is_dflash_family():
@@ -3483,7 +3492,7 @@ class Scheduler(
},
)
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
self.send_abort_output(abort_req, req)
return False
return True
@@ -3526,7 +3535,7 @@ class Scheduler(
req_to_abort = candidate_req
message = "The request is aborted by a higher priority request."
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(
req_to_abort,
finished_reason={
@@ -3767,7 +3776,7 @@ class Scheduler(
self._pending_chunked_abort_req = None
# A client abort leaves the message to the tokenizer.
finished_reason = req.discard_output_reason and reason.to_json()
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(req, finished_reason=finished_reason), req
)
logger.debug(f"Abort chunked prefill request. {req.rid=}")
@@ -4458,7 +4467,7 @@ class Scheduler(
self.new_token_ratio_tracker.current = new_token_ratio
for req in reqs_to_abort:
abort_reason: FINISH_ABORT = req.to_finish
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(req, finished_reason=abort_reason.to_json()),
req,
)
@@ -5584,7 +5593,7 @@ class Scheduler(
self.beam_coordinator.retire_group(req)
# Without the initiator's reason the tokenizer falls back to a
# generic abort message.
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(req, finished_reason=recv_req.finished_reason), req
)
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
@@ -5612,9 +5621,7 @@ class Scheduler(
recv_req.abort_all, recv_req.rid
):
self._release_aborted_request(req)
self.ipc_channels.send_to_tokenizer.send_output(
_make_abort_req(req), req
)
self.send_abort_output(_make_abort_req(req), req)
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache, checkpoint=False)
logger.debug(f"Abort dLLM queued request. {req.rid=}")
@@ -5677,9 +5684,7 @@ class Scheduler(
self.tree_cache,
get_disagg().disaggregation_decode_retraction_backup,
)
self.ipc_channels.send_to_tokenizer.send_output(
_make_abort_req(decode_req), decode_req
)
self.send_abort_output(_make_abort_req(decode_req), decode_req)
else:
remaining_retracted.append(decode_req)
self.disagg_decode_prealloc_queue.retracted_queue = remaining_retracted
+28 -2
View File
@@ -44,8 +44,8 @@ from sglang.srt.utils.flatten import (
from sglang.srt.utils.network import NetworkAddress
if TYPE_CHECKING:
from sglang.srt.managers.io_struct import BatchTokenIDOutput
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
from sglang.srt.managers.io_struct import AbortReq, BatchTokenIDOutput
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput, Req
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.rust_extensions._server import MmSpec, Server
@@ -383,6 +383,32 @@ class RustServer:
self.server.push_control_result(recv_req.rid, encoded)
def push_abort(self, abort: AbortReq, req: Req) -> None:
"""The terminal frame of a request the scheduler ended outside the output
stream (aborted while queued, preempted, retracted or rejected), which
Python's ``TokenizerManager._handle_abort_req`` builds from the same
``AbortReq``: a one-request batch carrying no new tokens and the abort
finish reason. The counts are the request's own, so the frame does not
reset the ones a stream already reported."""
finish_reason = abort.finished_reason or {
"type": "abort",
"message": abort.abort_message or "Abort in waiting queue",
}
header = msgspec.msgpack.encode(
[
[abort.rid],
[finish_reason],
[len(req.origin_input_ids)],
[0],
[req.cached_tokens],
[req.reasoning_tokens],
]
)
if not self.server.push_decode_result_batch(header, [b""]):
logger.warning(
"Rust egress closed; dropped abort of %s during shutdown", abort.rid
)
def push_health_ack(self) -> None:
"""Acknowledge the health probes skipped while the scheduler was busy."""
self.server.push_health_ack()
@@ -387,6 +387,7 @@ class TestGemma4RequestValidation(unittest.TestCase):
scheduler.output_streamer = Mock()
sender = Mock()
scheduler.ipc_channels = SimpleNamespace(send_to_tokenizer=sender)
scheduler.rust_server = None
recv_req = MagicMock(
session_params=None,
session_id=None,
@@ -96,6 +96,7 @@ def _queued_scheduler(req):
scheduler.tree_cache = MagicMock()
scheduler.beam_coordinator = MagicMock()
scheduler.ipc_channels = MagicMock()
scheduler.rust_server = None
scheduler.grammar_manager = MagicMock()
scheduler.collect_inflight_reqs = lambda: set()
return scheduler
@@ -67,6 +67,7 @@ class TestDisaggregationPriorityQueueing(unittest.TestCase):
scheduler.disagg_prefill_bootstrap_queue = MagicMock()
scheduler.disagg_decode_prealloc_queue = MagicMock()
scheduler.ipc_channels = MagicMock()
scheduler.rust_server = None
return scheduler
def _new_req(self, priority=None):
@@ -435,6 +436,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase):
scheduler.beam_coordinator = MagicMock()
scheduler.enable_hicache_storage = False
scheduler.ipc_channels = MagicMock()
scheduler.rust_server = None
scheduler.disaggregation_mode = DisaggregationMode.PREFILL
scheduler.release_aborted_prefill_waiting_req = MagicMock()
scheduler.dllm_config = None
@@ -52,6 +52,7 @@ def _make_scheduler(pending_req, *, chunked_req, running_reqs) -> Scheduler:
sched.disaggregation_mode = None
sched.enable_hicache_storage = False
sched.mm_receiver = None
sched.rust_server = None
sched.running_batch = SimpleNamespace(reqs=running_reqs)
sched.last_batch = None
return sched
@@ -240,6 +240,7 @@ class TestSchedulerPauseGeneration(CustomTestCase):
scheduler.tree_cache.req_to_token_pool.mamba_allocator = None
scheduler.new_token_ratio_tracker = SimpleNamespace(current=1.0)
scheduler.ipc_channels = MagicMock()
scheduler.rust_server = None
scheduler.beam_coordinator = MagicMock()
scheduler.update_running_batch(batch)
@@ -76,6 +76,7 @@ def _scheduler(waiting_queue, running_reqs=(), last_batch_reqs=()):
s.enable_hicache_storage = False
s.enable_unified_cache_external_linker = False
s.ipc_channels = SimpleNamespace(send_to_tokenizer=MagicMock())
s.rust_server = None
s.beam_coordinator = MagicMock()
s.running_batch = _batch(list(running_reqs))
s.last_batch = _batch(list(last_batch_reqs)) if last_batch_reqs else None
@@ -120,6 +120,36 @@ class TestRustServerExtension(CustomTestCase):
self.assertEqual(startup_time["load_weight"], 1.0)
self.assertEqual(startup_time["cuda_graph"]["decode"], 2.0)
def test_scheduler_aborts_reach_the_rust_egress(self):
"""An abort the scheduler sends outside the output stream becomes a
one-request terminal batch in Rust mode, keeping the request's counts;
without the Rust server it still goes to the tokenizer socket."""
import msgspec
from sglang.srt.managers.io_struct import AbortReq
from sglang.srt.managers.scheduler import Scheduler
req = SimpleNamespace(
origin_input_ids=[1, 2, 3], cached_tokens=2, reasoning_tokens=1
)
abort = AbortReq(rid="r#1", abort_message="gone")
server = RustServer(Mock(), http_port=30000)
scheduler = SimpleNamespace(rust_server=server, ipc_channels=Mock())
Scheduler.send_abort_output(scheduler, abort, req)
header, cols = server.server.push_decode_result_batch.call_args.args
self.assertEqual(
msgspec.msgpack.decode(header),
[["r#1"], [{"type": "abort", "message": "gone"}], [3], [0], [2], [1]],
)
self.assertEqual(cols, [b""])
scheduler.ipc_channels.send_to_tokenizer.send_output.assert_not_called()
scheduler.rust_server = None
Scheduler.send_abort_output(scheduler, abort, req)
scheduler.ipc_channels.send_to_tokenizer.send_output.assert_called_once_with(
abort, req
)
def test_launch_resolves_the_mm_pipeline_before_the_server_starts(self):
"""The listener answers once the server is constructed, so a model
without a Rust MM pipeline must be refused before that; a supported
@@ -478,11 +478,7 @@ class TestSpanlessAbortPaths(CustomTestCase):
scheduler = SimpleNamespace(
enable_priority_scheduling=False,
abort_on_priority_when_disabled=True,
ipc_channels=SimpleNamespace(
send_to_tokenizer=SimpleNamespace(
send_output=lambda obj, req_arg: sent.append(obj)
)
),
send_abort_output=lambda obj, req_arg: sent.append(obj),
)
with patch(