[Fix] rust-server: Deliver scheduler-side aborts to Rust frontend clients (#45899)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-23 09:04:54 -07:00
committed by GitHub
co-authored by metamergebot shodoco
parent eb8ef80842
commit 5a86d6ee17
10 changed files with 83 additions and 19 deletions
+17 -12
View File
@@ -2228,6 +2228,15 @@ class Scheduler(
self._event_loop_should_restart = False
raise role_switch.PdRoleSwitchRestart()
def send_abort_output(self, abort_req: AbortReq, req: Req) -> None:
"""Report a request the scheduler ends outside the output stream
(aborted while queued, preempted, retracted or rejected)."""
if self.rust_server is not None:
# The zmq tokenizer socket is not consumed in Rust mode.
self.rust_server.push_abort(abort_req, req)
else:
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
def send_control_output(self, output, recv_req) -> None:
"""Answer a control request, at dispatch or later (a deferred flush)."""
if self.rust_server is not None:
@@ -3083,7 +3092,7 @@ class Scheduler(
},
)
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
self.send_abort_output(abort_req, req)
return
if self.spec_algorithm.is_dflash_family():
@@ -3483,7 +3492,7 @@ class Scheduler(
},
)
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
self.send_abort_output(abort_req, req)
return False
return True
@@ -3526,7 +3535,7 @@ class Scheduler(
req_to_abort = candidate_req
message = "The request is aborted by a higher priority request."
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(
req_to_abort,
finished_reason={
@@ -3767,7 +3776,7 @@ class Scheduler(
self._pending_chunked_abort_req = None
# A client abort leaves the message to the tokenizer.
finished_reason = req.discard_output_reason and reason.to_json()
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(req, finished_reason=finished_reason), req
)
logger.debug(f"Abort chunked prefill request. {req.rid=}")
@@ -4458,7 +4467,7 @@ class Scheduler(
self.new_token_ratio_tracker.current = new_token_ratio
for req in reqs_to_abort:
abort_reason: FINISH_ABORT = req.to_finish
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(req, finished_reason=abort_reason.to_json()),
req,
)
@@ -5584,7 +5593,7 @@ class Scheduler(
self.beam_coordinator.retire_group(req)
# Without the initiator's reason the tokenizer falls back to a
# generic abort message.
self.ipc_channels.send_to_tokenizer.send_output(
self.send_abort_output(
_make_abort_req(req, finished_reason=recv_req.finished_reason), req
)
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
@@ -5612,9 +5621,7 @@ class Scheduler(
recv_req.abort_all, recv_req.rid
):
self._release_aborted_request(req)
self.ipc_channels.send_to_tokenizer.send_output(
_make_abort_req(req), req
)
self.send_abort_output(_make_abort_req(req), req)
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache, checkpoint=False)
logger.debug(f"Abort dLLM queued request. {req.rid=}")
@@ -5677,9 +5684,7 @@ class Scheduler(
self.tree_cache,
get_disagg().disaggregation_decode_retraction_backup,
)
self.ipc_channels.send_to_tokenizer.send_output(
_make_abort_req(decode_req), decode_req
)
self.send_abort_output(_make_abort_req(decode_req), decode_req)
else:
remaining_retracted.append(decode_req)
self.disagg_decode_prealloc_queue.retracted_queue = remaining_retracted
+28 -2
View File
@@ -44,8 +44,8 @@ from sglang.srt.utils.flatten import (
from sglang.srt.utils.network import NetworkAddress
if TYPE_CHECKING:
from sglang.srt.managers.io_struct import BatchTokenIDOutput
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
from sglang.srt.managers.io_struct import AbortReq, BatchTokenIDOutput
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput, Req
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.rust_extensions._server import MmSpec, Server
@@ -383,6 +383,32 @@ class RustServer:
self.server.push_control_result(recv_req.rid, encoded)
def push_abort(self, abort: AbortReq, req: Req) -> None:
"""The terminal frame of a request the scheduler ended outside the output
stream (aborted while queued, preempted, retracted or rejected), which
Python's ``TokenizerManager._handle_abort_req`` builds from the same
``AbortReq``: a one-request batch carrying no new tokens and the abort
finish reason. The counts are the request's own, so the frame does not
reset the ones a stream already reported."""
finish_reason = abort.finished_reason or {
"type": "abort",
"message": abort.abort_message or "Abort in waiting queue",
}
header = msgspec.msgpack.encode(
[
[abort.rid],
[finish_reason],
[len(req.origin_input_ids)],
[0],
[req.cached_tokens],
[req.reasoning_tokens],
]
)
if not self.server.push_decode_result_batch(header, [b""]):
logger.warning(
"Rust egress closed; dropped abort of %s during shutdown", abort.rid
)
def push_health_ack(self) -> None:
"""Acknowledge the health probes skipped while the scheduler was busy."""
self.server.push_health_ack()