[Fix] rust-server: Deliver scheduler-side aborts to Rust frontend clients (#45899)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
shodoco
parent
eb8ef80842
commit
5a86d6ee17
@@ -2228,6 +2228,15 @@ class Scheduler(
|
||||
self._event_loop_should_restart = False
|
||||
raise role_switch.PdRoleSwitchRestart()
|
||||
|
||||
def send_abort_output(self, abort_req: AbortReq, req: Req) -> None:
|
||||
"""Report a request the scheduler ends outside the output stream
|
||||
(aborted while queued, preempted, retracted or rejected)."""
|
||||
if self.rust_server is not None:
|
||||
# The zmq tokenizer socket is not consumed in Rust mode.
|
||||
self.rust_server.push_abort(abort_req, req)
|
||||
else:
|
||||
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
|
||||
|
||||
def send_control_output(self, output, recv_req) -> None:
|
||||
"""Answer a control request, at dispatch or later (a deferred flush)."""
|
||||
if self.rust_server is not None:
|
||||
@@ -3083,7 +3092,7 @@ class Scheduler(
|
||||
},
|
||||
)
|
||||
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
|
||||
self.send_abort_output(abort_req, req)
|
||||
return
|
||||
|
||||
if self.spec_algorithm.is_dflash_family():
|
||||
@@ -3483,7 +3492,7 @@ class Scheduler(
|
||||
},
|
||||
)
|
||||
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
|
||||
self.send_abort_output(abort_req, req)
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -3526,7 +3535,7 @@ class Scheduler(
|
||||
req_to_abort = candidate_req
|
||||
message = "The request is aborted by a higher priority request."
|
||||
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(
|
||||
req_to_abort,
|
||||
finished_reason={
|
||||
@@ -3767,7 +3776,7 @@ class Scheduler(
|
||||
self._pending_chunked_abort_req = None
|
||||
# A client abort leaves the message to the tokenizer.
|
||||
finished_reason = req.discard_output_reason and reason.to_json()
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(req, finished_reason=finished_reason), req
|
||||
)
|
||||
logger.debug(f"Abort chunked prefill request. {req.rid=}")
|
||||
@@ -4458,7 +4467,7 @@ class Scheduler(
|
||||
self.new_token_ratio_tracker.current = new_token_ratio
|
||||
for req in reqs_to_abort:
|
||||
abort_reason: FINISH_ABORT = req.to_finish
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(req, finished_reason=abort_reason.to_json()),
|
||||
req,
|
||||
)
|
||||
@@ -5584,7 +5593,7 @@ class Scheduler(
|
||||
self.beam_coordinator.retire_group(req)
|
||||
# Without the initiator's reason the tokenizer falls back to a
|
||||
# generic abort message.
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(req, finished_reason=recv_req.finished_reason), req
|
||||
)
|
||||
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
|
||||
@@ -5612,9 +5621,7 @@ class Scheduler(
|
||||
recv_req.abort_all, recv_req.rid
|
||||
):
|
||||
self._release_aborted_request(req)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
_make_abort_req(req), req
|
||||
)
|
||||
self.send_abort_output(_make_abort_req(req), req)
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
logger.debug(f"Abort dLLM queued request. {req.rid=}")
|
||||
@@ -5677,9 +5684,7 @@ class Scheduler(
|
||||
self.tree_cache,
|
||||
get_disagg().disaggregation_decode_retraction_backup,
|
||||
)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
_make_abort_req(decode_req), decode_req
|
||||
)
|
||||
self.send_abort_output(_make_abort_req(decode_req), decode_req)
|
||||
else:
|
||||
remaining_retracted.append(decode_req)
|
||||
self.disagg_decode_prealloc_queue.retracted_queue = remaining_retracted
|
||||
|
||||
@@ -44,8 +44,8 @@ from sglang.srt.utils.flatten import (
|
||||
from sglang.srt.utils.network import NetworkAddress
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.io_struct import BatchTokenIDOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
|
||||
from sglang.srt.managers.io_struct import AbortReq, BatchTokenIDOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput, Req
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.rust_extensions._server import MmSpec, Server
|
||||
|
||||
@@ -383,6 +383,32 @@ class RustServer:
|
||||
|
||||
self.server.push_control_result(recv_req.rid, encoded)
|
||||
|
||||
def push_abort(self, abort: AbortReq, req: Req) -> None:
|
||||
"""The terminal frame of a request the scheduler ended outside the output
|
||||
stream (aborted while queued, preempted, retracted or rejected), which
|
||||
Python's ``TokenizerManager._handle_abort_req`` builds from the same
|
||||
``AbortReq``: a one-request batch carrying no new tokens and the abort
|
||||
finish reason. The counts are the request's own, so the frame does not
|
||||
reset the ones a stream already reported."""
|
||||
finish_reason = abort.finished_reason or {
|
||||
"type": "abort",
|
||||
"message": abort.abort_message or "Abort in waiting queue",
|
||||
}
|
||||
header = msgspec.msgpack.encode(
|
||||
[
|
||||
[abort.rid],
|
||||
[finish_reason],
|
||||
[len(req.origin_input_ids)],
|
||||
[0],
|
||||
[req.cached_tokens],
|
||||
[req.reasoning_tokens],
|
||||
]
|
||||
)
|
||||
if not self.server.push_decode_result_batch(header, [b""]):
|
||||
logger.warning(
|
||||
"Rust egress closed; dropped abort of %s during shutdown", abort.rid
|
||||
)
|
||||
|
||||
def push_health_ack(self) -> None:
|
||||
"""Acknowledge the health probes skipped while the scheduler was busy."""
|
||||
self.server.push_health_ack()
|
||||
|
||||
Reference in New Issue
Block a user