[Fix] rust-server: Deliver scheduler-side aborts to Rust frontend clients (#45899)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
shodoco
parent
eb8ef80842
commit
5a86d6ee17
@@ -2228,6 +2228,15 @@ class Scheduler(
|
||||
self._event_loop_should_restart = False
|
||||
raise role_switch.PdRoleSwitchRestart()
|
||||
|
||||
def send_abort_output(self, abort_req: AbortReq, req: Req) -> None:
|
||||
"""Report a request the scheduler ends outside the output stream
|
||||
(aborted while queued, preempted, retracted or rejected)."""
|
||||
if self.rust_server is not None:
|
||||
# The zmq tokenizer socket is not consumed in Rust mode.
|
||||
self.rust_server.push_abort(abort_req, req)
|
||||
else:
|
||||
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
|
||||
|
||||
def send_control_output(self, output, recv_req) -> None:
|
||||
"""Answer a control request, at dispatch or later (a deferred flush)."""
|
||||
if self.rust_server is not None:
|
||||
@@ -3083,7 +3092,7 @@ class Scheduler(
|
||||
},
|
||||
)
|
||||
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
|
||||
self.send_abort_output(abort_req, req)
|
||||
return
|
||||
|
||||
if self.spec_algorithm.is_dflash_family():
|
||||
@@ -3483,7 +3492,7 @@ class Scheduler(
|
||||
},
|
||||
)
|
||||
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
|
||||
self.send_abort_output(abort_req, req)
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -3526,7 +3535,7 @@ class Scheduler(
|
||||
req_to_abort = candidate_req
|
||||
message = "The request is aborted by a higher priority request."
|
||||
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(
|
||||
req_to_abort,
|
||||
finished_reason={
|
||||
@@ -3767,7 +3776,7 @@ class Scheduler(
|
||||
self._pending_chunked_abort_req = None
|
||||
# A client abort leaves the message to the tokenizer.
|
||||
finished_reason = req.discard_output_reason and reason.to_json()
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(req, finished_reason=finished_reason), req
|
||||
)
|
||||
logger.debug(f"Abort chunked prefill request. {req.rid=}")
|
||||
@@ -4458,7 +4467,7 @@ class Scheduler(
|
||||
self.new_token_ratio_tracker.current = new_token_ratio
|
||||
for req in reqs_to_abort:
|
||||
abort_reason: FINISH_ABORT = req.to_finish
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(req, finished_reason=abort_reason.to_json()),
|
||||
req,
|
||||
)
|
||||
@@ -5584,7 +5593,7 @@ class Scheduler(
|
||||
self.beam_coordinator.retire_group(req)
|
||||
# Without the initiator's reason the tokenizer falls back to a
|
||||
# generic abort message.
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
self.send_abort_output(
|
||||
_make_abort_req(req, finished_reason=recv_req.finished_reason), req
|
||||
)
|
||||
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
|
||||
@@ -5612,9 +5621,7 @@ class Scheduler(
|
||||
recv_req.abort_all, recv_req.rid
|
||||
):
|
||||
self._release_aborted_request(req)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
_make_abort_req(req), req
|
||||
)
|
||||
self.send_abort_output(_make_abort_req(req), req)
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
logger.debug(f"Abort dLLM queued request. {req.rid=}")
|
||||
@@ -5677,9 +5684,7 @@ class Scheduler(
|
||||
self.tree_cache,
|
||||
get_disagg().disaggregation_decode_retraction_backup,
|
||||
)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(
|
||||
_make_abort_req(decode_req), decode_req
|
||||
)
|
||||
self.send_abort_output(_make_abort_req(decode_req), decode_req)
|
||||
else:
|
||||
remaining_retracted.append(decode_req)
|
||||
self.disagg_decode_prealloc_queue.retracted_queue = remaining_retracted
|
||||
|
||||
@@ -44,8 +44,8 @@ from sglang.srt.utils.flatten import (
|
||||
from sglang.srt.utils.network import NetworkAddress
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.io_struct import BatchTokenIDOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
|
||||
from sglang.srt.managers.io_struct import AbortReq, BatchTokenIDOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput, Req
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.rust_extensions._server import MmSpec, Server
|
||||
|
||||
@@ -383,6 +383,32 @@ class RustServer:
|
||||
|
||||
self.server.push_control_result(recv_req.rid, encoded)
|
||||
|
||||
def push_abort(self, abort: AbortReq, req: Req) -> None:
|
||||
"""The terminal frame of a request the scheduler ended outside the output
|
||||
stream (aborted while queued, preempted, retracted or rejected), which
|
||||
Python's ``TokenizerManager._handle_abort_req`` builds from the same
|
||||
``AbortReq``: a one-request batch carrying no new tokens and the abort
|
||||
finish reason. The counts are the request's own, so the frame does not
|
||||
reset the ones a stream already reported."""
|
||||
finish_reason = abort.finished_reason or {
|
||||
"type": "abort",
|
||||
"message": abort.abort_message or "Abort in waiting queue",
|
||||
}
|
||||
header = msgspec.msgpack.encode(
|
||||
[
|
||||
[abort.rid],
|
||||
[finish_reason],
|
||||
[len(req.origin_input_ids)],
|
||||
[0],
|
||||
[req.cached_tokens],
|
||||
[req.reasoning_tokens],
|
||||
]
|
||||
)
|
||||
if not self.server.push_decode_result_batch(header, [b""]):
|
||||
logger.warning(
|
||||
"Rust egress closed; dropped abort of %s during shutdown", abort.rid
|
||||
)
|
||||
|
||||
def push_health_ack(self) -> None:
|
||||
"""Acknowledge the health probes skipped while the scheduler was busy."""
|
||||
self.server.push_health_ack()
|
||||
|
||||
@@ -387,6 +387,7 @@ class TestGemma4RequestValidation(unittest.TestCase):
|
||||
scheduler.output_streamer = Mock()
|
||||
sender = Mock()
|
||||
scheduler.ipc_channels = SimpleNamespace(send_to_tokenizer=sender)
|
||||
scheduler.rust_server = None
|
||||
recv_req = MagicMock(
|
||||
session_params=None,
|
||||
session_id=None,
|
||||
|
||||
@@ -96,6 +96,7 @@ def _queued_scheduler(req):
|
||||
scheduler.tree_cache = MagicMock()
|
||||
scheduler.beam_coordinator = MagicMock()
|
||||
scheduler.ipc_channels = MagicMock()
|
||||
scheduler.rust_server = None
|
||||
scheduler.grammar_manager = MagicMock()
|
||||
scheduler.collect_inflight_reqs = lambda: set()
|
||||
return scheduler
|
||||
|
||||
@@ -67,6 +67,7 @@ class TestDisaggregationPriorityQueueing(unittest.TestCase):
|
||||
scheduler.disagg_prefill_bootstrap_queue = MagicMock()
|
||||
scheduler.disagg_decode_prealloc_queue = MagicMock()
|
||||
scheduler.ipc_channels = MagicMock()
|
||||
scheduler.rust_server = None
|
||||
return scheduler
|
||||
|
||||
def _new_req(self, priority=None):
|
||||
@@ -435,6 +436,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase):
|
||||
scheduler.beam_coordinator = MagicMock()
|
||||
scheduler.enable_hicache_storage = False
|
||||
scheduler.ipc_channels = MagicMock()
|
||||
scheduler.rust_server = None
|
||||
scheduler.disaggregation_mode = DisaggregationMode.PREFILL
|
||||
scheduler.release_aborted_prefill_waiting_req = MagicMock()
|
||||
scheduler.dllm_config = None
|
||||
|
||||
@@ -52,6 +52,7 @@ def _make_scheduler(pending_req, *, chunked_req, running_reqs) -> Scheduler:
|
||||
sched.disaggregation_mode = None
|
||||
sched.enable_hicache_storage = False
|
||||
sched.mm_receiver = None
|
||||
sched.rust_server = None
|
||||
sched.running_batch = SimpleNamespace(reqs=running_reqs)
|
||||
sched.last_batch = None
|
||||
return sched
|
||||
|
||||
@@ -240,6 +240,7 @@ class TestSchedulerPauseGeneration(CustomTestCase):
|
||||
scheduler.tree_cache.req_to_token_pool.mamba_allocator = None
|
||||
scheduler.new_token_ratio_tracker = SimpleNamespace(current=1.0)
|
||||
scheduler.ipc_channels = MagicMock()
|
||||
scheduler.rust_server = None
|
||||
scheduler.beam_coordinator = MagicMock()
|
||||
scheduler.update_running_batch(batch)
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ def _scheduler(waiting_queue, running_reqs=(), last_batch_reqs=()):
|
||||
s.enable_hicache_storage = False
|
||||
s.enable_unified_cache_external_linker = False
|
||||
s.ipc_channels = SimpleNamespace(send_to_tokenizer=MagicMock())
|
||||
s.rust_server = None
|
||||
s.beam_coordinator = MagicMock()
|
||||
s.running_batch = _batch(list(running_reqs))
|
||||
s.last_batch = _batch(list(last_batch_reqs)) if last_batch_reqs else None
|
||||
|
||||
@@ -120,6 +120,36 @@ class TestRustServerExtension(CustomTestCase):
|
||||
self.assertEqual(startup_time["load_weight"], 1.0)
|
||||
self.assertEqual(startup_time["cuda_graph"]["decode"], 2.0)
|
||||
|
||||
def test_scheduler_aborts_reach_the_rust_egress(self):
|
||||
"""An abort the scheduler sends outside the output stream becomes a
|
||||
one-request terminal batch in Rust mode, keeping the request's counts;
|
||||
without the Rust server it still goes to the tokenizer socket."""
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.managers.io_struct import AbortReq
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
|
||||
req = SimpleNamespace(
|
||||
origin_input_ids=[1, 2, 3], cached_tokens=2, reasoning_tokens=1
|
||||
)
|
||||
abort = AbortReq(rid="r#1", abort_message="gone")
|
||||
server = RustServer(Mock(), http_port=30000)
|
||||
scheduler = SimpleNamespace(rust_server=server, ipc_channels=Mock())
|
||||
Scheduler.send_abort_output(scheduler, abort, req)
|
||||
header, cols = server.server.push_decode_result_batch.call_args.args
|
||||
self.assertEqual(
|
||||
msgspec.msgpack.decode(header),
|
||||
[["r#1"], [{"type": "abort", "message": "gone"}], [3], [0], [2], [1]],
|
||||
)
|
||||
self.assertEqual(cols, [b""])
|
||||
scheduler.ipc_channels.send_to_tokenizer.send_output.assert_not_called()
|
||||
|
||||
scheduler.rust_server = None
|
||||
Scheduler.send_abort_output(scheduler, abort, req)
|
||||
scheduler.ipc_channels.send_to_tokenizer.send_output.assert_called_once_with(
|
||||
abort, req
|
||||
)
|
||||
|
||||
def test_launch_resolves_the_mm_pipeline_before_the_server_starts(self):
|
||||
"""The listener answers once the server is constructed, so a model
|
||||
without a Rust MM pipeline must be refused before that; a supported
|
||||
|
||||
@@ -478,11 +478,7 @@ class TestSpanlessAbortPaths(CustomTestCase):
|
||||
scheduler = SimpleNamespace(
|
||||
enable_priority_scheduling=False,
|
||||
abort_on_priority_when_disabled=True,
|
||||
ipc_channels=SimpleNamespace(
|
||||
send_to_tokenizer=SimpleNamespace(
|
||||
send_output=lambda obj, req_arg: sent.append(obj)
|
||||
)
|
||||
),
|
||||
send_abort_output=lambda obj, req_arg: sent.append(obj),
|
||||
)
|
||||
|
||||
with patch(
|
||||
|
||||
Reference in New Issue
Block a user