From ca72dad953171ea4f580d89f70f999fc89501067 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Wed, 21 Oct 2026 03:41:36 -0700 Subject: [PATCH] [Fix] rust-server: Answer health probes a busy scheduler skipped on the response ring (#45475) Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com> Co-authored-by: shodoco --- python/sglang/srt/managers/scheduler.py | 15 +++-- python/sglang/srt/rust_server/server.py | 4 ++ .../src/api_server/core/handle.rs | 36 ++++++++++-- rust/sglang-server/src/lib.rs | 6 ++ rust/sglang-server/src/message/response.rs | 8 +++ .../src/tokenizer_manager/channel.rs | 4 +- .../src/tokenizer_manager/from_scheduler.rs | 12 +++- .../test_scheduler_health_check_signal.py | 55 +++++++++++++++++++ 8 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 test/registered/unit/managers/test_scheduler_health_check_signal.py diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 966e823611..e1f2a0dac1 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -5108,11 +5108,18 @@ class Scheduler( # Return some signal for the health check. # This is used to prevent the health check signal being blocked by long context prefill. # However, one minor issue is that this code path does not check the status of detokenizer manager. - self.ipc_channels.send_to_tokenizer.send_output( - HealthCheckOutput( - http_worker_ipc=self.return_health_check_ipcs.popleft() + if self.rust_server is not None: + # The zmq tokenizer socket is not consumed in Rust mode. Rust + # probes share one heartbeat, so one acknowledgement answers + # every probe skipped so far. + self.return_health_check_ipcs.clear() + self.rust_server.push_health_ack() + else: + self.ipc_channels.send_to_tokenizer.send_output( + HealthCheckOutput( + http_worker_ipc=self.return_health_check_ipcs.popleft() + ) ) - ) def add_external_corpus( self, recv_req: AddExternalCorpusReqInput diff --git a/python/sglang/srt/rust_server/server.py b/python/sglang/srt/rust_server/server.py index 5a4598955c..6bac0f753b 100644 --- a/python/sglang/srt/rust_server/server.py +++ b/python/sglang/srt/rust_server/server.py @@ -383,6 +383,10 @@ class RustServer: self.server.push_control_result(recv_req.rid, encoded) + def push_health_ack(self) -> None: + """Acknowledge the health probes skipped while the scheduler was busy.""" + self.server.push_health_ack() + def push_generation(self, payload: BatchTokenIDOutput) -> None: """Egress redirect for generation output (replaces the zmq detokenizer). diff --git a/rust/sglang-server/src/api_server/core/handle.rs b/rust/sglang-server/src/api_server/core/handle.rs index b8cbad55ee..53c25f37a9 100644 --- a/rust/sglang-server/src/api_server/core/handle.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -986,7 +986,10 @@ mod tests { use crate::message::io_struct::GetInternalStateReq; use crate::message::multimodal::MmItem; use crate::message::request::MmData; - use crate::message::response::ChunkEvent; + use crate::message::response::{ChunkEvent, frame_health_ack}; + use crate::runtime::Runnable; + use crate::tokenizer_manager::from_scheduler::Dispatcher; + use crate::tokenizer_manager::wiring::Senders; struct Harness { handle: CoreHandle, @@ -1346,16 +1349,35 @@ mod tests { assert!(abort_rx.try_recv().is_err()); } + /// A busy scheduler skips the probe and acknowledges it after its next step. + /// The dispatcher counts the acknowledgement as scheduler output, so the + /// probe passes without any batch frame. #[tokio::test] - async fn healthy_probe_uses_shared_activity_and_cleans_up() { + async fn skipped_probe_passes_on_health_ack_and_cleans_up() { let (intake_tx, intake_rx) = flume::unbounded(); let (abort_tx, abort_rx) = flume::unbounded(); let activity: ActivityCounter = Default::default(); + let (from_scheduler_tx, from_scheduler_rx) = + crate::tokenizer_manager::channel::from_scheduler(8); + let (detok_tx, detok_rx) = flume::unbounded(); + let (_shutdown_tx, shutdown_rx) = flume::bounded(1); + let dispatcher = Dispatcher::new( + from_scheduler_rx, + Senders { + tok_manager_tx: flume::unbounded().0, + abort_tx: flume::unbounded().0, + tokenizer_tx: flume::unbounded().0, + detokenizer_tx: vec![detok_tx], + }, + activity.clone(), + shutdown_rx, + ); + let dispatcher = std::thread::spawn(move || dispatcher.run()); let handle = configured_handle( intake_tx, abort_tx, 8, - activity.clone(), + activity, true, false, Vec::new(), @@ -1371,7 +1393,7 @@ mod tests { !handle.abort(None), "internal probes must not enter the public abort-all registry" ); - activity.fetch_add(1, Ordering::Relaxed); + assert!(from_scheduler_tx.push(frame_health_ack())); assert_eq!(probe.await.unwrap().unwrap(), HealthStatus::Healthy); assert!(matches!( @@ -1379,6 +1401,12 @@ mod tests { AbortSource::Guard(aborted) if aborted == rid )); assert!(abort_rx.try_recv().is_err()); + drop(from_scheduler_tx); + dispatcher.join().unwrap(); + assert!( + detok_rx.try_recv().is_err(), + "an acknowledgement routes nothing" + ); } #[tokio::test] diff --git a/rust/sglang-server/src/lib.rs b/rust/sglang-server/src/lib.rs index 0f1246fa36..5616a0e93c 100644 --- a/rust/sglang-server/src/lib.rs +++ b/rust/sglang-server/src/lib.rs @@ -262,6 +262,12 @@ impl Server { self.push_frame(py, crate::message::response::frame_error(rid, message)) } + /// Acknowledge health probes the scheduler skipped while busy, after the + /// step that followed them. Blocks for backpressure; `False` only on shutdown. + pub fn push_health_ack(&self, py: Python<'_>) -> bool { + self.push_frame(py, crate::message::response::frame_health_ack()) + } + /// Spawn the MM worker pool for the pipeline in `spec` (built from the /// resolved processor config; see `RustMmProcessor.resolve_spec` and /// `RustServer._build_mm_spec`). Image-only requests are processed entirely diff --git a/rust/sglang-server/src/message/response.rs b/rust/sglang-server/src/message/response.rs index 2ed77702f5..5e9a510820 100644 --- a/rust/sglang-server/src/message/response.rs +++ b/rust/sglang-server/src/message/response.rs @@ -72,6 +72,9 @@ pub const DISPATCH_TAG_BATCH: u8 = 2; /// A per-request failure `[rid, message]`: the Python drain couldn't decode a /// header, so it routes a 400 back to that request instead of crashing the loop. pub const DISPATCH_TAG_ERROR: u8 = 3; +/// A bodiless acknowledgement of the health probes a busy scheduler skipped, +/// sent after its next step. +pub const DISPATCH_TAG_HEALTH_ACK: u8 = 4; /// Read `n` little-endian f32s from `data` at `*off`, advancing `*off`. `None` when /// the range runs past the buffer (a malformed / positional-ABI-drifted frame): the @@ -559,6 +562,11 @@ pub fn frame_error(rid: &str, message: &str) -> Bytes { Bytes::from(buf) } +/// Frame a health-probe acknowledgement: the tag alone. +pub fn frame_health_ack() -> Bytes { + Bytes::from_static(&[DISPATCH_TAG_HEALTH_ACK]) +} + /// One scheduler output increment — the common, always-present frame. `token_ids` /// / `prompt_tokens` / `finish_reason` arrive from Python (pre-decode); the detok /// shard fills `text` in place. Deltas — fold with `OutputAccumulator` for diff --git a/rust/sglang-server/src/tokenizer_manager/channel.rs b/rust/sglang-server/src/tokenizer_manager/channel.rs index 572ba81af9..604385986e 100644 --- a/rust/sglang-server/src/tokenizer_manager/channel.rs +++ b/rust/sglang-server/src/tokenizer_manager/channel.rs @@ -110,8 +110,8 @@ impl ToSchedulerRx { } /// Scheduler output (`Server.push_decode_result_batch` / `push_control_result` -/// / `push_error`) → Rust response dispatcher. The single producer is the -/// Python thread; the consumer is the dispatcher. +/// / `push_error` / `push_health_ack`) → Rust response dispatcher. The single +/// producer is the Python thread; the consumer is the dispatcher. #[derive(Clone)] pub struct FromSchedulerTx { tx: flume::Sender, diff --git a/rust/sglang-server/src/tokenizer_manager/from_scheduler.rs b/rust/sglang-server/src/tokenizer_manager/from_scheduler.rs index baaa41b046..b01b76cd1b 100644 --- a/rust/sglang-server/src/tokenizer_manager/from_scheduler.rs +++ b/rust/sglang-server/src/tokenizer_manager/from_scheduler.rs @@ -9,14 +9,15 @@ use bytes::Bytes; use crate::message::detok::DetokMsg; use crate::message::ids::Rid; use crate::message::response::{ - ChunkEvent, DISPATCH_TAG_BATCH, DISPATCH_TAG_ERROR, DISPATCH_TAG_RESULT, for_each_chunk, + ChunkEvent, DISPATCH_TAG_BATCH, DISPATCH_TAG_ERROR, DISPATCH_TAG_HEALTH_ACK, + DISPATCH_TAG_RESULT, for_each_chunk, }; use crate::runtime::Runnable; use crate::tokenizer_manager::channel::FromSchedulerRx; use crate::tokenizer_manager::wiring::{Senders, recv}; -/// A monotonic counter bumped once per from_scheduler frame the dispatcher drains. -/// Equivalent to the Python `TokenizerManager`'s +/// A monotonic counter bumped once per decode batch or health acknowledgement +/// the dispatcher drains. Equivalent to the Python `TokenizerManager`'s /// `last_receive_tstamp`: `/health_generate` watches it advance to confirm the /// scheduler → detok path is alive (the value itself is meaningless). pub type ActivityCounter = Arc; @@ -135,6 +136,11 @@ impl Runnable for Dispatcher { self.route(&rid, msg); } } + // A busy scheduler skips health probes and acknowledges them after + // its next step, which proves it alive without any batch output. + DISPATCH_TAG_HEALTH_ACK => { + self.activity.fetch_add(1, Ordering::Relaxed); + } other => tracing::warn!(tag = other, "from_scheduler: unknown frame tag"), } } diff --git a/test/registered/unit/managers/test_scheduler_health_check_signal.py b/test/registered/unit/managers/test_scheduler_health_check_signal.py new file mode 100644 index 0000000000..14bcc25a1f --- /dev/null +++ b/test/registered/unit/managers/test_scheduler_health_check_signal.py @@ -0,0 +1,55 @@ +"""A busy scheduler acknowledges the health probes it skipped after each step. + +The Python frontend reads the acknowledgement from the zmq tokenizer socket. +The embedded Rust server does not consume that socket, so in Rust mode the +acknowledgement goes on its response ring instead. +""" + +import unittest +from collections import deque +from unittest.mock import MagicMock + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() + +from sglang.srt.managers.io_struct import HealthCheckOutput # noqa: E402 +from sglang.srt.managers.scheduler import Scheduler # noqa: E402 +from sglang.srt.rust_server.server import RustServer # noqa: E402 + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestHealthCheckSignal(CustomTestCase): + def _scheduler(self, rust_server, pending) -> Scheduler: + s = Scheduler.__new__(Scheduler) + s.rust_server = rust_server + s.ipc_channels = MagicMock() + s.return_health_check_ipcs = deque(pending) + return s + + def test_rust_mode_acknowledges_all_skipped_probes_on_the_ring(self): + server = MagicMock() + s = self._scheduler(RustServer(server, http_port=0), [None, None]) + + s.maybe_send_health_check_signal() + s.maybe_send_health_check_signal() + + server.push_health_ack.assert_called_once_with() + self.assertFalse(s.return_health_check_ipcs) + s.ipc_channels.send_to_tokenizer.send_output.assert_not_called() + + def test_python_mode_acknowledges_one_probe_per_step_on_zmq(self): + s = self._scheduler(None, ["ipc-a", "ipc-b"]) + + s.maybe_send_health_check_signal() + + (output,), _ = s.ipc_channels.send_to_tokenizer.send_output.call_args + self.assertIsInstance(output, HealthCheckOutput) + self.assertEqual(output.http_worker_ipc, "ipc-a") + self.assertEqual(list(s.return_health_check_ipcs), ["ipc-b"]) + + +if __name__ == "__main__": + unittest.main()