diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 88b1a1d9cd..91a0f1edb6 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1310,7 +1310,7 @@ class Scheduler( self.flush_wrapper = SchedulerFlushWrapper( flush_cache=self.flush_cache, is_fully_idle=self.is_fully_idle, - ipc_channels=self.ipc_channels, + send_output=self.send_control_output, ) self._last_logged_elastic_radix_namespace: Optional[str] = None self.session_controller = SessionController(self.tree_cache) @@ -2189,16 +2189,11 @@ class Scheduler( output = self._request_dispatcher(recv_req) if output is not None: - if self.rust_server is not None: - # Embedded Rust server: every control-request response goes - # back through the egress ring (the zmq tokenizer socket is - # not consumed); the Rust api_server shapes it per-endpoint. - self.rust_server.push_control_output(recv_req, output) - elif isinstance(output, RpcReqOutput): + if self.rust_server is None and isinstance(output, RpcReqOutput): if self.ipc_channels.recv_from_rpc is not None: sock_send(self.ipc_channels.recv_from_rpc, output) else: - self.ipc_channels.send_to_tokenizer.send_output(output, recv_req) + self.send_control_output(output, recv_req) self.flush_wrapper.check_pending() if self.external_corpus_manager is not None: @@ -2211,6 +2206,16 @@ class Scheduler( self._event_loop_should_restart = False raise role_switch.PdRoleSwitchRestart() + def send_control_output(self, output, recv_req) -> None: + """Answer a control request, at dispatch or later (a deferred flush).""" + if self.rust_server is not None: + # Embedded Rust server: every control-request response goes + # back through the egress ring (the zmq tokenizer socket is + # not consumed); the Rust api_server shapes it per-endpoint. + self.rust_server.push_control_output(recv_req, output) + else: + self.ipc_channels.send_to_tokenizer.send_output(output, recv_req) + @staticmethod def _tokenized_requests(recv_req): if isinstance( diff --git a/python/sglang/srt/managers/scheduler_components/flush_wrapper.py b/python/sglang/srt/managers/scheduler_components/flush_wrapper.py index 2a3b603bc6..5dd7caafe2 100644 --- a/python/sglang/srt/managers/scheduler_components/flush_wrapper.py +++ b/python/sglang/srt/managers/scheduler_components/flush_wrapper.py @@ -3,9 +3,6 @@ import time from typing import Callable, Optional, Tuple from sglang.srt.managers.io_struct import FlushCacheReqInput, FlushCacheReqOutput -from sglang.srt.managers.scheduler_components.ipc_channels import ( - SchedulerIpcChannels, -) class SchedulerFlushWrapper: @@ -14,11 +11,12 @@ class SchedulerFlushWrapper: *, flush_cache: Callable[[], bool], is_fully_idle: Callable[[], bool], - ipc_channels: SchedulerIpcChannels, + send_output: Callable[[FlushCacheReqOutput, FlushCacheReqInput], None], ) -> None: self._flush_cache = flush_cache self._is_fully_idle = is_fully_idle - self._ipc_channels = ipc_channels + # Answers a deferred flush through the scheduler's control-reply path. + self._send_output = send_output self._pending: Optional[Tuple[FlushCacheReqInput, float]] = None def handle(self, recv_req: FlushCacheReqInput) -> Optional[FlushCacheReqOutput]: @@ -47,9 +45,7 @@ class SchedulerFlushWrapper: if self._is_fully_idle(): success = self._flush_cache() self._pending = None - self._ipc_channels.send_to_tokenizer.send_output( - FlushCacheReqOutput(success=success), pending_req - ) + self._send_output(FlushCacheReqOutput(success=success), pending_req) return if time.monotonic() >= deadline: @@ -57,7 +53,7 @@ class SchedulerFlushWrapper: "Deferred flush_cache timed out while waiting for idle state." ) self._pending = None - self._ipc_channels.send_to_tokenizer.send_output( + self._send_output( FlushCacheReqOutput( success=False, message="Timed out waiting for idle state." ), diff --git a/rust/sglang-server/src/api_server/core/handle.rs b/rust/sglang-server/src/api_server/core/handle.rs index 532b1a60e7..1f90689919 100644 --- a/rust/sglang-server/src/api_server/core/handle.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -1,6 +1,7 @@ //! The shared handle: submission with admission and cancellation ownership, //! runtime-to-semantic event translation, and the typed control operations -//! (model info, server info, weights check, detokenize, health probe). +//! (model info, server info, flush cache, weights check, tokenize, detokenize, +//! health probe). use std::collections::BTreeMap; use std::sync::{ @@ -13,7 +14,9 @@ use tokio::sync::mpsc; use crate::message::config::{DisaggregationMode, ServerArgs}; use crate::message::ids::Rid; -use crate::message::io_struct::{CheckWeightsReqInput, ControlRequest, GetInternalStateReq}; +use crate::message::io_struct::{ + CheckWeightsReqInput, ControlRequest, FlushCacheReqInput, GetInternalStateReq, +}; use crate::message::request::{GenerateRequest, Request, RequestKind}; use crate::message::response::{ResponseItem, ResponseSink}; use crate::message::sampling::SamplingParams; @@ -217,6 +220,21 @@ impl CoreHandle { }) } + /// Ask the scheduler to flush its caches, waiting up to `timeout_s` for it + /// to go idle. The scheduler runs its own flush handler in its own loop; + /// this only relays the request and its outcome. + pub(crate) async fn flush_cache(&self, timeout_s: f64) -> Result { + let bytes = self + .control(ControlRequest::FlushCacheReqInput(FlushCacheReqInput::new( + Rid::new().to_string(), + Some(timeout_s), + ))) + .await?; + rmp_serde::from_slice::(&bytes).map_err(|error| { + CoreError::InvalidResponse(format!("invalid flush-cache response: {error}")) + }) + } + /// Ask every scheduler rank to run the weight checker's `action` on the /// runners `selector` names, and relay each rank's checksum record. pub(crate) async fn check_weights( @@ -532,6 +550,14 @@ fn generation_event(item: ResponseItem) -> CoreEvent { } } +/// The scheduler's `FlushCacheReqOutput`: whether the flush ran, and why not. +#[derive(Debug, serde::Deserialize, PartialEq)] +pub(crate) struct FlushCacheOutcome { + pub(crate) success: bool, + #[serde(default)] + pub(crate) message: String, +} + /// The scheduler's `CheckWeightsReqOutput`: whether every rank succeeded, why /// not, and one checksum record per rank (`None` when no rank produced one). #[derive(Debug, serde::Deserialize, PartialEq)] diff --git a/rust/sglang-server/src/api_server/http/info.rs b/rust/sglang-server/src/api_server/http/info.rs index ec422208f3..d9c7716d63 100644 --- a/rust/sglang-server/src/api_server/http/info.rs +++ b/rust/sglang-server/src/api_server/http/info.rs @@ -1,5 +1,6 @@ //! Common control-plane endpoints — `/server_info`, `/get_model_info` -//! (+ `/model_info` alias), `/weights_checker`, backed by typed operations on +//! (+ `/model_info` alias), `/flush_cache`, `/weights_checker`, backed by typed +//! operations on //! [`crate::api_server::core::CoreHandle`]. Data-plane endpoints (incl. `/health*`, //! which round-trips a generate probe) live in the sibling `native_api` and //! `openai` modules; the shared `AppState` lives in the parent @@ -7,8 +8,8 @@ use axum::{ Json, Router, - body::Bytes, - extract::State, + body::{Body, Bytes}, + extract::{Query, State, rejection::QueryRejection}, http::StatusCode, response::{IntoResponse, Response}, routing::get, @@ -33,6 +34,7 @@ pub(super) fn routes() -> Router> { // alias). .route("/get_model_info", get(model_info)) .route("/model_info", get(model_info)) + .route("/flush_cache", get(flush_cache).post(flush_cache)) .route("/weights_checker", get(check_weights).post(check_weights)) } @@ -99,6 +101,62 @@ async fn server_info(State(state): State>) -> Response { } } +#[derive(Deserialize)] +struct FlushCacheQuery { + #[serde(default)] + timeout: f64, +} + +/// `GET|POST /flush_cache?timeout=` — the scheduler flushes its caches in +/// its own loop, after waiting up to `timeout` seconds for running requests to +/// finish. Python's status and plain-text body: 200 once flushed, 400 with the +/// scheduler's reason otherwise. +async fn flush_cache( + State(state): State>, + query: Result, QueryRejection>, +) -> Response { + let timeout = match query { + Ok(Query(query)) if query.timeout >= 0.0 => query.timeout, + Ok(_) => { + return native_error( + StatusCode::BAD_REQUEST, + "timeout must be greater than or equal to 0", + false, + ); + } + Err(rejection) => { + return native_error(StatusCode::BAD_REQUEST, &rejection.body_text(), false); + } + }; + let outcome = match state.core.flush_cache(timeout).await { + Ok(outcome) => outcome, + Err(CoreError::Unavailable) => { + return native_error( + StatusCode::SERVICE_UNAVAILABLE, + "service unavailable", + false, + ); + } + Err(error) => return native_error(error.http_status(), &error.to_string(), false), + }; + let (status, body) = if outcome.success { + ( + StatusCode::OK, + "Cache flushed.\nPlease check backend logs for more details. (When there are \ + running or waiting requests, the operation will not be performed.)\n" + .to_owned(), + ) + } else if outcome.message.is_empty() { + (StatusCode::BAD_REQUEST, "Flush cache failed.\n".to_owned()) + } else { + (StatusCode::BAD_REQUEST, outcome.message) + }; + // Python's bare `Response(content=...)`: no content type. + let mut response = Response::new(Body::from(body)); + *response.status_mut() = status; + response +} + /// Python's `CheckWeightsReqInput` body, every field optional. #[derive(Deserialize)] #[serde(default)] @@ -436,6 +494,103 @@ mod tests { assert!(abort_rx.try_recv().is_err()); } + /// `/flush_cache` relays one `FlushCacheReqInput` carrying the query + /// timeout and answers with Python's status and plain-text body; a + /// negative timeout is refused before anything reaches the scheduler. + #[tokio::test] + async fn flush_cache_relays_the_scheduler_outcome() { + for (method, uri, timeout, outcome, status, body) in [ + ( + "POST", + "/flush_cache?timeout=30", + 30.0, + (true, ""), + StatusCode::OK, + "Cache flushed.\nPlease check backend logs for more details. (When there \ + are running or waiting requests, the operation will not be performed.)\n", + ), + ( + "GET", + "/flush_cache", + 0.0, + (false, "Timed out waiting for idle state."), + StatusCode::BAD_REQUEST, + "Timed out waiting for idle state.", + ), + ( + "POST", + "/flush_cache", + 0.0, + (false, ""), + StatusCode::BAD_REQUEST, + "Flush cache failed.\n", + ), + ] { + let (state, intake_rx, _abort_rx) = test_state(ServerArgs::default()); + let response = tokio::spawn( + routes().with_state(state).oneshot( + HttpRequest::builder() + .method(method) + .uri(uri) + .body(Body::empty()) + .unwrap(), + ), + ); + let TmEvent::Intake { + request: runtime_request, + admission, + } = intake_rx.recv_async().await.unwrap() + else { + panic!("flush_cache must submit one control request"); + }; + assert!(admission.try_accept()); + let RequestKind::Control(control) = &runtime_request.kind else { + panic!("flush_cache must submit a control request"); + }; + let header = control.encode().unwrap(); + let header = rmpv::decode::read_value(&mut &header[..]).unwrap(); + assert_eq!(header[0].as_str(), Some("FlushCacheReqInput")); + assert_eq!(header[3].as_f64(), Some(timeout), "{uri}"); + + let mut payload = Vec::new(); + rmpv::encode::write_value( + &mut payload, + &rmpv::Value::Map(vec![ + (rmpv::Value::from("rid"), rmpv::Value::from("x")), + (rmpv::Value::from("http_worker_ipc"), rmpv::Value::Nil), + (rmpv::Value::from("success"), rmpv::Value::from(outcome.0)), + (rmpv::Value::from("message"), rmpv::Value::from(outcome.1)), + ]), + ) + .unwrap(); + runtime_request + .sink + .try_send(ResponseItem::Control(Bytes::from(payload))) + .unwrap(); + + let response = response.await.unwrap().unwrap(); + assert_eq!(response.status(), status, "{uri}"); + assert!(response.headers().get(CONTENT_TYPE).is_none()); + let bytes = to_bytes(response.into_body(), 1024).await.unwrap(); + assert_eq!(std::str::from_utf8(&bytes).unwrap(), body); + } + + let (state, intake_rx, _abort_rx) = test_state(ServerArgs::default()); + let response = routes() + .with_state(state) + .oneshot( + HttpRequest::builder() + .method("POST") + .uri("/flush_cache?timeout=-1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(intake_rx.try_recv().is_err()); + } + /// `/weights_checker` relays one `CheckWeightsReqInput` carrying the body's /// fields (Python's defaults for a missing body) and answers with Python's /// JSON: the rank records as sent, in order, with the SHA-256 of their diff --git a/rust/sglang-server/src/message/io_struct.rs b/rust/sglang-server/src/message/io_struct.rs index 2a185b5c8c..969a516216 100644 --- a/rust/sglang-server/src/message/io_struct.rs +++ b/rust/sglang-server/src/message/io_struct.rs @@ -92,6 +92,12 @@ control_messages! { /// `/server_info`'s control request: a bare `BaseReq` with no extra fields. GetInternalStateReq {} + /// `/flush_cache`'s control request: the scheduler flushes its caches in + /// its own loop, waiting up to `timeout_s` for running requests to drain. + FlushCacheReqInput { + timeout_s: Option, + } + /// `/weights_checker`'s control request: every scheduler rank runs the /// weight checker's `action` on the runners `selector` names. CheckWeightsReqInput { @@ -167,6 +173,12 @@ impl GetInternalStateReq { } } +impl FlushCacheReqInput { + pub fn new(rid: String, timeout_s: Option) -> Self { + Self { rid, timeout_s } + } +} + impl CheckWeightsReqInput { pub fn new( rid: String, @@ -218,6 +230,24 @@ mod tests { assert!(arr[5].is_nil()); } + #[test] + fn flush_cache_req_msgpack_shape() { + let b = FlushCacheReqInput::new("7".into(), Some(30.0)) + .encode() + .unwrap(); + let val = rmpv::decode::read_value(&mut &b[..]).unwrap(); + let arr = val.as_array().expect("array"); + assert_eq!( + arr.len(), + 4, + "FlushCacheReqInput = [tag, rid, http_ipc, timeout_s]" + ); + assert_eq!(arr[0].as_str(), Some("FlushCacheReqInput")); + assert_eq!(arr[1].as_str(), Some("7")); + assert!(arr[2].is_nil()); + assert_eq!(arr[3].as_f64(), Some(30.0)); + } + #[test] fn check_weights_req_msgpack_shape() { let b = CheckWeightsReqInput::new( diff --git a/test/registered/unit/managers/test_scheduler_flush_cache.py b/test/registered/unit/managers/test_scheduler_flush_cache.py index 1ba417dd7b..10ae832c47 100644 --- a/test/registered/unit/managers/test_scheduler_flush_cache.py +++ b/test/registered/unit/managers/test_scheduler_flush_cache.py @@ -2,7 +2,8 @@ import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch -from sglang.srt.managers.io_struct import FlushCacheReqInput +from sglang.srt.managers.io_struct import FlushCacheReqInput, FlushCacheReqOutput +from sglang.srt.managers.scheduler import Scheduler from sglang.srt.managers.scheduler_components.flush_wrapper import ( SchedulerFlushWrapper, ) @@ -17,15 +18,15 @@ class TestSchedulerFlushCache(unittest.TestCase): """The wrapper plus the three collaborators it was handed.""" flush_cache = MagicMock(return_value=True) is_fully_idle = MagicMock(return_value=False) - ipc_channels = MagicMock() + send_output = MagicMock() return SimpleNamespace( flush_cache=flush_cache, is_fully_idle=is_fully_idle, - ipc_channels=ipc_channels, + send_output=send_output, flush_wrapper=SchedulerFlushWrapper( flush_cache=flush_cache, is_fully_idle=is_fully_idle, - ipc_channels=ipc_channels, + send_output=send_output, ), ) @@ -89,8 +90,9 @@ class TestSchedulerFlushCache(unittest.TestCase): self.assertIsNone(scheduler.flush_wrapper._pending) scheduler.flush_cache.assert_called_once() - out = scheduler.ipc_channels.send_to_tokenizer.send_output.call_args.args[0] + out, answered = scheduler.send_output.call_args.args self.assertTrue(out.success) + self.assertIs(answered, req) def test_pending_flush_expires_on_timeout(self): scheduler = self._new_scheduler() @@ -105,7 +107,7 @@ class TestSchedulerFlushCache(unittest.TestCase): self.assertIsNone(scheduler.flush_wrapper._pending) scheduler.flush_cache.assert_not_called() - out = scheduler.ipc_channels.send_to_tokenizer.send_output.call_args.args[0] + out = scheduler.send_output.call_args.args[0] self.assertFalse(out.success) def test_pending_flush_survives_before_deadline(self): @@ -120,7 +122,23 @@ class TestSchedulerFlushCache(unittest.TestCase): scheduler.flush_wrapper.check_pending() self.assertIsNotNone(scheduler.flush_wrapper._pending) - scheduler.ipc_channels.send_to_tokenizer.send_output.assert_not_called() + scheduler.send_output.assert_not_called() + + def test_control_output_goes_to_the_embedded_rust_server(self): + """A control reply, deferred flush included, takes the Rust server's + egress ring when the scheduler embeds one, else the tokenizer socket.""" + output = FlushCacheReqOutput(success=True) + req = FlushCacheReqInput(rid="r", timeout_s=1.0) + rust = SimpleNamespace(rust_server=MagicMock(), ipc_channels=MagicMock()) + Scheduler.send_control_output(rust, output, req) + rust.rust_server.push_control_output.assert_called_once_with(req, output) + rust.ipc_channels.send_to_tokenizer.send_output.assert_not_called() + + python = SimpleNamespace(rust_server=None, ipc_channels=MagicMock()) + Scheduler.send_control_output(python, output, req) + python.ipc_channels.send_to_tokenizer.send_output.assert_called_once_with( + output, req + ) if __name__ == "__main__":