From fb92ba11279b02677da01ee6782afdb6cdb6a31c Mon Sep 17 00:00:00 2001 From: jain-ria Date: Fri, 2 Oct 2026 01:44:56 +0800 Subject: [PATCH] [Rust] Wire the gRPC server into the frontend lifecycle (#41964) Signed-off-by: jain-ria --- python/sglang/srt/arg_groups/serving_hook.py | 5 + python/sglang/srt/rust_server/config.py | 8 + rust/Cargo.lock | 1 + rust/sglang-server/Cargo.toml | 1 + rust/sglang-server/src/grpc/mod.rs | 17 +- rust/sglang-server/src/grpc/server.rs | 83 +++++ rust/sglang-server/src/lib.rs | 5 +- rust/sglang-server/src/message/config.rs | 51 +++ rust/sglang-server/src/utils/runtime.rs | 290 +++++++++++++++++- rust/sglang-server/src/utils/startup.rs | 51 ++- .../entrypoints/test_rust_server_dp_ports.py | 45 ++- .../unit/server_args/test_server_args.py | 8 + 12 files changed, 533 insertions(+), 32 deletions(-) create mode 100644 rust/sglang-server/src/grpc/server.rs diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py index e18fc6648a..452637ee11 100644 --- a/python/sglang/srt/arg_groups/serving_hook.py +++ b/python/sglang/srt/arg_groups/serving_hook.py @@ -368,6 +368,11 @@ def handle_deprecated_args(server_args: Any): ) if cfg.grpc_port is None: raise ValueError("--sidecar requires --grpc-port or SGLANG_GRPC_PORT.") + if envs.SGLANG_RUST_SERVER.get(): + raise ValueError( + "--sidecar is not yet supported with SGLANG_RUST_SERVER: " + "Rust-server mode does not run the Python sidecar lifecycle." + ) if native_grpc: if cfg.use_ray: raise ValueError( diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index e7137ea3a1..742a5a0b89 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -47,6 +47,11 @@ def _build_server_args( "prefill": ext.DisaggregationMode.Prefill, "decode": ext.DisaggregationMode.Decode, }[get_disagg().disaggregation_mode] + grpc_port = ( + None + if get_serving().smg_grpc_mode or get_serving().grpc_mode + else get_serving().grpc_port + ) return ext.ServerArgs( model_path=get_model().model_path, served_model_name=get_serving().served_model_name, @@ -56,6 +61,9 @@ def _build_server_args( weight_version=get_serving().weight_version, host=get_serving().host, port=get_serving().port, + # The shared field is also populated for legacy SMG mode; only forward + # it when it selects SGLang's native gRPC transport. + grpc_port=grpc_port, log_level=get_observability().log_level, log_level_http=get_observability().log_level_http, chat_template=get_serving().chat_template, diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0860fcf145..2e2b522046 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -3814,6 +3814,7 @@ dependencies = [ "socket2 0.6.5", "thiserror", "tokio", + "tokio-stream", "tonic", "tower 0.5.3", "tracing", diff --git a/rust/sglang-server/Cargo.toml b/rust/sglang-server/Cargo.toml index e97f5cf66f..58a0feb28e 100644 --- a/rust/sglang-server/Cargo.toml +++ b/rust/sglang-server/Cargo.toml @@ -69,6 +69,7 @@ sglang-grpc-types = { path = "../../proto" } # naturally while the crate keeps its own artifact name in the shared target/. sglang_mm = { package = "sglang-mm", path = "../sglang-mm", default-features = false } tonic = { workspace = true } +tokio-stream = { workspace = true } [dev-dependencies] # `Router::oneshot` for handler-level router tests. diff --git a/rust/sglang-server/src/grpc/mod.rs b/rust/sglang-server/src/grpc/mod.rs index fdeb9bb6f0..1a3a5e5f16 100644 --- a/rust/sglang-server/src/grpc/mod.rs +++ b/rust/sglang-server/src/grpc/mod.rs @@ -1,9 +1,8 @@ //! Tonic adapter for the canonical `sglang.runtime.v1` API. //! -//! This module only translates protobuf requests and responses. The shared -//! [`FrontendHandle`] owns preprocessing, admission, runtime communication, and -//! request cancellation; listener construction and server lifecycle are added -//! by the next stack layer. +//! This module translates protobuf requests and responses and mounts that thin +//! adapter on Tonic. The shared [`FrontendHandle`] owns preprocessing, +//! admission, runtime communication, and request cancellation. //! //! The thin Tonic-service structure, streamed response approach, and test //! strategy build on Rain Jiang's multi-protocol prototype in @@ -23,11 +22,13 @@ use crate::message::config::{PreferredSamplingParams, ServerArgs}; mod convert; mod response; +mod server; + +pub(crate) use server::serve; #[cfg(test)] mod tests; -#[cfg_attr(test, allow(dead_code))] const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300); type ResponseStream = Pin> + Send + 'static>>; @@ -43,18 +44,12 @@ struct AdapterConfig { } /// Tonic-facing implementation backed by the transport-neutral Rust frontend. -/// -/// The listener follow-up constructs this service. The temporary dead-code -/// allowance keeps this adapter-only stack layer warning-clean on its own. -#[cfg_attr(not(test), allow(dead_code))] pub(crate) struct GrpcService { frontend: FrontendHandle, config: AdapterConfig, } -#[cfg_attr(not(test), allow(dead_code))] impl GrpcService { - #[cfg_attr(test, allow(dead_code))] pub(crate) fn new(frontend: FrontendHandle, server_args: &ServerArgs) -> Self { Self { frontend, diff --git a/rust/sglang-server/src/grpc/server.rs b/rust/sglang-server/src/grpc/server.rs new file mode 100644 index 0000000000..2b440b70ab --- /dev/null +++ b/rust/sglang-server/src/grpc/server.rs @@ -0,0 +1,83 @@ +//! Tonic listener lifecycle for the native Rust frontend. +//! +//! The pre-bound socket, shared API runtime, and shutdown shape build on Rain +//! Jiang's multi-protocol prototype in `sgl-project/sglang#36923`. Unlike that +//! prototype, the listener mounts the canonical `runtime.v1` adapter on the +//! transport-neutral [`crate::frontend::FrontendHandle`] contract. + +use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangServiceServer; +use tokio_stream::wrappers::TcpListenerStream; + +use super::GrpcService; + +/// Keep the native Rust listener aligned with the existing Python-backed +/// runtime.v1 server. Tonic's 4 MiB default is too small for large token or +/// multimodal request bodies. +const DEFAULT_GRPC_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024; + +fn max_message_size() -> usize { + match std::env::var("SGLANG_TONIC_PAYLOAD") { + Ok(raw) => match raw.parse::() { + Ok(bytes) if bytes > 0 => { + tracing::info!( + bytes, + "using SGLANG_TONIC_PAYLOAD override for gRPC max message size" + ); + bytes + } + _ => { + tracing::warn!( + value = %raw, + default = DEFAULT_GRPC_MAX_MESSAGE_SIZE, + "ignoring invalid SGLANG_TONIC_PAYLOAD; using default" + ); + DEFAULT_GRPC_MAX_MESSAGE_SIZE + } + }, + Err(_) => DEFAULT_GRPC_MAX_MESSAGE_SIZE, + } +} + +/// Serve the already-constructed adapter on a pre-bound listener until the +/// runtime-wide shutdown signal fires. +/// +/// Selecting away from `serve_with_incoming` stops accepting without waiting +/// indefinitely for unfinished streams. Once both transport futures return, +/// the owner drops their shared API runtime, which cancels Tonic's active +/// connection tasks within the frontend's bounded thread-join deadline. +pub(crate) async fn serve( + listener: std::net::TcpListener, + service: GrpcService, + shutdown: flume::Receiver<()>, +) { + let addr = listener.local_addr().ok(); + let listener = match tokio::net::TcpListener::from_std(listener) { + Ok(listener) => listener, + Err(error) => { + tracing::error!(%error, "failed to adopt pre-bound gRPC listener"); + return; + } + }; + let incoming = TcpListenerStream::new(listener); + let max_message_size = max_message_size(); + let service = SglangServiceServer::new(service) + .max_decoding_message_size(max_message_size) + .max_encoding_message_size(max_message_size); + let serve = tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming(incoming); + + if let Some(addr) = addr { + tracing::info!(%addr, "gRPC server listening"); + } + tokio::select! { + result = serve => { + if let Err(error) = result { + tracing::error!(%error, "gRPC serve exited"); + } + } + _ = shutdown.recv_async() => { + tracing::info!("shutdown: stopping gRPC accepts; API runtime teardown cancels in-flight RPCs"); + } + } +} diff --git a/rust/sglang-server/src/lib.rs b/rust/sglang-server/src/lib.rs index e6d9a18d18..64116e14a8 100644 --- a/rust/sglang-server/src/lib.rs +++ b/rust/sglang-server/src/lib.rs @@ -41,7 +41,7 @@ use pyo3::pybacked::PyBackedBytes; use pyo3::types::PyBytes; use crate::message::config::RuntimeConfig; -use crate::utils::startup::{listen_addr, value_error}; +use crate::utils::startup::{grpc_listen_addr, listen_addr, value_error}; use crate::utils::{logging, runtime, shm::ShmSegment}; /// One drained request handed to Python by [`Server::recv_requests`]: the @@ -194,10 +194,13 @@ impl Server { // their offset so this boundary has one source of truth for the address. let http_addr = listen_addr(&server_args, port_offset) .map_err(|e| value_error("bad listen address", e))?; + let grpc_addr = grpc_listen_addr(&server_args, port_offset) + .map_err(|e| value_error("bad gRPC listen address", e))?; let cfg = RuntimeConfig { rust_server_args: RustServerServerArgs { http_addr, + grpc_addr, http_api_worker_num: server_args.http_api_worker_num(), to_scheduler_cap, from_scheduler_cap, diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index 13d0ec5256..59fabea7ca 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -27,6 +27,8 @@ use serde::Serialize; #[derive(Clone, Debug)] pub struct RustServerServerArgs { pub http_addr: SocketAddr, + /// The gRPC transport's listen address; `None` keeps it disabled. + pub grpc_addr: Option, pub http_api_worker_num: usize, pub to_scheduler_cap: usize, pub from_scheduler_cap: usize, @@ -40,6 +42,7 @@ impl Default for RustServerServerArgs { fn default() -> Self { Self { http_addr: "127.0.0.1:30000".parse().unwrap(), + grpc_addr: None, http_api_worker_num: 2, to_scheduler_cap: 8192, from_scheduler_cap: 8192, @@ -98,6 +101,9 @@ pub struct ServerArgs { /// HTTP bind address (see [`Self::bind`]). pub host: String, pub port: u16, + /// Optional gRPC base port. The same per-rank offset as HTTP is applied at + /// the Python→Rust startup boundary; `None` keeps gRPC disabled. + pub grpc_port: Option, /// Log levels driving the access log — uvicorn runs at /// `log_level_http or log_level` (see [`Self::http_access_log_enabled`]). pub log_level: String, @@ -165,6 +171,7 @@ impl ServerArgs { weight_version, host, port, + grpc_port, log_level, log_level_http, chat_template, @@ -197,6 +204,7 @@ impl ServerArgs { weight_version: Option, host: String, port: u16, + grpc_port: Option, log_level: String, log_level_http: Option, chat_template: Option, @@ -227,6 +235,7 @@ impl ServerArgs { weight_version, host, port, + grpc_port, log_level, log_level_http, chat_template, @@ -265,6 +274,7 @@ impl Default for ServerArgs { weight_version: None, host: "127.0.0.1".into(), port: 30000, + grpc_port: None, log_level: "info".into(), log_level_http: None, chat_template: None, @@ -550,6 +560,17 @@ impl ServerArgs { if self.served_model_name.is_empty() { return Err("empty 'served_model_name' in server_args".into()); } + if let Some(grpc_port) = self.grpc_port { + if grpc_port == 0 { + return Err("'grpc_port' must be between 1 and 65535".into()); + } + if grpc_port == self.port { + return Err(format!( + "'grpc_port' ({grpc_port}) must differ from 'port' ({})", + self.port + )); + } + } if let Some(preferred) = &self.preferred_sampling_params { super::sampling::SamplingParamsInput::from_preferred(&preferred.0) .map_err(|e| format!("invalid preferred_sampling_params: {e}"))?; @@ -657,6 +678,36 @@ mod tests { assert!(sa.validate().is_ok()); } + #[test] + fn grpc_is_disabled_by_default_and_rejects_invalid_ports() { + assert!(ServerArgs::default().grpc_port.is_none()); + assert!(RustServerServerArgs::default().grpc_addr.is_none()); + + let base = ServerArgs { + served_model_name: "m".into(), + ..Default::default() + }; + assert!(base.validate().is_ok()); + assert!( + ServerArgs { + grpc_port: Some(0), + ..base.clone() + } + .validate() + .unwrap_err() + .contains("between 1 and 65535") + ); + assert!( + ServerArgs { + grpc_port: Some(base.port), + ..base + } + .validate() + .unwrap_err() + .contains("must differ") + ); + } + /// `--log-level-http` overrides `--log-level` for the access log; unset or /// empty falls through. #[test] diff --git a/rust/sglang-server/src/utils/runtime.rs b/rust/sglang-server/src/utils/runtime.rs index fc938ba35f..378c2554b9 100644 --- a/rust/sglang-server/src/utils/runtime.rs +++ b/rust/sglang-server/src/utils/runtime.rs @@ -3,7 +3,7 @@ //! `recv_requests` and `push_decode_result_batch`. //! //! Thread layout: -//! * API server — tokio multi-thread runtime (I/O bound), pinned core set A +//! * API transports — one tokio multi-thread runtime (I/O bound), pinned core set A //! * Tokenizer — N pinned OS threads (CPU bound), core set B //! * Detokenizer — M pinned OS threads (CPU bound), core set C //! * To_scheduler — 1 thread driving the FSM @@ -12,7 +12,7 @@ //! [`Runtime::start_mm_workers`] (multimodal models only) //! //! Keeping CPU-bound tokenize/detokenize off the async executor avoids stalling -//! axum's worker threads. +//! the HTTP/gRPC worker threads. use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -180,6 +180,22 @@ pub fn start(cfg: RuntimeConfig) -> Result { .as_ref() .map(|t| Arc::new(tokenizer::DynamoTokenizer::new(t.clone())) as _); + // The potentially slow/fallible tokenizer load above happens before the + // ports become visible. Own both sockets before starting any worker or + // transport thread, though, so startup remains all-or-nothing: if either + // configured port is unavailable, both local listeners are dropped and no + // partial runtime needs cleanup. + let http_addr = cfg.rust_server_args.http_addr; + let http_listener = bind_tcp_listener(http_addr) + .map_err(|e| format!("binding HTTP listener on {http_addr} failed: {e}"))?; + let grpc_listener = match cfg.rust_server_args.grpc_addr { + Some(addr) => Some( + bind_tcp_listener(addr) + .map_err(|e| format!("binding gRPC listener on {addr} failed: {e}"))?, + ), + None => None, + }; + // --- Detokenizer shards (pinned, CPU bound) --- { // Default: a real tokenizer decodes to text. `None` (→ `Skip`, raw @@ -298,18 +314,15 @@ pub fn start(cfg: RuntimeConfig) -> Result { }, ); - // --- HTTP adapter (tokio, I/O bound) --- + // --- HTTP + optional gRPC adapters (one tokio runtime, I/O bound) --- { let cfg = cfg.clone(); let api_cores = plan.as_ref().map(|p| p.api.clone()); - let frontend = frontend.clone(); let shutdown_rx = shutdown_rx.clone(); - // Bind synchronously so an unavailable port (EADDRINUSE) is a hard - // startup error. The `?` drops `shutdown_tx`/`senders`, which stops the - // launcher process. - let http_addr = cfg.rust_server_args.http_addr; - let listener = bind_tcp_listener(http_addr) - .map_err(|e| format!("binding API listener on {} failed: {e}", http_addr))?; + let grpc_server = grpc_listener.map(|listener| { + let service = crate::grpc::GrpcService::new(frontend.clone(), &cfg.server_args); + (listener, service) + }); let handle = std::thread::Builder::new() .name("api-runtime".into()) .spawn(move || { @@ -327,12 +340,19 @@ pub fn start(cfg: RuntimeConfig) -> Result { }); } let rt = builder.build().expect("build api runtime"); - rt.block_on(api_server::app::serve( - listener, - frontend, - cfg.server_args.clone(), - shutdown_rx, - )) + rt.block_on(async move { + let http = api_server::app::serve( + http_listener, + frontend, + cfg.server_args.clone(), + shutdown_rx.clone(), + ); + if let Some((listener, service)) = grpc_server { + tokio::join!(http, crate::grpc::serve(listener, service, shutdown_rx)); + } else { + http.await; + } + }) }) .expect("spawn api runtime"); threads.push(handle); @@ -354,6 +374,71 @@ pub fn start(cfg: RuntimeConfig) -> Result { mod tests { use super::*; use crate::message::config::{RuntimeConfig, RustServerServerArgs, ServerArgs}; + use crate::message::response::{BatchHeader, frame_decode_batch_cols}; + use sglang_grpc_types::sglang::runtime::v1 as proto; + use sglang_grpc_types::sglang::runtime::v1::sglang_service_client::SglangServiceClient; + + fn free_loopback_addrs() -> (std::net::SocketAddr, std::net::SocketAddr) { + // Hold both probes at once so the OS cannot return the same ephemeral + // port twice. Release them together immediately before runtime startup. + let first = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let second = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addrs = (first.local_addr().unwrap(), second.local_addr().unwrap()); + drop((first, second)); + addrs + } + + fn test_config( + http_addr: std::net::SocketAddr, + grpc_addr: Option, + ) -> RuntimeConfig { + RuntimeConfig { + rust_server_args: RustServerServerArgs { + http_addr, + grpc_addr, + http_api_worker_num: 1, + ..Default::default() + }, + server_args: Arc::new(test_server_args()), + } + } + + async fn connect_grpc( + addr: std::net::SocketAddr, + ) -> SglangServiceClient { + let endpoint = tonic::transport::Endpoint::from_shared(format!("http://{addr}")) + .unwrap() + .connect_timeout(std::time::Duration::from_secs(2)); + SglangServiceClient::new(endpoint.connect().await.expect("connect gRPC client")) + } + + fn scheduler_rid(header: &[u8]) -> String { + let value = rmpv::decode::read_value(&mut std::io::Cursor::new(header)).unwrap(); + value + .as_array() + .and_then(|fields| fields.get(1)) + .and_then(rmpv::Value::as_str) + .expect("TokenizedGenerateReqInput rid") + .to_owned() + } + + fn terminal_token_frame(rid: String, output_ids: &[i64]) -> bytes::Bytes { + let header = BatchHeader { + rids: vec![rid], + finish_reasons: vec![Some( + serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap(), + )], + prompt_tokens: vec![3], + tok_lens: vec![output_ids.len() as u32], + ..Default::default() + }; + let header = rmp_serde::to_vec(&header).unwrap(); + let data = output_ids + .iter() + .flat_map(|token| token.to_le_bytes()) + .collect::>(); + frame_decode_batch_cols(&header, &[&data]) + } /// Minimal boot config: no tokenizer load, complete `model_config` (from /// `Default`), unified role. @@ -532,4 +617,177 @@ mod tests { }; assert!(err.contains("bind"), "error should mention bind: {err}"); } + + /// A configured gRPC bind is part of startup, not a best-effort side task: + /// failure returns synchronously and releases the HTTP listener acquired + /// immediately before it. + #[test] + fn grpc_port_conflict_fails_startup_without_leaking_http_listener() { + let grpc_hog = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let grpc_addr = grpc_hog.local_addr().unwrap(); + // Choose HTTP while the gRPC port remains occupied, guaranteeing that + // the two addresses differ. + let http_probe = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let http_addr = http_probe.local_addr().unwrap(); + drop(http_probe); + + let error = match start(test_config(http_addr, Some(grpc_addr))) { + Ok(_) => panic!("gRPC bind conflict must fail startup"), + Err(error) => error, + }; + assert!(error.contains("gRPC listener"), "unexpected error: {error}"); + assert!(error.contains(&grpc_addr.to_string())); + + let rebound = std::net::TcpListener::bind(http_addr) + .expect("failed startup must release its pre-bound HTTP listener"); + drop(rebound); + } + + /// End to end over the configured socket: the generated runtime.v1 client + /// reaches the existing scheduler ring, and an existing scheduler frame is + /// translated back into a protobuf response. Shutdown then closes both + /// transport listeners. + #[test] + fn configured_grpc_serves_generation_and_closes_with_http() { + use std::time::Duration; + + let (http_addr, grpc_addr) = free_loopback_addrs(); + let rt = start(test_config(http_addr, Some(grpc_addr))).expect("start runtime"); + assert!(std::net::TcpStream::connect(http_addr).is_ok()); + + let client_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut stream = client_runtime.block_on(async { + let mut client = connect_grpc(grpc_addr).await; + tokio::time::timeout( + Duration::from_secs(5), + client.generate(proto::GenerateRequest { + input_ids: vec![1, 2, 3], + stream: Some(true), + rid: Some("network-generation".into()), + ..Default::default() + }), + ) + .await + .expect("timed out starting Generate RPC") + .expect("Generate RPC") + .into_inner() + }); + + assert!( + rt.to_scheduler_rx.wait(Duration::from_secs(2)), + "gRPC request did not reach the scheduler ring" + ); + let requests = rt.to_scheduler_rx.drain(1); + assert_eq!(requests.len(), 1); + let input = requests[0] + .buffers + .iter() + .find(|buffer| buffer.name == "input_ids") + .expect("input token buffer"); + assert_eq!(input.shape, vec![3]); + let rid = scheduler_rid(&requests[0].header); + assert!( + rt.from_scheduler_tx + .push(terminal_token_frame(rid, &[9, 10])) + ); + + let response = client_runtime + .block_on(async { + tokio::time::timeout(Duration::from_secs(2), stream.message()).await + }) + .expect("timed out waiting for gRPC response") + .expect("gRPC stream error") + .expect("gRPC stream ended without a response"); + assert_eq!(response.output_ids, vec![9, 10]); + assert!(response.finished); + assert_eq!(response.meta_info["id"], r#""network-generation""#); + + rt.request_shutdown(); + assert!(std::net::TcpStream::connect(http_addr).is_err()); + assert!(std::net::TcpStream::connect(grpc_addr).is_err()); + } + + /// The existing runtime.v1 listener accepts bodies above Tonic's 4 MiB + /// default. Keep that transport policy when mounting the same protocol on + /// the Rust frontend; reaching adapter validation proves the body decoded. + #[test] + fn grpc_listener_preserves_existing_large_message_limit() { + use tonic::Code; + + let (http_addr, grpc_addr) = free_loopback_addrs(); + let rt = start(test_config(http_addr, Some(grpc_addr))).expect("start runtime"); + let client_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let error = client_runtime.block_on(async { + let client = connect_grpc(grpc_addr).await; + let mut client = client.max_encoding_message_size(64 * 1024 * 1024); + tokio::time::timeout( + std::time::Duration::from_secs(10), + client.generate(proto::GenerateRequest { + input_ids: vec![1], + // This field is intentionally unsupported. An + // UNIMPLEMENTED status proves the >4 MiB protobuf reached + // the adapter instead of Tonic rejecting it on size. + routing_key: Some("x".repeat(5 * 1024 * 1024)), + ..Default::default() + }), + ) + .await + .expect("timed out sending large Generate RPC") + .expect_err("unsupported routing_key must fail") + }); + assert_eq!(error.code(), Code::Unimplemented); + + rt.request_shutdown(); + assert!(std::net::TcpStream::connect(grpc_addr).is_err()); + } + + /// Tonic's unbounded graceful-drain API waits for open HTTP/2 connections. + /// Keep a generated response stream live to ensure the frontend's shared + /// shutdown remains bounded and cancels it instead. + #[test] + fn grpc_shutdown_returns_promptly_with_in_flight_generation() { + use std::time::{Duration, Instant}; + + let (http_addr, grpc_addr) = free_loopback_addrs(); + let rt = start(test_config(http_addr, Some(grpc_addr))).expect("start runtime"); + let client_runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let (_client, _stream) = client_runtime.block_on(async { + let mut client = connect_grpc(grpc_addr).await; + let stream = tokio::time::timeout( + Duration::from_secs(5), + client.generate(proto::GenerateRequest { + input_ids: vec![1, 2, 3], + stream: Some(true), + rid: Some("in-flight".into()), + ..Default::default() + }), + ) + .await + .expect("timed out starting Generate RPC") + .expect("Generate RPC") + .into_inner(); + (client, stream) + }); + assert!(rt.to_scheduler_rx.wait(Duration::from_secs(2))); + + let started = Instant::now(); + rt.request_shutdown(); + assert!( + started.elapsed() < Duration::from_secs(3), + "shutdown waited for an open gRPC stream: {:?}", + started.elapsed() + ); + assert!(std::net::TcpStream::connect(http_addr).is_err()); + assert!(std::net::TcpStream::connect(grpc_addr).is_err()); + } } diff --git a/rust/sglang-server/src/utils/startup.rs b/rust/sglang-server/src/utils/startup.rs index bde58332bf..6cb94e21f0 100644 --- a/rust/sglang-server/src/utils/startup.rs +++ b/rust/sglang-server/src/utils/startup.rs @@ -15,12 +15,29 @@ pub(crate) fn value_error(context: &str, err: impl std::fmt::Display) -> PyErr { pub(crate) fn listen_addr( server_args: &ServerArgs, port_offset: Option, +) -> Result { + listen_addr_for_port(server_args, server_args.port, port_offset) +} + +pub(crate) fn grpc_listen_addr( + server_args: &ServerArgs, + port_offset: Option, +) -> Result, String> { + server_args + .grpc_port + .map(|port| listen_addr_for_port(server_args, port, port_offset)) + .transpose() +} + +fn listen_addr_for_port( + server_args: &ServerArgs, + base_port: u16, + port_offset: Option, ) -> Result { let offset = port_offset.unwrap_or_default(); - let port = server_args - .port + let port = base_port .checked_add(offset) - .ok_or_else(|| format!("port {} + offset {offset} exceeds 65535", server_args.port))?; + .ok_or_else(|| format!("port {base_port} + offset {offset} exceeds 65535"))?; let mut addr: SocketAddr = server_args .bind() .parse() @@ -58,4 +75,32 @@ mod tests { }; assert!(listen_addr(&args, Some(1)).unwrap_err().contains("exceeds")); } + + #[test] + fn grpc_listen_addr_is_optional_and_uses_the_same_offset() { + let mut args = ServerArgs { + host: "::".into(), + ..Default::default() + }; + assert_eq!(grpc_listen_addr(&args, Some(7)).unwrap(), None); + + args.grpc_port = Some(50_051); + assert_eq!( + grpc_listen_addr(&args, Some(7)).unwrap(), + Some("[::]:50058".parse().unwrap()) + ); + } + + #[test] + fn grpc_listen_addr_rejects_port_overflow() { + let args = ServerArgs { + grpc_port: Some(u16::MAX), + ..Default::default() + }; + assert!( + grpc_listen_addr(&args, Some(1)) + .unwrap_err() + .contains("exceeds") + ); + } } diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index 55b0aa785e..b4cd4e7b00 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -1,17 +1,60 @@ from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from sglang.srt import rust_extensions from sglang.srt.entrypoints.engine import node_hosts_rust_server from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.rust_server import config as rust_config from sglang.srt.rust_server import server as rust_server from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=3, suite="base-a-test-cpu") +def _scheduler_for_typed_config(): + return SimpleNamespace( + server_args=SimpleNamespace(enable_return_hidden_states=False), + model_config=SimpleNamespace( + context_len=2048, + vocab_size=1000, + is_multimodal=False, + hf_config=SimpleNamespace(model_type=None), + get_default_sampling_params=lambda: {}, + ), + rust_server_tokenizer_path=lambda: "tokenizer", + max_total_num_tokens=1024, + ) + + +@pytest.mark.parametrize( + "legacy_args,expected", + [({}, 50051), ({"smg_grpc_mode": True}, None), ({"grpc_mode": True}, None)], + ids=["native", "legacy-smg", "deprecated-legacy-smg"], +) +def test_typed_config_only_forwards_native_grpc_port(legacy_args, expected): + extension = SimpleNamespace( + DisaggregationMode=SimpleNamespace( + Null="null", Prefill="prefill", Decode="decode" + ), + ModelConfig=MagicMock(return_value="model-config"), + DefaultSamplingParams=MagicMock(return_value="sampling-defaults"), + ServerArgs=MagicMock(return_value="server-args"), + ) + with ( + get_context().override_server_args(grpc_port=50051, **legacy_args), + patch.object(rust_extensions, "load_rust_extension", return_value=extension), + patch.object(rust_config, "compute_num_reserved_tokens", return_value=0), + ): + assert ( + rust_config._build_server_args(_scheduler_for_typed_config()) + == "server-args" + ) + + assert extension.ServerArgs.call_args.kwargs["grpc_port"] == expected + + @pytest.mark.parametrize( "nnodes,tp_size,attn_dp_size,ep_join_mode,ranks,expected", [ diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index be2872e2f8..3dfd227254 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -3492,6 +3492,14 @@ class TestGrpcServerArgs(CustomTestCase): with self.assertRaisesRegex(ValueError, "must not be empty"): handle_deprecated_args(sa) + def test_sidecar_rejects_rust_server_lifecycle(self): + sa = self._args(sidecar="example.sidecar", grpc_port=50051) + with ( + envs.SGLANG_RUST_SERVER.override(True), + self.assertRaisesRegex(ValueError, "does not run the Python sidecar"), + ): + handle_deprecated_args(sa) + def test_sidecar_sets_endpoint_env_before_import_and_calls_main(self): main = MagicMock()