From 574ead753a88ace24fb5870ff0f60e3db4ce09cb Mon Sep 17 00:00:00 2001 From: Kan Wu Date: Sat, 1 Aug 2026 12:13:13 -0700 Subject: [PATCH] [rust-server] fix TCP-layer TTFT stalls (#33026) Co-authored-by: Claude Fable 5 Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com> --- rust/sglang-server/src/api_server.rs | 18 +++++++++++++----- rust/sglang-server/src/runtime.rs | 22 ++++++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/rust/sglang-server/src/api_server.rs b/rust/sglang-server/src/api_server.rs index 911989b29..70d6eb7ff 100644 --- a/rust/sglang-server/src/api_server.rs +++ b/rust/sglang-server/src/api_server.rs @@ -31,7 +31,7 @@ struct AppState { } pub async fn serve( - listener: std::net::TcpListener, + socket: tokio::net::TcpSocket, senders: Senders, egress_buf: usize, server_args: Arc, @@ -61,12 +61,12 @@ pub async fn serve( .with_state(state); let app = log::apply(app, &server_args); - // The listener was already bound synchronously in `runtime::start` (so a port - // conflict fails startup); adopt it into the tokio reactor here. - let listener = match tokio::net::TcpListener::from_std(listener) { + // The socket was already bound synchronously in `runtime::start` (so a port + // conflict fails startup); start listening here, on the api runtime. + let listener = match socket.listen(1024) { Ok(l) => l, Err(e) => { - tracing::error!(error = %e, "failed to adopt pre-bound listener"); + tracing::error!(error = %e, "listen failed"); return; } }; @@ -79,6 +79,14 @@ pub async fn serve( // runtime drops → detached handlers cancel → their `AbortGuard`s fire, release // `Senders` clones → tok/detok channels close → workers exit. Full drain is // deferred (see `request_shutdown`). + // Match Python (asyncio sets TCP_NODELAY); avoids a ~13 ms + // Nagle/delayed-ACK penalty on keep-alive connections. + use axum::serve::ListenerExt; + let listener = listener.tap_io(|io| { + if let Err(e) = io.set_nodelay(true) { + tracing::debug!(error = %e, "set_nodelay failed"); + } + }); // `with_connect_info` exposes the peer address to the access-log middleware. let serve = axum::serve( listener, diff --git a/rust/sglang-server/src/runtime.rs b/rust/sglang-server/src/runtime.rs index 7119b0a86..ee966bba3 100644 --- a/rust/sglang-server/src/runtime.rs +++ b/rust/sglang-server/src/runtime.rs @@ -94,11 +94,21 @@ impl Drop for Runtime { pub fn start(cfg: RuntimeConfig) -> Result { // Bind the API server port before spawning any thread, so an unavailable // port (EADDRINUSE) is a hard startup error. - let listener = std::net::TcpListener::bind(cfg.rust_server_args.http_addr) - .map_err(|e| format!("bind {} failed: {e}", cfg.rust_server_args.http_addr))?; - listener - .set_nonblocking(true) - .map_err(|e| format!("listener set_nonblocking failed: {e}"))?; + let addr = cfg.rust_server_args.http_addr; + let socket = match addr { + std::net::SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4(), + std::net::SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6(), + } + .map_err(|e| format!("socket for {addr} failed: {e}"))?; + socket + .set_reuseaddr(true) + .map_err(|e| format!("set_reuseaddr failed: {e}"))?; + if let Err(e) = socket.set_recv_buffer_size(16 * 1024 * 1024) { + eprintln!("warning: set_recv_buffer_size on listener failed: {e}"); + } + socket + .bind(addr) + .map_err(|e| format!("bind {addr} failed: {e}"))?; let (shutdown_tx, shutdown_rx) = flume::unbounded::<()>(); let mut threads = Vec::new(); @@ -267,7 +277,7 @@ pub fn start(cfg: RuntimeConfig) -> Result { } let rt = builder.build().expect("build api runtime"); rt.block_on(api_server::serve( - listener, + socket, senders, cfg.rust_server_args.channel_cap, cfg.server_args.clone(),