From 8186eeb93930cb04e4942378651db64dbc117b02 Mon Sep 17 00:00:00 2001
From: Kan Wu <wukanustc@gmail.com>
Date: Sun, 2 Aug 2026 20:52:29 -0700
Subject: [PATCH] [rust-server] Reland: fix TCP-layer TTFT stalls (#33026)
 (#33269)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
---
 rust/Cargo.lock                      |  1 +
 rust/Cargo.toml                      |  1 +
 rust/sglang-server/Cargo.toml        |  1 +
 rust/sglang-server/src/api_server.rs |  8 ++++++++
 rust/sglang-server/src/runtime.rs    | 25 ++++++++++++++++++++++---
 5 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 4c85b1539..355859b8a 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2571,6 +2571,7 @@ dependencies = [
  "rmpv",
  "serde",
  "serde_json",
+ "socket2 0.6.4",
  "thiserror",
  "tokio",
  "tracing",
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 943cd515b..914eac885 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -21,6 +21,7 @@ futures = "0.3"
 pyo3 = { version = "0.29.0", features = ["extension-module"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+socket2 = "0.6"
 thiserror = "2"
 tokio = { version = "1", features = ["full"] }
 tokio-stream = { version = "0.1", features = ["net"] }
diff --git a/rust/sglang-server/Cargo.toml b/rust/sglang-server/Cargo.toml
index 9f3862bde..8d87255d4 100644
--- a/rust/sglang-server/Cargo.toml
+++ b/rust/sglang-server/Cargo.toml
@@ -24,6 +24,7 @@ futures = { workspace = true }
 pyo3 = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+socket2 = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
diff --git a/rust/sglang-server/src/api_server.rs b/rust/sglang-server/src/api_server.rs
index 2674f3560..80a8da927 100644
--- a/rust/sglang-server/src/api_server.rs
+++ b/rust/sglang-server/src/api_server.rs
@@ -88,6 +88,14 @@ pub async fn serve(
     // runtime drops → detached handlers cancel → their `AbortGuard`s fire, release
     // `Senders` clones → tok/detok channels close → workers exit. Full drain is
     // deferred (see `request_shutdown`).
+    // Match Python (asyncio sets TCP_NODELAY); avoids a ~13 ms
+    // Nagle/delayed-ACK penalty on keep-alive connections.
+    use axum::serve::ListenerExt;
+    let listener = listener.tap_io(|io| {
+        if let Err(e) = io.set_nodelay(true) {
+            tracing::debug!(error = %e, "set_nodelay failed");
+        }
+    });
     // `with_connect_info` exposes the peer address to the access-log middleware.
     let serve = axum::serve(
         listener,
diff --git a/rust/sglang-server/src/runtime.rs b/rust/sglang-server/src/runtime.rs
index 7119b0a86..bb2d84c40 100644
--- a/rust/sglang-server/src/runtime.rs
+++ b/rust/sglang-server/src/runtime.rs
@@ -93,9 +93,28 @@ impl Drop for Runtime {
 /// startup misconfiguration (e.g. no tokenizer for a non-skip server).
 pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
     // Bind the API server port before spawning any thread, so an unavailable
-    // port (EADDRINUSE) is a hard startup error.
-    let listener = std::net::TcpListener::bind(cfg.rust_server_args.http_addr)
-        .map_err(|e| format!("bind {} failed: {e}", cfg.rust_server_args.http_addr))?;
+    // port (EADDRINUSE) is a hard startup error. socket2 rather than
+    // `std::net::TcpListener` so SO_RCVBUF can be set before `listen`.
+    let addr = cfg.rust_server_args.http_addr;
+    let socket = socket2::Socket::new(
+        socket2::Domain::for_address(addr),
+        socket2::Type::STREAM,
+        Some(socket2::Protocol::TCP),
+    )
+    .map_err(|e| format!("socket for {addr} failed: {e}"))?;
+    socket
+        .set_reuse_address(true)
+        .map_err(|e| format!("set_reuseaddr failed: {e}"))?;
+    if let Err(e) = socket.set_recv_buffer_size(16 * 1024 * 1024) {
+        eprintln!("warning: set_recv_buffer_size on listener failed: {e}");
+    }
+    socket
+        .bind(&addr.into())
+        .map_err(|e| format!("bind {addr} failed: {e}"))?;
+    socket
+        .listen(1024)
+        .map_err(|e| format!("listen on {addr} failed: {e}"))?;
+    let listener: std::net::TcpListener = socket.into();
     listener
         .set_nonblocking(true)
         .map_err(|e| format!("listener set_nonblocking failed: {e}"))?;