From 80263b558081265f70398ca2ae5fd008c12eccde Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:43:54 +0800 Subject: [PATCH] [Fix] sgl-router: Do not abort forwarded requests on reqwest's default 30 s TCP_USER_TIMEOUT (#45696) --- sgl-router/src/proxy/mod.rs | 5 ++ sgl-router/tests/proxy/timeout.rs | 94 +++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/sgl-router/src/proxy/mod.rs b/sgl-router/src/proxy/mod.rs index 63cd54ce66..0b52c880af 100644 --- a/sgl-router/src/proxy/mod.rs +++ b/sgl-router/src/proxy/mod.rs @@ -139,6 +139,11 @@ fn build_client(protocol: WireProtocol) -> Result { // Under SGLang's default 5 s keep-alive, so idle connections close here first. .pool_idle_timeout(Duration::from_secs(3)) .connect_timeout(Duration::from_secs(5)); + // reqwest sets TCP_USER_TIMEOUT to 30 s by default, so a forwarded request is aborted + // with ETIMEDOUT when the engine reads none of its body for 30 s, e.g. while its HTTP + // process is busy with a burst of long prompts. request_timeout bounds requests instead. + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + let builder = builder.tcp_user_timeout(None); match protocol { WireProtocol::Http1 => builder, WireProtocol::H2c => builder.http2_prior_knowledge(), diff --git a/sgl-router/tests/proxy/timeout.rs b/sgl-router/tests/proxy/timeout.rs index 70e919b7d3..ff143e2945 100644 --- a/sgl-router/tests/proxy/timeout.rs +++ b/sgl-router/tests/proxy/timeout.rs @@ -221,3 +221,97 @@ async fn streaming_request_times_out_waiting_for_headers() { assert_eq!(res.status(), StatusCode::OK); } } + +/// A worker that accepts the connection but reads none of the request for `stall`, then +/// reads the whole request and answers 200. Returns its URL. +#[cfg(target_os = "linux")] +async fn spawn_slow_reading_worker(stall: Duration) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + tokio::time::sleep(stall).await; + let mut buf = Vec::new(); + let mut chunk = vec![0u8; 1 << 16]; + let header_end = loop { + let n = sock.read(&mut chunk).await.unwrap(); + assert!( + n > 0, + "router closed the connection before sending the request" + ); + buf.extend_from_slice(&chunk[..n]); + if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break i + 4; + } + }; + let headers = String::from_utf8_lossy(&buf[..header_end]).to_ascii_lowercase(); + let content_length: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .expect("request must carry content-length") + .trim() + .parse() + .unwrap(); + while buf.len() < header_end + content_length { + let n = sock.read(&mut chunk).await.unwrap(); + assert!(n > 0, "router closed the connection mid-body"); + buf.extend_from_slice(&chunk[..n]); + } + sock.write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}") + .await + .unwrap(); + }); + url +} + +/// An engine whose HTTP process reads nothing for longer than 30 s (busy with a burst of +/// long prompts) must still get the request once it reads: the forwarding client must not +/// abort its unacknowledged send on TCP_USER_TIMEOUT, which reqwest sets to 30 s by default. +/// Takes about 40 s; run with `cargo test --test proxy -- --ignored reads_nothing`. +#[cfg(target_os = "linux")] +#[tokio::test] +#[ignore = "takes about 40 s"] +async fn request_survives_worker_that_reads_nothing_for_over_30s() { + let url = spawn_slow_reading_worker(Duration::from_secs(40)).await; + let mut cfg = config(&url); + cfg.model.disable_input_ids_forwarding = true; + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + let _ = registry.add(WorkerSpec { + id: WorkerId("w1".into()), + url, + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + ..Default::default() + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(Duration::from_secs(120)).unwrap()); + let ctx = Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)); + let app = build_router(ctx); + + // Far more than the socket buffers hold while the worker reads nothing, so the + // router's send stalls on a zero window. + let prompt = "a".repeat(16 << 20); + let req = Request::post("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": prompt}], + "stream": false + })) + .unwrap(), + )) + .unwrap(); + let res = tokio::time::timeout(Duration::from_secs(90), app.oneshot(req)) + .await + .expect("router must answer once the worker reads") + .unwrap(); + assert_eq!( + res.status(), + StatusCode::OK, + "x-router-error-code: {:?}", + res.headers().get("x-router-error-code") + ); +}