[Fix] sgl-router: Do not abort forwarded requests on reqwest's default 30 s TCP_USER_TIMEOUT (#45696)
This commit is contained in:
@@ -139,6 +139,11 @@ fn build_client(protocol: WireProtocol) -> Result<Client, anyhow::Error> {
|
||||
// Under SGLang's default 5 s keep-alive, so idle connections close here first.
|
||||
.pool_idle_timeout(Duration::from_secs(3))
|
||||
.connect_timeout(Duration::from_secs(5));
|
||||
// reqwest sets TCP_USER_TIMEOUT to 30 s by default, so a forwarded request is aborted
|
||||
// with ETIMEDOUT when the engine reads none of its body for 30 s, e.g. while its HTTP
|
||||
// process is busy with a burst of long prompts. request_timeout bounds requests instead.
|
||||
#[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))]
|
||||
let builder = builder.tcp_user_timeout(None);
|
||||
match protocol {
|
||||
WireProtocol::Http1 => builder,
|
||||
WireProtocol::H2c => builder.http2_prior_knowledge(),
|
||||
|
||||
@@ -221,3 +221,97 @@ async fn streaming_request_times_out_waiting_for_headers() {
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
}
|
||||
}
|
||||
|
||||
/// A worker that accepts the connection but reads none of the request for `stall`, then
|
||||
/// reads the whole request and answers 200. Returns its URL.
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn spawn_slow_reading_worker(stall: Duration) -> String {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move {
|
||||
let (mut sock, _) = listener.accept().await.unwrap();
|
||||
tokio::time::sleep(stall).await;
|
||||
let mut buf = Vec::new();
|
||||
let mut chunk = vec![0u8; 1 << 16];
|
||||
let header_end = loop {
|
||||
let n = sock.read(&mut chunk).await.unwrap();
|
||||
assert!(
|
||||
n > 0,
|
||||
"router closed the connection before sending the request"
|
||||
);
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
|
||||
break i + 4;
|
||||
}
|
||||
};
|
||||
let headers = String::from_utf8_lossy(&buf[..header_end]).to_ascii_lowercase();
|
||||
let content_length: usize = headers
|
||||
.lines()
|
||||
.find_map(|l| l.strip_prefix("content-length:"))
|
||||
.expect("request must carry content-length")
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
while buf.len() < header_end + content_length {
|
||||
let n = sock.read(&mut chunk).await.unwrap();
|
||||
assert!(n > 0, "router closed the connection mid-body");
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
}
|
||||
sock.write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}")
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
url
|
||||
}
|
||||
|
||||
/// An engine whose HTTP process reads nothing for longer than 30 s (busy with a burst of
|
||||
/// long prompts) must still get the request once it reads: the forwarding client must not
|
||||
/// abort its unacknowledged send on TCP_USER_TIMEOUT, which reqwest sets to 30 s by default.
|
||||
/// Takes about 40 s; run with `cargo test --test proxy -- --ignored reads_nothing`.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
#[ignore = "takes about 40 s"]
|
||||
async fn request_survives_worker_that_reads_nothing_for_over_30s() {
|
||||
let url = spawn_slow_reading_worker(Duration::from_secs(40)).await;
|
||||
let mut cfg = config(&url);
|
||||
cfg.model.disable_input_ids_forwarding = true;
|
||||
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
|
||||
let registry = Arc::new(WorkerRegistry::default());
|
||||
let _ = registry.add(WorkerSpec {
|
||||
id: WorkerId("w1".into()),
|
||||
url,
|
||||
mode: WorkerMode::Plain,
|
||||
model_ids: vec![ModelId("tiny".into())],
|
||||
..Default::default()
|
||||
});
|
||||
let policies = Arc::new(build_policy_registry(&cfg).unwrap());
|
||||
let proxy = Arc::new(Proxy::new(Duration::from_secs(120)).unwrap());
|
||||
let ctx = Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies));
|
||||
let app = build_router(ctx);
|
||||
|
||||
// Far more than the socket buffers hold while the worker reads nothing, so the
|
||||
// router's send stalls on a zero window.
|
||||
let prompt = "a".repeat(16 << 20);
|
||||
let req = Request::post("/v1/chat/completions")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({
|
||||
"model": "tiny",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": false
|
||||
}))
|
||||
.unwrap(),
|
||||
))
|
||||
.unwrap();
|
||||
let res = tokio::time::timeout(Duration::from_secs(90), app.oneshot(req))
|
||||
.await
|
||||
.expect("router must answer once the worker reads")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
res.status(),
|
||||
StatusCode::OK,
|
||||
"x-router-error-code: {:?}",
|
||||
res.headers().get("x-router-error-code")
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user