sglang/experimental/sgl-router/tests/proxy/timeout.rs

// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0

//! Tests that the router does not wedge indefinitely when an upstream
//! worker accepts the TCP connection but never sends response headers.
//!
//! Without a configured `.timeout(...)` on the reqwest client, a stalled
//! backend hangs the axum handler future forever and the test harness
//! would just timeout. We assert here that the router returns a fast,
//! clean 504 (`upstream_timeout`) instead — a timeout is a gateway timeout,
//! the same status class as the stale-deadline cancel.

use axum::body::Body;
use axum::http::{Request, StatusCode};
use http_body_util::BodyExt;
use sgl_router::config::{
    Config, DiscoveryBackend, InflightLoadConfig, ModelConfig, ObservabilityConfig, PolicyKind,
    ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig,
};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry;
use sgl_router::proxy::Proxy;
use sgl_router::server::app::build_router;
use sgl_router::server::app_context::AppContext;
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::WorkerRegistry;
use std::sync::Arc;
use std::time::Duration;
use tower::ServiceExt;

fn config(_worker_url: &str) -> Config {
    Config {
        server: ServerConfig {
            host: "0".into(),
            port: 0,
            ..Default::default()
        },
        observability: ObservabilityConfig::default(),
        model: ModelConfig {
            id: "tiny".into(),
            tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(),
            disable_input_ids_forwarding: false,
            policy: PolicyKind::RoundRobin,
            decode_policy: Default::default(),
            bucket_config: None,
            circuit_breaker: None,
            cache_aware: None,
            sticky: None,
            affinity: None,
            fused: None,
            eligibility: None,
            sampling_overrides: Default::default(),
        },
        discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
            urls: vec!["http://placeholder:0".into()],
        }),
        proxy: ProxyConfig::default(),
        router_inflight_load: InflightLoadConfig::default(),
    }
}

#[tokio::test]
async fn non_streaming_request_times_out_when_worker_hangs() {
    // Worker accepts and then sleeps for 5s; router timeout is 200ms.
    let worker =
        crate::common::mock_worker::MockWorker::start_hanging(Duration::from_secs(5)).await;
    let cfg = config(&worker.url);
    let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
    let registry = Arc::new(WorkerRegistry::default());
    let _ = registry.add(WorkerSpec {
        id: WorkerId("w1".into()),
        url: worker.url.clone(),
        mode: WorkerMode::Plain,
        model_ids: vec![ModelId("tiny".into())],
        bootstrap_port: None,
    });
    let policies = Arc::new(build_policy_registry(&cfg).unwrap());
    let proxy = Arc::new(Proxy::new(Duration::from_millis(200)).unwrap());
    let ctx = Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies));
    let app = build_router(ctx.clone());

    let req = Request::builder()
        .method("POST")
        .uri("/v1/chat/completions")
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_vec(&serde_json::json!({
                "model": "tiny",
                "messages": [{"role": "user", "content": "hi"}],
                "stream": false
            }))
            .unwrap(),
        ))
        .unwrap();

    let started = std::time::Instant::now();
    // Outer guard so a regression doesn't wedge CI forever.
    let res = tokio::time::timeout(Duration::from_secs(2), app.oneshot(req))
        .await
        .expect("router must return within 2s when proxy timeout is 200ms")
        .unwrap();
    let elapsed = started.elapsed();
    assert!(
        elapsed < Duration::from_secs(1),
        "router must short-circuit on upstream timeout; elapsed {elapsed:?}"
    );
    assert_eq!(res.status(), StatusCode::GATEWAY_TIMEOUT);
    assert_eq!(
        res.headers().get("x-router-error-code").unwrap(),
        "upstream_timeout"
    );
    let bytes = res.into_body().collect().await.unwrap().to_bytes();
    let body_str = String::from_utf8_lossy(&bytes);
    // A hung worker is the most common hard worker failure there is, so it must
    // land in `outcome="error"` — the series a per-worker error-ratio alert
    // watches. Deriving the outcome from the 504 status instead would silently
    // reclassify it as `cancelled` and blind that alert.
    assert!(
        ctx.metrics
            .render()
            .lines()
            .any(|l| l.starts_with("sgl_router_worker_requests_total{")
                && l.contains(r#"outcome="error""#)),
        "an upstream timeout must be counted outcome=error, not cancelled:\n{}",
        ctx.metrics.render(),
    );
    assert!(
        body_str.contains("\"code\":\"upstream_timeout\""),
        "body: {body_str}"
    );
    // No leak of worker URL or reqwest source chain to the client.
    assert!(
        !body_str.contains(&worker.url),
        "worker URL must not leak in client-visible body: {body_str}"
    );
}