minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Router] Add composable scoring and eligibility policies (#37731)

Browse Source 
Co-authored-by: inkcherry <mingzhi.liu@amd.com>
This commit is contained in:
Vincent Gao
2026-09-04 00:01:41 +08:00
committed by GitHub
co-authored by inkcherry
parent 392841f47c
commit 54cadad151
36 changed files with 2256 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experimental/sgl-router/tests/proxy/cache_aware_input_ids.rs
+35 -1
View File
@@ -12,10 +12,18 @@
//! doesn't render tool schemas, so its ids would diverge from the engine).
//! * A request with multimodal (array) content → `input_ids` omitted (a text
//! tokenizer can't represent image content).
//!
//! The model id contains `deepseek-v4` so the tokenizer registry auto-attaches
//! the built-in V4 chat encoder — the engine-equivalent path — without a
//! template fixture.
use axum::body::Body;
use axum::http::{Request, StatusCode};
use serde_json::{json, Value};
use sgl_router::config::{
ActiveLoadConfig, CacheAwareConfig, Config, DiscoveryBackend, ModelConfig, ObservabilityConfig,
PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig,
};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry;
use sgl_router::policies::kv_events::{BlockSizeOracle, HashTree};
@@ -28,9 +36,35 @@ use std::sync::Arc;
use std::time::Duration;
use tower::ServiceExt;
use crate::common::cache_aware_fixture::{config, MODEL};
use crate::common::mock_worker::MockWorker;
const MODEL: &str = "deepseek-v4-tiny";
fn config() -> Config {
Config {
server: ServerConfig {
host: "0".into(),
port: 0,
},
observability: ObservabilityConfig::default(),
model: ModelConfig {
id: MODEL.into(),
tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(),
policy: PolicyKind::CacheAwareZmq,
circuit_breaker: None,
cache_aware: Some(CacheAwareConfig::default()),
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
}),
proxy: ProxyConfig::default(),
active_load: ActiveLoadConfig::default(),
}
}
fn build_ctx(url: String) -> Arc<AppContext> {
let cfg = config();
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
experimental/sgl-router/tests/proxy/chat_routing.rs
+2
View File
@@ -37,6 +37,8 @@ fn config_for(_worker_url: &str) -> Config {
circuit_breaker: None,
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs
+2
View File
@@ -29,6 +29,8 @@ pub fn config() -> Config {
circuit_breaker: None,
cache_aware: Some(CacheAwareConfig::default()),
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/failover.rs
+2
View File
@@ -41,6 +41,8 @@ async fn failover_when_one_worker_dies() {
}),
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec![w1.url.clone(), w2.url.clone(), w3.url.clone()],
experimental/sgl-router/tests/proxy/graceful_shutdown.rs
+2
View File
@@ -47,6 +47,8 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc<AppContext> {
circuit_breaker: None,
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/header_forwarding.rs
+2
View File
@@ -34,6 +34,8 @@ async fn forwards_whitelisted_headers_strips_others() {
circuit_breaker: None,
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs
+2
View File
@@ -49,6 +49,8 @@ fn config() -> Config {
circuit_breaker: None,
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/pd_pool_isolation.rs
+2
View File
@@ -48,6 +48,8 @@ fn config() -> Config {
circuit_breaker: None,
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs
+2
View File
@@ -45,6 +45,8 @@ fn config() -> Config {
circuit_breaker: None,
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/sticky_input_ids.rs
+4 -2
View File
@@ -25,7 +25,7 @@ use axum::http::{Request, StatusCode};
use serde_json::{json, Value};
use sgl_router::config::{
ActiveLoadConfig, Config, DiscoveryBackend, ModelConfig, ObservabilityConfig, PolicyKind,
ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, StickyConfig,
ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind,
};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry;
@@ -60,10 +60,12 @@ fn config() -> Config {
// mid-test; round-robin fallback for the initial pin of a key.
sticky: Some(StickyConfig {
header_name: HEADER.to_string(),
fallback_policy: PolicyKind::RoundRobin,
fallback_policy: StickyFallbackKind::RoundRobin,
idle_secs: 3600,
eviction_interval_secs: 3600,
}),
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/sticky_routing.rs
+4 -2
View File
@@ -8,7 +8,7 @@
use sgl_router::config::{
ActiveLoadConfig, Config, DiscoveryBackend, ModelConfig, ObservabilityConfig, PolicyKind,
ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, StickyConfig,
ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind,
};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry;
@@ -46,10 +46,12 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc<AppContext
cache_aware: None,
sticky: Some(StickyConfig {
header_name: header_name.to_string(),
fallback_policy: PolicyKind::RoundRobin,
fallback_policy: StickyFallbackKind::RoundRobin,
idle_secs: 3600,
eviction_interval_secs: 3600,
}),
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
experimental/sgl-router/tests/proxy/timeout.rs
+2
View File
@@ -41,6 +41,8 @@ fn config(_worker_url: &str) -> Config {
circuit_breaker: None,
cache_aware: None,
sticky: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],