[Router] Add load-aware prefill admission and bounded policy proposals (#37843)

Signed-off-by: Vincent Gao <vincentbo@linux.alibaba.com>
Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Vincent Gao
2026-09-05 11:53:26 +08:00
committed by GitHub
co-authored by Kangyan Zhou Claude Opus 4.8 Shangming Cai
parent d50e9a9756
commit ecd97de1fc
55 changed files with 6834 additions and 348 deletions
@@ -25,6 +25,7 @@ use sgl_router::config::{
PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig,
};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::engine_load::EngineLoadTable;
use sgl_router::policies::factory::build_registry;
use sgl_router::policies::kv_events::{BlockSizeOracle, HashTree};
use sgl_router::proxy::Proxy;
@@ -54,6 +55,7 @@ fn config() -> Config {
circuit_breaker: None,
cache_aware: Some(CacheAwareConfig::default()),
sticky: None,
affinity: None,
fused: None,
eligibility: None,
},
@@ -88,6 +90,7 @@ fn build_ctx(url: String) -> Arc<AppContext> {
Arc::new(HashTree::new()),
Arc::clone(&tokenizers),
BlockSizeOracle::new(),
EngineLoadTable::new(),
)
.unwrap(),
);