[sgl-router] Move pressure and prefix signals out of legacy policies (#42428)

Co-authored-by: Kan Wu <kan.wu@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-10-05 18:10:29 +08:00
committed by GitHub
co-authored by Kan Wu Claude Opus 5.5
parent 26a1377c50
commit a96b146491
23 changed files with 471 additions and 437 deletions
+2 -5
View File
@@ -10,10 +10,7 @@ use sgl_router::{
KvIndexerEndpointConfig, LogFormat, PolicyKind,
},
discovery::{spawn_discovery, ModelId},
policies::{
factory::build_registry as build_policy_registry, prefix_provider::RadixTreePrefixProvider,
PolicyRegistry,
},
policies::{factory::build_registry as build_policy_registry, PolicyRegistry},
policies_reorg::factory::build_resolver as build_reorg_resolver,
proxy::Proxy,
server::{
@@ -22,7 +19,7 @@ use sgl_router::{
shutdown::drain_for_termination,
},
state::{
kv_events::{BlockSizeOracle, BootstrapTracker, KvEventIndex},
kv_events::{BlockSizeOracle, BootstrapTracker, KvEventIndex, RadixTreePrefixProvider},
load_monitor::router_inflight_load::{
spawn_janitor, JanitorHandle, RouterInflightLoadRegistry, SystemTimeClock,
},
+18 -290
View File
@@ -19,12 +19,14 @@
use crate::policies::power_of_two::select_k_with_snapshot;
use crate::policies::{CacheCandidate, CacheCandidateProposal, GuardHints, SelectionProposal};
pub(crate) use crate::state::load_monitor::engine_ranking::{
compare_decode_engines, compare_prefill_engines, CandidateLoads,
};
use crate::state::load_monitor::engine_reported_load::{
EngineReportedLoadSnapshot, EngineReportedSchedulingLoad, EngineReportedWorkerLoad,
EngineReportedLoadSnapshot, EngineReportedSchedulingLoad,
};
use crate::workers::Worker;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::sync::Arc;
/// A prefill candidate domain and its optional queue budget.
@@ -266,7 +268,7 @@ pub fn resolve_cache_candidates(
// candidate missing native monitor data would otherwise break the
// lookup's full-coverage check and silently downgrade every pressure
// comparison (and `prefill_pressure_source`) to router-local.
let loads = FreshLoadLookup::new(
let loads = CandidateLoads::new(
Some(snapshot),
evaluated.iter().copied().map(|candidate| &candidate.worker),
);
@@ -308,7 +310,7 @@ pub fn resolve_cache_candidates(
// comparison would fall back to router-local load — tie at
// zero for every owner, decided by worker id. The pin ranks
// the rejected owners, so it must see the rejected owners.
let pin_loads = FreshLoadLookup::new(
let pin_loads = CandidateLoads::new(
Some(snapshot),
queue_gate_rejected
.iter()
@@ -322,7 +324,7 @@ pub fn resolve_cache_candidates(
})
.min_by(|left, right| {
pin_loads
.compare_prefill_pressure(&left.worker, &right.worker)
.compare_prefill_engines(&left.worker, &right.worker)
.then_with(|| left.worker.id.0.cmp(&right.worker.id.0))
})
})
@@ -510,7 +512,7 @@ pub fn resolve_decode(
.is_some_and(|worker| is_decode_admitted(worker, request_kv_tokens, snapshot));
let (selected, reason) = match (primary_admitted, backup.as_ref(), backup_admitted) {
(true, Some(backup), true) => {
if compare_decode_pressure(&proposal.primary, backup, Some(snapshot)).is_gt() {
if compare_decode_engines(&proposal.primary, backup, Some(snapshot)).is_gt() {
(Arc::clone(backup), DecisionReason::BackupPressureGuard)
} else {
(Arc::clone(&proposal.primary), DecisionReason::Primary)
@@ -589,7 +591,7 @@ fn is_decode_admitted(
fn is_cache_candidate_admitted(
candidate: &CacheCandidate,
request_input_tokens: u64,
loads: &FreshLoadLookup<'_>,
loads: &CandidateLoads<'_>,
) -> bool {
let Some(load) = loads.get(&candidate.worker.id) else {
return true;
@@ -606,7 +608,7 @@ fn compare_cache_candidates(
left: &CacheCandidate,
right: &CacheCandidate,
proposal: &CacheCandidateProposal,
loads: &FreshLoadLookup<'_>,
loads: &CandidateLoads<'_>,
enable_pressure_guard: bool,
) -> Ordering {
let work_delta = left.uncached_tokens.abs_diff(right.uncached_tokens);
@@ -614,7 +616,7 @@ fn compare_cache_candidates(
return left
.uncached_tokens
.cmp(&right.uncached_tokens)
.then_with(|| loads.compare_prefill_pressure(&left.worker, &right.worker))
.then_with(|| loads.compare_prefill_engines(&left.worker, &right.worker))
.then_with(|| left.worker.id.0.cmp(&right.worker.id.0));
}
if enable_pressure_guard {
@@ -641,14 +643,14 @@ fn compare_cache_candidates(
}
left.uncached_tokens
.cmp(&right.uncached_tokens)
.then_with(|| loads.compare_prefill_pressure(&left.worker, &right.worker))
.then_with(|| loads.compare_prefill_engines(&left.worker, &right.worker))
.then_with(|| left.worker.id.0.cmp(&right.worker.id.0))
}
fn cache_pressure_guard_comparable(
left: &CacheCandidate,
right: &CacheCandidate,
loads: &FreshLoadLookup<'_>,
loads: &CandidateLoads<'_>,
) -> bool {
loads.comparable_get(&left.worker.id).is_some()
&& loads.comparable_get(&right.worker.id).is_some()
@@ -660,7 +662,7 @@ fn materially_more_pressured(
absolute_threshold_tokens: u64,
absolute_threshold_ms: Option<f64>,
relative_threshold: f64,
loads: &FreshLoadLookup<'_>,
loads: &CandidateLoads<'_>,
) -> bool {
let (Some(candidate_load), Some(other_load)) = (
loads.comparable_get(&candidate.id),
@@ -689,176 +691,6 @@ fn materially_more_pressured(
> other_load.num_waiting_uncached_tokens as f64 * relative_threshold
}
/// Constant-time request view over one captured load snapshot.
///
/// External values are compared only when every candidate is present. Mixed
/// candidate sets use Router-local active load to preserve ordering.
pub(crate) struct FreshLoadLookup<'a> {
by_worker_id: HashMap<String, &'a EngineReportedSchedulingLoad>,
basic_by_worker_id: HashMap<String, &'a EngineReportedWorkerLoad>,
local_active_by_worker_id: HashMap<String, usize>,
compare_engine: bool,
compare_basic_engine: bool,
}
impl<'a> FreshLoadLookup<'a> {
pub(crate) fn new<'w>(
snapshot: Option<&'a EngineReportedLoadSnapshot>,
workers: impl IntoIterator<Item = &'w Arc<Worker>>,
) -> Self {
let workers: Vec<&Arc<Worker>> = workers.into_iter().collect();
let local_active_by_worker_id: HashMap<String, usize> = workers
.iter()
.map(|worker| (worker.id.0.clone(), worker.router_inflight_load()))
.collect();
let by_worker_id = snapshot
.into_iter()
.flat_map(|snapshot| {
workers.iter().filter_map(move |worker| {
snapshot
.fresh_native_cache_load_for_url(&worker.url)
.map(|load| (worker.id.0.clone(), load))
})
})
.collect::<HashMap<_, _>>();
let basic_by_worker_id = snapshot
.into_iter()
.flat_map(|snapshot| {
workers.iter().filter_map(move |worker| {
snapshot
.fresh_load_for_url(&worker.url)
.map(|load| (worker.id.0.clone(), load))
})
})
.collect::<HashMap<_, _>>();
let compare_engine = !local_active_by_worker_id.is_empty()
&& by_worker_id.len() == local_active_by_worker_id.len();
let compare_basic_engine = !local_active_by_worker_id.is_empty()
&& basic_by_worker_id.len() == local_active_by_worker_id.len();
Self {
by_worker_id,
basic_by_worker_id,
local_active_by_worker_id,
compare_engine,
compare_basic_engine,
}
}
pub(crate) fn get(
&self,
worker_id: &crate::discovery::WorkerId,
) -> Option<&'a EngineReportedSchedulingLoad> {
self.by_worker_id.get(worker_id.0.as_str()).copied()
}
fn comparable_get(
&self,
worker_id: &crate::discovery::WorkerId,
) -> Option<&'a EngineReportedSchedulingLoad> {
self.compare_engine.then(|| self.get(worker_id)).flatten()
}
fn pressure_key(&self, worker: &Arc<Worker>) -> PressureKey<'a> {
PressureKey {
load: self.comparable_get(&worker.id),
local_active: self
.local_active_by_worker_id
.get(worker.id.0.as_str())
.copied()
.unwrap_or(usize::MAX),
}
}
fn compare_prefill_keys(&self, left: &PressureKey<'a>, right: &PressureKey<'a>) -> Ordering {
match (left.load, right.load) {
(Some(left_load), Some(right_load)) => compare_prefill_load(left_load, right_load)
.then_with(|| left.local_active.cmp(&right.local_active)),
_ => left.local_active.cmp(&right.local_active),
}
}
fn compare_decode_keys(&self, left: &PressureKey<'a>, right: &PressureKey<'a>) -> Ordering {
match (left.load, right.load) {
(Some(left_load), Some(right_load)) => compare_decode_load(left_load, right_load)
.then_with(|| left.local_active.cmp(&right.local_active)),
_ => left.local_active.cmp(&right.local_active),
}
}
pub(crate) fn compare_prefill_pressure(
&self,
left: &Arc<Worker>,
right: &Arc<Worker>,
) -> Ordering {
self.compare_prefill_keys(&self.pressure_key(left), &self.pressure_key(right))
}
pub(crate) fn prefill_pressure_source(&self) -> &'static str {
if self.compare_engine
&& self
.by_worker_id
.values()
.all(|load| load.estimated_prefill_queue_ms.is_some())
{
"estimated_prefill_queue_ms"
} else if self.compare_engine {
"native_queue_tokens"
} else {
"router_local"
}
}
/// Returns a queue depth consistent with admission for this request.
///
/// A fully covered candidate set uses `waiting + running`; otherwise the
/// whole set uses Router-local active load. Dispatches after the snapshot
/// are added to the reported value.
pub(crate) fn score_load(&self, worker: &Arc<Worker>) -> usize {
self.compare_basic_engine
.then(|| self.basic_by_worker_id.get(worker.id.0.as_str()).copied())
.flatten()
.map(|load| {
let recent_dispatches = worker
.slots_acquired_since(load.captured_at)
.try_into()
.unwrap_or(u64::MAX);
load.num_waiting_reqs
.saturating_add(load.num_running_reqs)
.saturating_add(recent_dispatches)
.try_into()
.unwrap_or(usize::MAX)
})
.unwrap_or_else(|| {
self.local_active_by_worker_id
.get(worker.id.0.as_str())
.copied()
.unwrap_or(usize::MAX)
})
}
fn min_by_pressure_key(
&self,
candidates: Vec<Arc<Worker>>,
compare: impl Fn(&Self, &PressureKey<'a>, &PressureKey<'a>) -> Ordering,
) -> Option<Arc<Worker>> {
let mut candidates = candidates.into_iter();
let mut best = candidates.next()?;
let mut best_key = self.pressure_key(&best);
for candidate in candidates {
let key = self.pressure_key(&candidate);
if compare(self, &key, &best_key).is_lt() {
best = candidate;
best_key = key;
}
}
Some(best)
}
}
struct PressureKey<'a> {
load: Option<&'a EngineReportedSchedulingLoad>,
local_active: usize,
}
fn range_fallback(
range: &CandidateRange<'_>,
legal: &[Arc<Worker>],
@@ -901,9 +733,9 @@ fn range_fallback(
// Scoped to the pool actually ranked: an admitted-but-gated worker
// missing native monitor data would otherwise downgrade the comparison
// for the whole unqueued tier to router-local.
let loads = FreshLoadLookup::new(Some(snapshot), pool.iter());
let loads = CandidateLoads::new(Some(snapshot), pool.iter());
loads
.min_by_pressure_key(pool, FreshLoadLookup::compare_prefill_keys)
.select_min_by_load_key(pool, CandidateLoads::compare_prefill_load_keys)
.map(|worker| (worker, DecisionReason::RangeFallback))
}
@@ -932,100 +764,12 @@ fn decode_domain_fallback(
.filter(|worker| is_decode_admitted(worker, request_kv_tokens, snapshot))
.cloned()
.collect::<Vec<_>>();
let loads = FreshLoadLookup::new(Some(snapshot), admitted.iter());
let loads = CandidateLoads::new(Some(snapshot), admitted.iter());
loads
.min_by_pressure_key(admitted, FreshLoadLookup::compare_decode_keys)
.select_min_by_load_key(admitted, CandidateLoads::compare_decode_load_keys)
.map(|worker| (worker, DecisionReason::RangeFallback))
}
/// Compares prefill pressure by queue time when available, then by the V3 load tuple.
pub(crate) fn compare_prefill_pressure(
left: &Arc<Worker>,
right: &Arc<Worker>,
snapshot: Option<&EngineReportedLoadSnapshot>,
) -> Ordering {
match snapshot.and_then(|snapshot| {
Some((
snapshot.fresh_native_cache_load_for_url(&left.url)?,
snapshot.fresh_native_cache_load_for_url(&right.url)?,
))
}) {
Some((left_load, right_load)) => {
compare_prefill_load(left_load, right_load).then_with(|| {
left.router_inflight_load()
.cmp(&right.router_inflight_load())
})
}
None => left
.router_inflight_load()
.cmp(&right.router_inflight_load()),
}
}
fn prefill_pressure_key(load: &EngineReportedSchedulingLoad) -> (u64, u64, u64) {
(
load.num_waiting_uncached_tokens,
load.num_waiting_reqs,
load.num_running_reqs,
)
}
fn compare_prefill_load(
left: &EngineReportedSchedulingLoad,
right: &EngineReportedSchedulingLoad,
) -> Ordering {
match (
left.estimated_prefill_queue_ms,
right.estimated_prefill_queue_ms,
) {
(Some(left_ms), Some(right_ms)) => left_ms
.total_cmp(&right_ms)
.then_with(|| prefill_pressure_key(left).cmp(&prefill_pressure_key(right))),
_ => prefill_pressure_key(left).cmp(&prefill_pressure_key(right)),
}
}
/// Compares decode pressure from LoadStat without treating unknown capacity as zero.
pub(crate) fn compare_decode_pressure(
left: &Arc<Worker>,
right: &Arc<Worker>,
snapshot: Option<&EngineReportedLoadSnapshot>,
) -> Ordering {
match snapshot.and_then(|snapshot| {
Some((
snapshot.fresh_native_cache_load_for_url(&left.url)?,
snapshot.fresh_native_cache_load_for_url(&right.url)?,
))
}) {
Some((left_load, right_load)) => {
compare_decode_load(left_load, right_load).then_with(|| {
left.router_inflight_load()
.cmp(&right.router_inflight_load())
})
}
None => left
.router_inflight_load()
.cmp(&right.router_inflight_load()),
}
}
fn compare_decode_load(
left: &EngineReportedSchedulingLoad,
right: &EngineReportedSchedulingLoad,
) -> Ordering {
let kv_usage = match (left.max_total_num_tokens, right.max_total_num_tokens) {
(left_cap, right_cap) if left_cap > 0 && right_cap > 0 => u128::from(left.num_used_tokens)
.saturating_mul(u128::from(right_cap))
.cmp(&u128::from(right.num_used_tokens).saturating_mul(u128::from(left_cap))),
_ => Ordering::Equal,
};
left.num_waiting_reqs
.cmp(&right.num_waiting_reqs)
.then_with(|| left.num_running_reqs.cmp(&right.num_running_reqs))
.then(kv_usage)
.then_with(|| left.num_used_tokens.cmp(&right.num_used_tokens))
}
fn pressure_guard_prefers_backup(
primary: &Arc<Worker>,
backup: &Arc<Worker>,
@@ -1199,22 +943,6 @@ mod tests {
assert_eq!(decision.load_snapshot_version, explicit.version);
}
#[test]
fn prefill_pressure_uses_waiting_then_running_requests() {
let busy = worker("busy");
let idle = worker("idle");
let loads = snapshot(&[(&busy, 1, 8, 10, 100), (&idle, 9, 2, 90, 100)]);
assert!(compare_prefill_pressure(&busy, &idle, Some(&loads)).is_gt());
}
#[test]
fn missing_snapshot_uses_local_active_load() {
let left = worker("left");
let right = worker("right");
let _guard = left.load_guard();
assert!(compare_prefill_pressure(&left, &right, None).is_gt());
}
#[test]
fn complete_monitor_pressure_guard_overrides_a_near_cache_gain() {
let congested = worker("congested");
@@ -4,7 +4,7 @@
//! Builds bounded cache-aware candidates from ingress Indexer results.
use crate::config::AffinityConfig;
use crate::policies::admission::FreshLoadLookup;
use crate::policies::admission::CandidateLoads;
use crate::policies::power_of_two::PowerOfTwoChoicesPolicy;
use crate::policies::{
CacheCandidate, CacheCandidateProposal, Policy, PrefillProposal, ProposalKind,
@@ -88,7 +88,7 @@ impl CacheAwarePolicy {
if limit == 0 {
return None;
}
let loads = FreshLoadLookup::new(
let loads = CandidateLoads::new(
ctx.load_snapshot(),
candidates.iter().map(|candidate| &candidate.worker),
);
@@ -139,12 +139,12 @@ impl CacheAwarePolicy {
fn compare_candidate_seed(
left: &CacheCandidate,
right: &CacheCandidate,
loads: &FreshLoadLookup<'_>,
loads: &CandidateLoads<'_>,
) -> Ordering {
right
.matched_prefix_tokens
.cmp(&left.matched_prefix_tokens)
.then_with(|| loads.compare_prefill_pressure(&left.worker, &right.worker))
.then_with(|| loads.compare_prefill_engines(&left.worker, &right.worker))
.then_with(|| left.worker.id.0.cmp(&right.worker.id.0))
}
@@ -5,7 +5,7 @@
use crate::config::DecodePolicyKind;
use crate::policies::admission::{
compare_decode_pressure, resolve_decode, CandidateDomain, DecisionReason, FinalDecision,
compare_decode_engines, resolve_decode, CandidateDomain, DecisionReason, FinalDecision,
RoutingStage,
};
use crate::policies::registry::select_decode_with_affinity;
@@ -126,7 +126,7 @@ impl DecodePolicy for DecodePowerOfTwoPolicy {
let left = &domain.workers[i];
let right = &domain.workers[j];
let (primary, backup) =
if compare_decode_pressure(left, right, ctx.load_snapshot()).is_gt() {
if compare_decode_engines(left, right, ctx.load_snapshot()).is_gt() {
(Arc::clone(right), Arc::clone(left))
} else {
(Arc::clone(left), Arc::clone(right))
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
use crate::policies::admission::FreshLoadLookup;
use crate::policies::admission::CandidateLoads;
use crate::policies::scoring::ScoringPolicy;
use crate::policies::SelectionContext;
use crate::workers::Worker;
@@ -34,7 +34,7 @@ impl ScoringPolicy for LoadBasedPolicy {
/// Purely a preference: "everybody is busy" is not a reason to refuse to
/// route, so this term never constrains. Capacity is `--filter`'s job.
fn scores(&self, workers: &[Arc<Worker>], ctx: &SelectionContext<'_>) -> Vec<f32> {
let lookup = FreshLoadLookup::new(ctx.load_snapshot(), workers.iter());
let lookup = CandidateLoads::new(ctx.load_snapshot(), workers.iter());
let loads: Vec<usize> = workers.iter().map(|w| lookup.score_load(w)).collect();
let lo = loads.iter().min().copied().unwrap_or(0);
let span = (loads.iter().max().copied().unwrap_or(0) - lo) as f32;
+12 -75
View File
@@ -9,7 +9,6 @@ pub mod dp_rank;
pub mod factory;
pub mod load_based;
pub mod power_of_two;
pub mod prefix_provider;
pub mod random;
pub mod registry;
pub mod round_robin;
@@ -22,6 +21,7 @@ use crate::discovery::ModelId;
use crate::policies::buckets::{BucketRequest, BucketSelector};
use crate::policies::scoring::{EligibilityFilter, ScoringPolicy};
use crate::server::metrics::MetricsRegistry;
pub(crate) use crate::state::kv_events::PrefixLookupResult;
use crate::state::load_monitor::engine_reported_load::EngineReportedLoadSnapshot;
use crate::tokenizer::{adapter, TokenizerRegistry};
use crate::workers::Worker;
@@ -37,16 +37,6 @@ pub struct RequestTokens {
pub rendered_from_chat: bool,
}
/// External indexer answer prepared by the async ingress path for the
/// synchronous cache-aware policy.
pub struct ExternalPrefixSignal {
pub outcome: sgl_kv_indexer::PrefixOutcome,
pub query_blocks: usize,
/// The query's block hashes when the local tree answered, so routing can
/// record the placement without rehashing.
pub block_hashes: Option<Arc<[i64]>>,
}
/// Whether the caller pre-tokenized the prompt (`input_ids` present and not
/// null). Such a request is never re-rendered: its ids drive routing and the
/// body is forwarded untouched, malformed values included, for the engine to
@@ -176,7 +166,7 @@ pub struct SelectionContext<'a> {
candidate_range_id: &'a str,
input_tokens: Option<u64>,
request_tokens: Option<&'a [u32]>,
external_prefix: Option<&'a ExternalPrefixSignal>,
external_prefix: Option<&'a PrefixLookupResult>,
load_snapshot: Option<&'a EngineReportedLoadSnapshot>,
prefill_cache_bucket: Option<(&'a BucketSelector, BucketRequest)>,
affinity_lookup_enabled: bool,
@@ -246,10 +236,7 @@ impl<'a> SelectionContext<'a> {
self
}
pub fn with_external_prefix(
mut self,
external_prefix: Option<&'a ExternalPrefixSignal>,
) -> Self {
pub fn with_external_prefix(mut self, external_prefix: Option<&'a PrefixLookupResult>) -> Self {
self.external_prefix = external_prefix;
self
}
@@ -312,7 +299,7 @@ impl<'a> SelectionContext<'a> {
self.request_tokens
}
pub fn external_prefix(&self) -> Option<&ExternalPrefixSignal> {
pub fn external_prefix(&self) -> Option<&PrefixLookupResult> {
self.external_prefix
}
@@ -597,7 +584,7 @@ mod tests {
use crate::config::{AffinityConfig, SessionAffinityMode};
use crate::discovery::{WorkerId, WorkerMode, WorkerSpec};
use crate::policies::admission::{
resolve_cache_candidates, resolve_prefill, CandidateRange, DecisionReason, FreshLoadLookup,
resolve_cache_candidates, resolve_prefill, CandidateRange, DecisionReason,
};
use crate::policies::cache_aware::CacheAwarePolicy;
use crate::policies::power_of_two::PowerOfTwoChoicesPolicy;
@@ -891,24 +878,13 @@ mod tests {
assert_eq!(returned.kind, ProposalKind::SessionAffinity);
}
#[test]
fn decode_pressure_tie_is_not_broken_by_worker_id() {
let a = worker("a");
let z = worker("z");
assert_eq!(
admission::compare_decode_pressure(&a, &z, None),
std::cmp::Ordering::Equal,
"P2 must preserve random sampling when observable pressure is equal"
);
}
#[test]
fn cache_affinity_uses_longest_routable_prefix_holder() {
let model = ModelId("model".into());
let hot = worker("hot");
let other = worker("other");
let workers = vec![Arc::clone(&hot), Arc::clone(&other)];
let signal = ExternalPrefixSignal {
let signal = PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches: vec![
sgl_kv_indexer::PrefixMatch {
@@ -952,7 +928,7 @@ mod tests {
let hot = worker("hot");
let warm = worker("warm");
let workers = vec![Arc::clone(&hot), Arc::clone(&warm)];
let signal = ExternalPrefixSignal {
let signal = PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches: vec![
sgl_kv_indexer::PrefixMatch {
@@ -1018,7 +994,7 @@ mod tests {
address: worker.url.clone(),
})
.collect();
let signal = ExternalPrefixSignal {
let signal = PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches,
best_prefix_blocks: workers.len() as u32,
@@ -1075,7 +1051,7 @@ mod tests {
address: worker.url.clone(),
})
.collect();
let signal = ExternalPrefixSignal {
let signal = PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches,
best_prefix_blocks: 4,
@@ -1116,7 +1092,7 @@ mod tests {
let half = worker("half");
let below_ratio = worker("below-ratio");
let workers = vec![Arc::clone(&half), Arc::clone(&below_ratio)];
let signal = ExternalPrefixSignal {
let signal = PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches: vec![
sgl_kv_indexer::PrefixMatch {
@@ -1163,7 +1139,7 @@ mod tests {
let model = ModelId("model".into());
let weak = worker("weak");
let workers = vec![Arc::clone(&weak)];
let signal = ExternalPrefixSignal {
let signal = PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches: vec![sgl_kv_indexer::PrefixMatch {
matched_prefix_blocks: 3,
@@ -1195,7 +1171,7 @@ mod tests {
let model = ModelId("model".into());
let holder = worker("holder");
let workers = vec![Arc::clone(&holder)];
let signal = ExternalPrefixSignal {
let signal = PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches: vec![sgl_kv_indexer::PrefixMatch {
matched_prefix_blocks: 2_048,
@@ -1269,45 +1245,6 @@ mod tests {
)
}
#[test]
fn mixed_freshness_uses_one_captured_local_level_for_the_candidate_set() {
let aggregate_idle = worker("aggregate-idle");
let aggregate_busy = worker("aggregate-busy");
let stale = worker("stale");
aggregate_idle
.active_requests
.store(5, std::sync::atomic::Ordering::Relaxed);
aggregate_busy
.active_requests
.store(1, std::sync::atomic::Ordering::Relaxed);
let snapshot = snapshot(&[
(
&aggregate_idle,
TestEngineLoad {
num_waiting_reqs: 0,
..TestEngineLoad::default()
},
),
(
&aggregate_busy,
TestEngineLoad {
num_waiting_reqs: 1_000,
..TestEngineLoad::default()
},
),
]);
let lookup =
FreshLoadLookup::new(Some(&snapshot), [&aggregate_idle, &aggregate_busy, &stale]);
assert!(lookup.get(&aggregate_idle.id).is_some());
assert!(lookup.get(&stale.id).is_none());
assert_eq!(
lookup.compare_prefill_pressure(&aggregate_idle, &aggregate_busy),
std::cmp::Ordering::Greater,
"one stale member makes the complete candidate set compare by the captured local level"
);
}
fn cache_candidate(
worker: &Arc<Worker>,
matched_prefix_tokens: u64,
@@ -14,7 +14,7 @@
//! never a sort.
use crate::config::DEFAULT_MIN_LOAD_CHOICES;
use crate::policies::admission::{compare_prefill_pressure, queue_gate_admits};
use crate::policies::admission::{compare_prefill_engines, queue_gate_admits};
use crate::policies::{Policy, ProposalKind, SelectionContext, SelectionProposal};
use crate::state::load_monitor::engine_reported_load::EngineReportedLoadSnapshot;
use crate::workers::Worker;
@@ -128,7 +128,7 @@ fn sample_pool<'w>(
/// two-worker fleet, so a fixed scan order would pin every fallback
/// dispatch to the first worker.
///
/// Both tiers scan linearly and never sort. `compare_prefill_pressure`
/// Both tiers scan linearly and never sort. `compare_prefill_engines`
/// is only a pairwise comparison: two workers that both publish
/// `estimated_prefill_queue_ms` are ordered on that estimate, and any
/// other pair on the waiting-token tuple, so it is not a total order
@@ -164,12 +164,12 @@ fn best_two_of_sample(
// Only a strict improvement displaces the incumbent, so a tie keeps
// whichever member the draw presented first.
if best.is_none_or(|current| {
compare_prefill_pressure(&pool[index], &pool[current], snapshot).is_lt()
compare_prefill_engines(&pool[index], &pool[current], snapshot).is_lt()
}) {
runner_up = best;
best = Some(index);
} else if runner_up.is_none_or(|current| {
compare_prefill_pressure(&pool[index], &pool[current], snapshot).is_lt()
compare_prefill_engines(&pool[index], &pool[current], snapshot).is_lt()
}) {
runner_up = Some(index);
}
@@ -231,7 +231,7 @@ mod tests {
/// A fleet where only some workers publish `estimated_prefill_queue_ms`.
/// An idle worker has no throughput delta to derive one from, so this is
/// the steady state, not an edge case - and it makes
/// `compare_prefill_pressure` intransitive: a slow worker with a shallow
/// `compare_prefill_engines` intransitive: a slow worker with a shallow
/// queue loses to a fast worker with a deep one on the estimate, while
/// both are ordered against an estimate-less worker on waiting tokens.
fn mixed_estimate_snapshot(workers: &[Arc<Worker>]) -> EngineReportedLoadSnapshot {
@@ -265,7 +265,7 @@ mod tests {
)
}
/// `compare_prefill_pressure` is a pairwise comparison, not a total
/// `compare_prefill_engines` is a pairwise comparison, not a total
/// order, so the k-way minimum must be a linear scan. Sorting a sample
/// this size panics with "user-provided comparison function does not
/// correctly implement a total order" - every request that samples a
@@ -36,7 +36,7 @@ use crate::policies::decode::{
build_decode_policy, resolve_decode_with_capacity_fallback, DecodeSelectionContext,
};
use crate::policies::{
ExternalPrefixSignal, Policy, PrefillProposal, ProposalKind, SelectionContext,
Policy, PrefillProposal, PrefixLookupResult, ProposalKind, SelectionContext,
};
use crate::server::metrics::{CacheAwareDecision, MetricsRegistry, PolicySelectionFailureReason};
use crate::state::load_monitor::engine_reported_load::EngineReportedLoadSnapshot;
@@ -59,7 +59,7 @@ pub(crate) struct PrefillSelectionInputs<'a> {
/// `request_input_tokens` only for a batch.
pub request_sequence_tokens: u64,
pub request_tokens: Option<&'a [u32]>,
pub external_prefix: Option<&'a ExternalPrefixSignal>,
pub external_prefix: Option<&'a PrefixLookupResult>,
/// Required whenever `policy.uses_shared_prefill_admission()`; the
/// per-domain rung panics without it. `Policy::needs_load_snapshot`
/// defaults to `uses_shared_prefill_admission`, which is what keeps the
@@ -707,7 +707,7 @@ mod tests {
use crate::policies::buckets::BucketSelector;
use crate::policies::cache_aware::CacheAwarePolicy;
use crate::policies::power_of_two::PowerOfTwoChoicesPolicy;
use crate::policies::{ExternalPrefixSignal, Policy, ProposalKind, SelectionProposal};
use crate::policies::{Policy, PrefixLookupResult, ProposalKind, SelectionProposal};
use crate::server::metrics::{
CacheAwareDecision, MetricsRegistry, PolicySelectionFailureReason,
};
@@ -785,8 +785,8 @@ mod tests {
}
/// An indexer hit placing `matched_prefix_blocks` on each named worker.
fn prefix_signal(matches: &[(&Arc<Worker>, u32)], query_blocks: usize) -> ExternalPrefixSignal {
ExternalPrefixSignal {
fn prefix_signal(matches: &[(&Arc<Worker>, u32)], query_blocks: usize) -> PrefixLookupResult {
PrefixLookupResult {
outcome: sgl_kv_indexer::PrefixOutcome::Matched {
matches: matches
.iter()
@@ -5,7 +5,7 @@
use crate::config::{AffinityConfig, SessionAffinityMode};
use crate::discovery::WorkerId;
use crate::policies::admission::compare_prefill_pressure;
use crate::policies::admission::compare_prefill_engines;
use crate::policies::power_of_two::PowerOfTwoChoicesPolicy;
use crate::policies::{GuardHints, Policy, ProposalKind, SelectionContext, SelectionProposal};
use crate::state::load_monitor::router_inflight_load::JanitorHandle;
@@ -201,7 +201,7 @@ fn sampled_backup_excluding(
};
let left = &workers[first];
let right = &workers[second];
if compare_prefill_pressure(left, right, ctx.load_snapshot()).is_gt() {
if compare_prefill_engines(left, right, ctx.load_snapshot()).is_gt() {
Some(Arc::clone(right))
} else {
Some(Arc::clone(left))
@@ -14,10 +14,11 @@ use sgl_kv_indexer::{PrefixIndex, PrefixIndexError, PrefixOutcome};
use tokio::sync::OnceCell;
use crate::config::{AffinityConfig, AffinityMode};
use crate::policies::admission::FreshLoadLookup;
use crate::policies::prefix_provider::RadixTreePrefixProvider;
use crate::policies::ExternalPrefixSignal;
use crate::state::kv_events::{compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle};
use crate::state::kv_events::{
compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, PrefixLookupResult,
RadixTreePrefixProvider,
};
use crate::state::load_monitor::engine_ranking::CandidateLoads;
use crate::state::load_monitor::engine_reported_load::{
EngineReportedLoadSnapshot, EngineReportedLoadTable,
};
@@ -28,7 +29,7 @@ use super::affinity;
use super::power_of_two::PowerOfTwoPolicy;
use super::{Pick, PickError, PickRequest, Policy, Rejection, Stage};
type Signal = Option<Arc<ExternalPrefixSignal>>;
type Signal = Option<Arc<PrefixLookupResult>>;
type Lookup = Arc<OnceCell<Signal>>;
/// Local radix tree or remote indexer. Groups sharing an index namespace share
@@ -74,7 +75,7 @@ impl CacheSource {
return Ok(None);
}
match index.match_prefix(hashes).await {
Ok(outcome) => Ok(Some(Arc::new(ExternalPrefixSignal {
Ok(outcome) => Ok(Some(Arc::new(PrefixLookupResult {
outcome,
query_blocks,
block_hashes: None,
@@ -105,7 +106,7 @@ impl fmt::Debug for PrefixMemo {
impl PrefixMemo {
/// The local tree's answer, once a policy has looked it up.
pub fn local_signal(&self) -> Option<Arc<ExternalPrefixSignal>> {
pub fn local_signal(&self) -> Option<Arc<PrefixLookupResult>> {
let cells = self.cells.lock().unwrap_or_else(|e| e.into_inner());
cells
.iter()
@@ -133,10 +134,10 @@ struct Candidate<'a> {
}
/// Less uncached work first, then lower prefill pressure, then worker id.
fn rank(loads: &FreshLoadLookup<'_>, left: &Candidate<'_>, right: &Candidate<'_>) -> Ordering {
fn rank(loads: &CandidateLoads<'_>, left: &Candidate<'_>, right: &Candidate<'_>) -> Ordering {
left.uncached_tokens
.cmp(&right.uncached_tokens)
.then_with(|| loads.compare_prefill_pressure(left.engine, right.engine))
.then_with(|| loads.compare_prefill_engines(left.engine, right.engine))
.then_with(|| left.engine.id.0.cmp(&right.engine.id.0))
}
@@ -185,10 +186,10 @@ impl CacheAwarePolicy {
&self,
engines: &'e [Arc<Worker>],
request: &PickRequest<'_>,
signal: Option<&ExternalPrefixSignal>,
signal: Option<&PrefixLookupResult>,
load: &EngineReportedLoadSnapshot,
) -> Vec<Candidate<'e>> {
let Some(ExternalPrefixSignal {
let Some(PrefixLookupResult {
outcome: PrefixOutcome::Matched { matches, .. },
query_blocks,
..
@@ -229,7 +230,7 @@ impl CacheAwarePolicy {
.len()
.min(config.cache_candidate_max_workers)
.min(config.cache_candidate_min_workers.max(proportional));
let loads = FreshLoadLookup::new(Some(load), candidates.iter().map(|c| c.engine));
let loads = CandidateLoads::new(Some(load), candidates.iter().map(|c| c.engine));
candidates.sort_by(|left, right| rank(&loads, left, right));
candidates.truncate(limit);
candidates
@@ -7,7 +7,7 @@ use anyhow::{bail, ensure, Result};
use crate::buckets_reorg::{Bucket, BucketGroups, BucketResolver, EngineGroup};
use crate::config::{DecodePolicyKind, FilterKind, ModelConfig, PolicyKind, SessionAffinityMode};
use crate::policies::prefix_provider::RadixTreePrefixProvider;
use crate::state::kv_events::RadixTreePrefixProvider;
use crate::state::{
kv_events::KvEventIndex, load_monitor::router_inflight_load::JanitorHandle, AffinityStore,
};
@@ -7,14 +7,14 @@ use std::time::Instant;
use futures::future::BoxFuture;
use rand::Rng;
use crate::policies::admission::{compare_decode_pressure, compare_prefill_pressure};
use crate::state::load_monitor::engine_ranking::{compare_decode_engines, compare_prefill_engines};
use crate::state::load_monitor::engine_reported_load::EngineReportedLoadTable;
use crate::workers::Worker;
use super::admission::{AdmissionLimits, Decision, EngineAdmission, EngineMetrics};
use super::{Pick, PickError, PickRequest, Policy, Rejection, Stage};
/// Samples two distinct engines and selects the one with lower stage pressure.
/// Samples two distinct engines and selects the better-ranked one for its stage.
/// Checks admission only on the selected engine; rejection never resamples.
#[derive(Debug)]
pub struct PowerOfTwoPolicy {
@@ -54,13 +54,13 @@ impl Policy for PowerOfTwoPolicy {
j += 1;
}
let (left, right) = (&engines[i], &engines[j]);
let pressure = match request.stage {
let ordering = match request.stage {
Stage::Plain | Stage::Prefill => {
compare_prefill_pressure(left, right, Some(&load))
compare_prefill_engines(left, right, Some(&load))
}
Stage::Decode => compare_decode_pressure(left, right, Some(&load)),
Stage::Decode => compare_decode_engines(left, right, Some(&load)),
};
Arc::clone(if pressure.is_gt() { right } else { left })
Arc::clone(if ordering.is_gt() { right } else { left })
}
};
let metrics = EngineMetrics::observe(&engine, &load);
@@ -6,12 +6,13 @@ use crate::config::Config;
use crate::discovery::ModelId;
use crate::policies::buckets::BucketSelector;
use crate::policies::prefix_provider::RadixTreePrefixProvider;
use crate::policies::PolicyRegistry;
use crate::proxy::Proxy;
use crate::server::inflight::InflightHttp;
use crate::server::metrics::MetricsRegistry;
use crate::state::kv_events::{BlockSizeOracle, KvEventIndex, KvIndexMetrics};
use crate::state::kv_events::{
BlockSizeOracle, KvEventIndex, KvIndexMetrics, RadixTreePrefixProvider,
};
use crate::state::load_monitor::engine_reported_load::EngineReportedLoadTable;
use crate::state::load_monitor::router_inflight_load::RouterInflightLoadRegistry;
use crate::tokenizer::TokenizerRegistry;
@@ -12,7 +12,7 @@ use crate::policies::registry::{PdPoolResolver, PdResolveError};
use crate::policies::selection::{
select_decode_peer, select_prefill_worker, DecodeSelectionInputs, PrefillSelectionInputs,
};
use crate::policies::{ExternalPrefixSignal, Policy};
use crate::policies::{Policy, PrefixLookupResult};
use crate::server::app_context::{AppContext, ChatRouting};
use crate::server::error::ApiError;
use crate::server::metrics::PolicySelectionFailureReason;
@@ -229,11 +229,7 @@ async fn select_workers(
/// Credit the chosen prefill with the prompt's prefix until KV events confirm
/// it. Called only once the whole selection succeeded, so a request that is
/// never dispatched credits nobody.
fn record_prefill_route(
ctx: &AppContext,
signal: Option<&ExternalPrefixSignal>,
prefill_url: &str,
) {
fn record_prefill_route(ctx: &AppContext, signal: Option<&PrefixLookupResult>, prefill_url: &str) {
if let (Some(provider), Some(signal)) = (&ctx.radix_tree_prefix_provider, signal) {
provider.record_route(signal, prefill_url);
}
@@ -287,7 +283,7 @@ fn capture_load_snapshot(
}
struct RoutingContext<'a> {
prefix_matches: Option<ExternalPrefixSignal>,
prefix_matches: Option<PrefixLookupResult>,
load_snapshot: Option<EngineReportedLoadSnapshot>,
ttft_slo_ms: Option<u64>,
tps_slo: Option<f64>,
@@ -414,7 +410,7 @@ fn pick_decode_worker(
async fn lookup_prefix_matches(
ctx: &AppContext,
request: &PreparedRequest,
) -> Result<Option<ExternalPrefixSignal>, ApiError> {
) -> Result<Option<PrefixLookupResult>, ApiError> {
let signal = match (
ctx.prefix_index.as_ref(),
request.tokens.as_ref(),
@@ -433,7 +429,7 @@ async fn lookup_prefix_matches(
} else {
resolve_prefix_query(index.match_prefix(hashes).await, &request.model.0)?
};
Some(ExternalPrefixSignal {
Some(PrefixLookupResult {
outcome,
query_blocks,
block_hashes: None,
@@ -26,6 +26,7 @@ pub mod discovery;
pub mod hash;
pub mod index;
pub mod pending;
pub mod prefix_provider;
pub mod subscriber;
pub mod tally;
pub mod tree;
@@ -38,6 +39,7 @@ pub use discovery::{fetch_event_config, EventConfig};
pub use hash::{compute_block_hashes, compute_block_hashes_bigram, sha256_to_i64};
pub use index::{KvEventIndex, KvIndexMetrics};
pub use pending::PendingPrefixes;
pub use prefix_provider::{PrefixLookupResult, RadixTreePrefixProvider};
pub use subscriber::{KvEventSubscriberRegistry, SubKind, WorkerEvent};
pub use tally::{EventKind, EventTally, ReplayOutcome, TallyRow};
pub use tree::{
@@ -1,14 +1,21 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
use super::ExternalPrefixSignal;
use crate::state::kv_events::{
compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, HashTree,
};
use super::{compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, HashTree};
use sgl_kv_indexer::{PrefixMatch, PrefixOutcome};
use std::collections::BTreeMap;
use std::sync::Arc;
/// Prefix lookup result from the local radix tree or remote KV indexer,
/// consumed by cache-aware routing.
pub struct PrefixLookupResult {
pub outcome: PrefixOutcome,
pub query_blocks: usize,
/// The query's block hashes when the local tree answered, so routing can
/// record the placement without rehashing.
pub block_hashes: Option<Arc<[i64]>>,
}
#[derive(Clone, Debug)]
pub struct RadixTreePrefixProvider {
tree: Arc<HashTree>,
@@ -23,7 +30,7 @@ impl RadixTreePrefixProvider {
}
}
pub fn match_request_tokens(&self, tokens: &[u32]) -> Option<ExternalPrefixSignal> {
pub fn match_request_tokens(&self, tokens: &[u32]) -> Option<PrefixLookupResult> {
let hashes = self.block_hashes(tokens)?;
let mut depth_by_url = BTreeMap::<String, u32>::new();
@@ -55,7 +62,7 @@ impl RadixTreePrefixProvider {
}
};
// Empty is still returned so routing can record where this prompt went.
Some(ExternalPrefixSignal {
Some(PrefixLookupResult {
outcome,
query_blocks: hashes.len(),
block_hashes: self.tree.pending().is_enabled().then(|| hashes.into()),
@@ -63,7 +70,7 @@ impl RadixTreePrefixProvider {
}
/// Remember that `signal`'s prompt was just routed to `url`.
pub fn record_route(&self, signal: &ExternalPrefixSignal, url: &str) {
pub fn record_route(&self, signal: &PrefixLookupResult, url: &str) {
if let Some(hashes) = &signal.block_hashes {
self.tree.pending().record(url, hashes);
}
@@ -0,0 +1,362 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
//! Stage-specific engine ranking over one captured load snapshot.
use super::engine_reported_load::{
EngineReportedLoadSnapshot, EngineReportedSchedulingLoad, EngineReportedWorkerLoad,
};
use crate::workers::Worker;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::sync::Arc;
/// Ranks prefill engines by queue time when available, then by the V3 load tuple.
pub(crate) fn compare_prefill_engines(
left: &Arc<Worker>,
right: &Arc<Worker>,
snapshot: Option<&EngineReportedLoadSnapshot>,
) -> Ordering {
match snapshot.and_then(|snapshot| {
Some((
snapshot.fresh_native_cache_load_for_url(&left.url)?,
snapshot.fresh_native_cache_load_for_url(&right.url)?,
))
}) {
Some((left_load, right_load)) => {
compare_prefill_load(left_load, right_load).then_with(|| {
left.router_inflight_load()
.cmp(&right.router_inflight_load())
})
}
None => left
.router_inflight_load()
.cmp(&right.router_inflight_load()),
}
}
fn prefill_load_key(load: &EngineReportedSchedulingLoad) -> (u64, u64, u64) {
(
load.num_waiting_uncached_tokens,
load.num_waiting_reqs,
load.num_running_reqs,
)
}
fn compare_prefill_load(
left: &EngineReportedSchedulingLoad,
right: &EngineReportedSchedulingLoad,
) -> Ordering {
match (
left.estimated_prefill_queue_ms,
right.estimated_prefill_queue_ms,
) {
(Some(left_ms), Some(right_ms)) => left_ms
.total_cmp(&right_ms)
.then_with(|| prefill_load_key(left).cmp(&prefill_load_key(right))),
_ => prefill_load_key(left).cmp(&prefill_load_key(right)),
}
}
/// Ranks decode engines by LoadStat without treating unknown capacity as zero.
pub(crate) fn compare_decode_engines(
left: &Arc<Worker>,
right: &Arc<Worker>,
snapshot: Option<&EngineReportedLoadSnapshot>,
) -> Ordering {
match snapshot.and_then(|snapshot| {
Some((
snapshot.fresh_native_cache_load_for_url(&left.url)?,
snapshot.fresh_native_cache_load_for_url(&right.url)?,
))
}) {
Some((left_load, right_load)) => {
compare_decode_load(left_load, right_load).then_with(|| {
left.router_inflight_load()
.cmp(&right.router_inflight_load())
})
}
None => left
.router_inflight_load()
.cmp(&right.router_inflight_load()),
}
}
fn compare_decode_load(
left: &EngineReportedSchedulingLoad,
right: &EngineReportedSchedulingLoad,
) -> Ordering {
let kv_usage = match (left.max_total_num_tokens, right.max_total_num_tokens) {
(left_cap, right_cap) if left_cap > 0 && right_cap > 0 => u128::from(left.num_used_tokens)
.saturating_mul(u128::from(right_cap))
.cmp(&u128::from(right.num_used_tokens).saturating_mul(u128::from(left_cap))),
_ => Ordering::Equal,
};
left.num_waiting_reqs
.cmp(&right.num_waiting_reqs)
.then_with(|| left.num_running_reqs.cmp(&right.num_running_reqs))
.then(kv_usage)
.then_with(|| left.num_used_tokens.cmp(&right.num_used_tokens))
}
/// Constant-time request view over one captured load snapshot.
///
/// External values are compared only when every candidate is present. Mixed
/// candidate sets use Router-local active load to preserve ordering.
///
/// The key-level helpers (`comparable_get`, `compare_*_keys`,
/// `select_min_by_load_key`) are `pub(crate)` only for legacy admission.
pub(crate) struct CandidateLoads<'a> {
by_worker_id: HashMap<String, &'a EngineReportedSchedulingLoad>,
basic_by_worker_id: HashMap<String, &'a EngineReportedWorkerLoad>,
local_active_by_worker_id: HashMap<String, usize>,
compare_engine: bool,
compare_basic_engine: bool,
}
impl<'a> CandidateLoads<'a> {
pub(crate) fn new<'w>(
snapshot: Option<&'a EngineReportedLoadSnapshot>,
workers: impl IntoIterator<Item = &'w Arc<Worker>>,
) -> Self {
let workers: Vec<&Arc<Worker>> = workers.into_iter().collect();
let local_active_by_worker_id: HashMap<String, usize> = workers
.iter()
.map(|worker| (worker.id.0.clone(), worker.router_inflight_load()))
.collect();
let by_worker_id = snapshot
.into_iter()
.flat_map(|snapshot| {
workers.iter().filter_map(move |worker| {
snapshot
.fresh_native_cache_load_for_url(&worker.url)
.map(|load| (worker.id.0.clone(), load))
})
})
.collect::<HashMap<_, _>>();
let basic_by_worker_id = snapshot
.into_iter()
.flat_map(|snapshot| {
workers.iter().filter_map(move |worker| {
snapshot
.fresh_load_for_url(&worker.url)
.map(|load| (worker.id.0.clone(), load))
})
})
.collect::<HashMap<_, _>>();
let compare_engine = !local_active_by_worker_id.is_empty()
&& by_worker_id.len() == local_active_by_worker_id.len();
let compare_basic_engine = !local_active_by_worker_id.is_empty()
&& basic_by_worker_id.len() == local_active_by_worker_id.len();
Self {
by_worker_id,
basic_by_worker_id,
local_active_by_worker_id,
compare_engine,
compare_basic_engine,
}
}
pub(crate) fn get(
&self,
worker_id: &crate::discovery::WorkerId,
) -> Option<&'a EngineReportedSchedulingLoad> {
self.by_worker_id.get(worker_id.0.as_str()).copied()
}
pub(crate) fn comparable_get(
&self,
worker_id: &crate::discovery::WorkerId,
) -> Option<&'a EngineReportedSchedulingLoad> {
self.compare_engine.then(|| self.get(worker_id)).flatten()
}
fn load_key(&self, worker: &Arc<Worker>) -> LoadKey<'a> {
LoadKey {
load: self.comparable_get(&worker.id),
local_active: self
.local_active_by_worker_id
.get(worker.id.0.as_str())
.copied()
.unwrap_or(usize::MAX),
}
}
pub(crate) fn compare_prefill_load_keys(
&self,
left: &LoadKey<'a>,
right: &LoadKey<'a>,
) -> Ordering {
match (left.load, right.load) {
(Some(left_load), Some(right_load)) => compare_prefill_load(left_load, right_load)
.then_with(|| left.local_active.cmp(&right.local_active)),
_ => left.local_active.cmp(&right.local_active),
}
}
pub(crate) fn compare_decode_load_keys(
&self,
left: &LoadKey<'a>,
right: &LoadKey<'a>,
) -> Ordering {
match (left.load, right.load) {
(Some(left_load), Some(right_load)) => compare_decode_load(left_load, right_load)
.then_with(|| left.local_active.cmp(&right.local_active)),
_ => left.local_active.cmp(&right.local_active),
}
}
pub(crate) fn compare_prefill_engines(
&self,
left: &Arc<Worker>,
right: &Arc<Worker>,
) -> Ordering {
self.compare_prefill_load_keys(&self.load_key(left), &self.load_key(right))
}
pub(crate) fn prefill_pressure_source(&self) -> &'static str {
if self.compare_engine
&& self
.by_worker_id
.values()
.all(|load| load.estimated_prefill_queue_ms.is_some())
{
"estimated_prefill_queue_ms"
} else if self.compare_engine {
"native_queue_tokens"
} else {
"router_local"
}
}
/// Returns a queue depth consistent with admission for this request.
///
/// A fully covered candidate set uses `waiting + running`; otherwise the
/// whole set uses Router-local active load. Dispatches after the snapshot
/// are added to the reported value.
pub(crate) fn score_load(&self, worker: &Arc<Worker>) -> usize {
self.compare_basic_engine
.then(|| self.basic_by_worker_id.get(worker.id.0.as_str()).copied())
.flatten()
.map(|load| {
let recent_dispatches = worker
.slots_acquired_since(load.captured_at)
.try_into()
.unwrap_or(u64::MAX);
load.num_waiting_reqs
.saturating_add(load.num_running_reqs)
.saturating_add(recent_dispatches)
.try_into()
.unwrap_or(usize::MAX)
})
.unwrap_or_else(|| {
self.local_active_by_worker_id
.get(worker.id.0.as_str())
.copied()
.unwrap_or(usize::MAX)
})
}
pub(crate) fn select_min_by_load_key(
&self,
candidates: Vec<Arc<Worker>>,
compare: impl Fn(&Self, &LoadKey<'a>, &LoadKey<'a>) -> Ordering,
) -> Option<Arc<Worker>> {
let mut candidates = candidates.into_iter();
let mut best = candidates.next()?;
let mut best_key = self.load_key(&best);
for candidate in candidates {
let key = self.load_key(&candidate);
if compare(self, &key, &best_key).is_lt() {
best = candidate;
best_key = key;
}
}
Some(best)
}
}
pub(crate) struct LoadKey<'a> {
load: Option<&'a EngineReportedSchedulingLoad>,
local_active: usize,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use std::time::Instant;
fn worker(id: &str) -> Arc<Worker> {
Arc::new(Worker::new(WorkerSpec {
id: WorkerId(id.into()),
url: format!("http://{id}:30000"),
mode: WorkerMode::Plain,
model_ids: vec![ModelId("model".into())],
..Default::default()
}))
}
/// `(worker, running, waiting)`; waiting doubles as uncached tokens.
fn snapshot(entries: &[(&Arc<Worker>, u64, u64)]) -> EngineReportedLoadSnapshot {
EngineReportedLoadSnapshot::from_native_cache_workers(
7,
entries
.iter()
.map(|(worker, running, waiting)| {
(
worker.url.clone(),
EngineReportedSchedulingLoad {
num_running_reqs: *running,
num_waiting_reqs: *waiting,
num_waiting_uncached_tokens: *waiting,
num_used_tokens: 0,
num_total_tokens: 0,
max_total_num_tokens: 100,
max_running_requests: 64,
prefill_throughput_tokens_per_s: None,
estimated_prefill_queue_ms: None,
captured_at: Instant::now(),
},
)
})
.collect(),
)
}
#[test]
fn prefill_ranking_uses_waiting_then_running_requests() {
let busy = worker("busy");
let idle = worker("idle");
let loads = snapshot(&[(&busy, 1, 8), (&idle, 9, 2)]);
assert!(compare_prefill_engines(&busy, &idle, Some(&loads)).is_gt());
}
#[test]
fn missing_snapshot_uses_local_active_load() {
let left = worker("left");
let right = worker("right");
let _guard = left.load_guard();
assert!(compare_prefill_engines(&left, &right, None).is_gt());
}
#[test]
fn decode_ranking_tie_is_not_broken_by_worker_id() {
let (a, z) = (worker("a"), worker("z"));
assert_eq!(compare_decode_engines(&a, &z, None), Ordering::Equal);
}
#[test]
fn one_stale_candidate_makes_the_set_compare_by_local_load() {
let idle = worker("idle");
let busy = worker("busy");
let stale = worker("stale");
let _idle_guards: Vec<_> = (0..5).map(|_| idle.load_guard()).collect();
let _busy_guard = busy.load_guard();
let snapshot = snapshot(&[(&idle, 0, 0), (&busy, 0, 1_000)]);
let loads = CandidateLoads::new(Some(&snapshot), [&idle, &busy, &stale]);
assert!(loads.get(&idle.id).is_some());
assert!(loads.get(&stale.id).is_none());
assert!(loads.compare_prefill_engines(&idle, &busy).is_gt());
}
}
@@ -3,5 +3,6 @@
//! Load signals: engine-reported load and router-local in-flight accounting.
pub mod engine_ranking;
pub mod engine_reported_load;
pub mod router_inflight_load;
@@ -5,8 +5,9 @@ use std::collections::HashMap;
use std::sync::Arc;
use sgl_kv_indexer::PrefixOutcome;
use sgl_router::policies::prefix_provider::RadixTreePrefixProvider;
use sgl_router::state::kv_events::{compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId};
use sgl_router::state::kv_events::{
compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId, RadixTreePrefixProvider,
};
#[test]
fn radix_tree_reports_contiguous_prefix_depth_per_worker() {
@@ -9,13 +9,13 @@ use sgl_kv_indexer::{PrefixIndex, PrefixIndexError, PrefixMatch, PrefixOutcome};
use sgl_router::buckets_reorg::{Bucket, BucketGroups, BucketResolver, EngineGroup};
use sgl_router::config::{AffinityConfig, AffinityMode};
use sgl_router::discovery::{ModelId, WorkerId, WorkerSpec};
use sgl_router::policies::prefix_provider::RadixTreePrefixProvider;
use sgl_router::policies_reorg::admission::{Decision, EngineAdmission, EngineMetrics};
use sgl_router::policies_reorg::cache_aware::{CacheAwarePolicy, CacheSource, PrefixMemo};
use sgl_router::policies_reorg::power_of_two::PowerOfTwoPolicy;
use sgl_router::policies_reorg::{PickError, PickRequest, Policy, Stage};
use sgl_router::state::kv_events::{
compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, HashTree, KvWorkerId,
RadixTreePrefixProvider,
};
use sgl_router::state::load_monitor::engine_reported_load::{
EngineReportedLoadTable, LoadStat, NativeCacheRankLoad,
@@ -633,10 +633,9 @@ async fn invalid_policy_signal_stops_bucket_iteration() {
#[tokio::test]
async fn cache_aware_routes_tokenized_prompt_and_rechecks_the_next_bucket() {
use sgl_router::config::AffinityConfig;
use sgl_router::policies::prefix_provider::RadixTreePrefixProvider;
use sgl_router::policies_reorg::cache_aware::{CacheAwarePolicy, CacheSource};
use sgl_router::state::kv_events::{
compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId,
compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId, RadixTreePrefixProvider,
};
use sgl_router::state::load_monitor::engine_reported_load::EngineReportedLoadTable;
@@ -16,13 +16,14 @@ use sgl_router::config::{
};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry;
use sgl_router::policies::prefix_provider::RadixTreePrefixProvider;
use sgl_router::policies::PolicyRegistry;
use sgl_router::policies_reorg::factory::build_resolver;
use sgl_router::proxy::Proxy;
use sgl_router::server::app::build_router;
use sgl_router::server::app_context::{AppContext, ChatRouting};
use sgl_router::state::kv_events::{BlockSizeOracle, HashTree, KvEventIndex};
use sgl_router::state::kv_events::{
BlockSizeOracle, HashTree, KvEventIndex, RadixTreePrefixProvider,
};
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::WorkerRegistry;
@@ -12,12 +12,13 @@ use serde_json::json;
use sgl_router::config::{Config, PolicyKind, StickyConfig, StickyFallbackKind};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry;
use sgl_router::policies::prefix_provider::RadixTreePrefixProvider;
use sgl_router::policies::request_tokens_for;
use sgl_router::proxy::Proxy;
use sgl_router::server::app::build_router;
use sgl_router::server::app_context::AppContext;
use sgl_router::state::kv_events::{compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId};
use sgl_router::state::kv_events::{
compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId, RadixTreePrefixProvider,
};
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::{EngineProfile, WireProtocol, WorkerRegistry};
use tower::ServiceExt;