From a96b1464917805fa5edb167242f5be32eab8ea57 Mon Sep 17 00:00:00 2001 From: Kan Wu Date: Mon, 5 Oct 2026 03:10:29 -0700 Subject: [PATCH] [sgl-router] Move pressure and prefix signals out of legacy policies (#42428) Co-authored-by: Kan Wu Co-authored-by: Claude Opus 5.5 --- experimental/sgl-router/src/main.rs | 7 +- .../sgl-router/src/policies/admission.rs | 308 +-------------- .../sgl-router/src/policies/cache_aware.rs | 8 +- .../sgl-router/src/policies/decode.rs | 4 +- .../sgl-router/src/policies/load_based.rs | 4 +- experimental/sgl-router/src/policies/mod.rs | 87 +---- .../sgl-router/src/policies/power_of_two.rs | 12 +- .../sgl-router/src/policies/selection.rs | 10 +- .../sgl-router/src/policies/session_aware.rs | 4 +- .../src/policies_reorg/cache_aware.rs | 25 +- .../sgl-router/src/policies_reorg/factory.rs | 2 +- .../src/policies_reorg/power_of_two.rs | 12 +- .../sgl-router/src/server/app_context.rs | 5 +- .../sgl-router/src/server/routes/chat.rs | 14 +- .../sgl-router/src/state/kv_events/mod.rs | 2 + .../kv_events}/prefix_provider.rs | 21 +- .../src/state/load_monitor/engine_ranking.rs | 362 ++++++++++++++++++ .../sgl-router/src/state/load_monitor/mod.rs | 1 + .../policies/cache_prefix_provider.rs | 5 +- .../component/policies_reorg_cache_aware.rs | 2 +- .../tests/proxy/chat_routing/reorg.rs | 3 +- .../tests/proxy/common/cache_aware_fixture.rs | 5 +- .../sgl-router/tests/proxy/dp_rank_routing.rs | 5 +- 23 files changed, 471 insertions(+), 437 deletions(-) rename experimental/sgl-router/src/{policies => state/kv_events}/prefix_provider.rs (85%) create mode 100644 experimental/sgl-router/src/state/load_monitor/engine_ranking.rs diff --git a/experimental/sgl-router/src/main.rs b/experimental/sgl-router/src/main.rs index 92970259e5..2687a8e39a 100644 --- a/experimental/sgl-router/src/main.rs +++ b/experimental/sgl-router/src/main.rs @@ -10,10 +10,7 @@ use sgl_router::{ KvIndexerEndpointConfig, LogFormat, PolicyKind, }, discovery::{spawn_discovery, ModelId}, - policies::{ - factory::build_registry as build_policy_registry, prefix_provider::RadixTreePrefixProvider, - PolicyRegistry, - }, + policies::{factory::build_registry as build_policy_registry, PolicyRegistry}, policies_reorg::factory::build_resolver as build_reorg_resolver, proxy::Proxy, server::{ @@ -22,7 +19,7 @@ use sgl_router::{ shutdown::drain_for_termination, }, state::{ - kv_events::{BlockSizeOracle, BootstrapTracker, KvEventIndex}, + kv_events::{BlockSizeOracle, BootstrapTracker, KvEventIndex, RadixTreePrefixProvider}, load_monitor::router_inflight_load::{ spawn_janitor, JanitorHandle, RouterInflightLoadRegistry, SystemTimeClock, }, diff --git a/experimental/sgl-router/src/policies/admission.rs b/experimental/sgl-router/src/policies/admission.rs index 1952a3db21..c4eb2a2be4 100644 --- a/experimental/sgl-router/src/policies/admission.rs +++ b/experimental/sgl-router/src/policies/admission.rs @@ -19,12 +19,14 @@ use crate::policies::power_of_two::select_k_with_snapshot; use crate::policies::{CacheCandidate, CacheCandidateProposal, GuardHints, SelectionProposal}; +pub(crate) use crate::state::load_monitor::engine_ranking::{ + compare_decode_engines, compare_prefill_engines, CandidateLoads, +}; use crate::state::load_monitor::engine_reported_load::{ - EngineReportedLoadSnapshot, EngineReportedSchedulingLoad, EngineReportedWorkerLoad, + EngineReportedLoadSnapshot, EngineReportedSchedulingLoad, }; use crate::workers::Worker; use std::cmp::Ordering; -use std::collections::HashMap; use std::sync::Arc; /// A prefill candidate domain and its optional queue budget. @@ -266,7 +268,7 @@ pub fn resolve_cache_candidates( // candidate missing native monitor data would otherwise break the // lookup's full-coverage check and silently downgrade every pressure // comparison (and `prefill_pressure_source`) to router-local. - let loads = FreshLoadLookup::new( + let loads = CandidateLoads::new( Some(snapshot), evaluated.iter().copied().map(|candidate| &candidate.worker), ); @@ -308,7 +310,7 @@ pub fn resolve_cache_candidates( // comparison would fall back to router-local load — tie at // zero for every owner, decided by worker id. The pin ranks // the rejected owners, so it must see the rejected owners. - let pin_loads = FreshLoadLookup::new( + let pin_loads = CandidateLoads::new( Some(snapshot), queue_gate_rejected .iter() @@ -322,7 +324,7 @@ pub fn resolve_cache_candidates( }) .min_by(|left, right| { pin_loads - .compare_prefill_pressure(&left.worker, &right.worker) + .compare_prefill_engines(&left.worker, &right.worker) .then_with(|| left.worker.id.0.cmp(&right.worker.id.0)) }) }) @@ -510,7 +512,7 @@ pub fn resolve_decode( .is_some_and(|worker| is_decode_admitted(worker, request_kv_tokens, snapshot)); let (selected, reason) = match (primary_admitted, backup.as_ref(), backup_admitted) { (true, Some(backup), true) => { - if compare_decode_pressure(&proposal.primary, backup, Some(snapshot)).is_gt() { + if compare_decode_engines(&proposal.primary, backup, Some(snapshot)).is_gt() { (Arc::clone(backup), DecisionReason::BackupPressureGuard) } else { (Arc::clone(&proposal.primary), DecisionReason::Primary) @@ -589,7 +591,7 @@ fn is_decode_admitted( fn is_cache_candidate_admitted( candidate: &CacheCandidate, request_input_tokens: u64, - loads: &FreshLoadLookup<'_>, + loads: &CandidateLoads<'_>, ) -> bool { let Some(load) = loads.get(&candidate.worker.id) else { return true; @@ -606,7 +608,7 @@ fn compare_cache_candidates( left: &CacheCandidate, right: &CacheCandidate, proposal: &CacheCandidateProposal, - loads: &FreshLoadLookup<'_>, + loads: &CandidateLoads<'_>, enable_pressure_guard: bool, ) -> Ordering { let work_delta = left.uncached_tokens.abs_diff(right.uncached_tokens); @@ -614,7 +616,7 @@ fn compare_cache_candidates( return left .uncached_tokens .cmp(&right.uncached_tokens) - .then_with(|| loads.compare_prefill_pressure(&left.worker, &right.worker)) + .then_with(|| loads.compare_prefill_engines(&left.worker, &right.worker)) .then_with(|| left.worker.id.0.cmp(&right.worker.id.0)); } if enable_pressure_guard { @@ -641,14 +643,14 @@ fn compare_cache_candidates( } left.uncached_tokens .cmp(&right.uncached_tokens) - .then_with(|| loads.compare_prefill_pressure(&left.worker, &right.worker)) + .then_with(|| loads.compare_prefill_engines(&left.worker, &right.worker)) .then_with(|| left.worker.id.0.cmp(&right.worker.id.0)) } fn cache_pressure_guard_comparable( left: &CacheCandidate, right: &CacheCandidate, - loads: &FreshLoadLookup<'_>, + loads: &CandidateLoads<'_>, ) -> bool { loads.comparable_get(&left.worker.id).is_some() && loads.comparable_get(&right.worker.id).is_some() @@ -660,7 +662,7 @@ fn materially_more_pressured( absolute_threshold_tokens: u64, absolute_threshold_ms: Option, relative_threshold: f64, - loads: &FreshLoadLookup<'_>, + loads: &CandidateLoads<'_>, ) -> bool { let (Some(candidate_load), Some(other_load)) = ( loads.comparable_get(&candidate.id), @@ -689,176 +691,6 @@ fn materially_more_pressured( > other_load.num_waiting_uncached_tokens as f64 * relative_threshold } -/// Constant-time request view over one captured load snapshot. -/// -/// External values are compared only when every candidate is present. Mixed -/// candidate sets use Router-local active load to preserve ordering. -pub(crate) struct FreshLoadLookup<'a> { - by_worker_id: HashMap, - basic_by_worker_id: HashMap, - local_active_by_worker_id: HashMap, - compare_engine: bool, - compare_basic_engine: bool, -} - -impl<'a> FreshLoadLookup<'a> { - pub(crate) fn new<'w>( - snapshot: Option<&'a EngineReportedLoadSnapshot>, - workers: impl IntoIterator>, - ) -> Self { - let workers: Vec<&Arc> = workers.into_iter().collect(); - let local_active_by_worker_id: HashMap = workers - .iter() - .map(|worker| (worker.id.0.clone(), worker.router_inflight_load())) - .collect(); - let by_worker_id = snapshot - .into_iter() - .flat_map(|snapshot| { - workers.iter().filter_map(move |worker| { - snapshot - .fresh_native_cache_load_for_url(&worker.url) - .map(|load| (worker.id.0.clone(), load)) - }) - }) - .collect::>(); - let basic_by_worker_id = snapshot - .into_iter() - .flat_map(|snapshot| { - workers.iter().filter_map(move |worker| { - snapshot - .fresh_load_for_url(&worker.url) - .map(|load| (worker.id.0.clone(), load)) - }) - }) - .collect::>(); - let compare_engine = !local_active_by_worker_id.is_empty() - && by_worker_id.len() == local_active_by_worker_id.len(); - let compare_basic_engine = !local_active_by_worker_id.is_empty() - && basic_by_worker_id.len() == local_active_by_worker_id.len(); - Self { - by_worker_id, - basic_by_worker_id, - local_active_by_worker_id, - compare_engine, - compare_basic_engine, - } - } - - pub(crate) fn get( - &self, - worker_id: &crate::discovery::WorkerId, - ) -> Option<&'a EngineReportedSchedulingLoad> { - self.by_worker_id.get(worker_id.0.as_str()).copied() - } - - fn comparable_get( - &self, - worker_id: &crate::discovery::WorkerId, - ) -> Option<&'a EngineReportedSchedulingLoad> { - self.compare_engine.then(|| self.get(worker_id)).flatten() - } - - fn pressure_key(&self, worker: &Arc) -> PressureKey<'a> { - PressureKey { - load: self.comparable_get(&worker.id), - local_active: self - .local_active_by_worker_id - .get(worker.id.0.as_str()) - .copied() - .unwrap_or(usize::MAX), - } - } - - fn compare_prefill_keys(&self, left: &PressureKey<'a>, right: &PressureKey<'a>) -> Ordering { - match (left.load, right.load) { - (Some(left_load), Some(right_load)) => compare_prefill_load(left_load, right_load) - .then_with(|| left.local_active.cmp(&right.local_active)), - _ => left.local_active.cmp(&right.local_active), - } - } - - fn compare_decode_keys(&self, left: &PressureKey<'a>, right: &PressureKey<'a>) -> Ordering { - match (left.load, right.load) { - (Some(left_load), Some(right_load)) => compare_decode_load(left_load, right_load) - .then_with(|| left.local_active.cmp(&right.local_active)), - _ => left.local_active.cmp(&right.local_active), - } - } - - pub(crate) fn compare_prefill_pressure( - &self, - left: &Arc, - right: &Arc, - ) -> Ordering { - self.compare_prefill_keys(&self.pressure_key(left), &self.pressure_key(right)) - } - - pub(crate) fn prefill_pressure_source(&self) -> &'static str { - if self.compare_engine - && self - .by_worker_id - .values() - .all(|load| load.estimated_prefill_queue_ms.is_some()) - { - "estimated_prefill_queue_ms" - } else if self.compare_engine { - "native_queue_tokens" - } else { - "router_local" - } - } - - /// Returns a queue depth consistent with admission for this request. - /// - /// A fully covered candidate set uses `waiting + running`; otherwise the - /// whole set uses Router-local active load. Dispatches after the snapshot - /// are added to the reported value. - pub(crate) fn score_load(&self, worker: &Arc) -> usize { - self.compare_basic_engine - .then(|| self.basic_by_worker_id.get(worker.id.0.as_str()).copied()) - .flatten() - .map(|load| { - let recent_dispatches = worker - .slots_acquired_since(load.captured_at) - .try_into() - .unwrap_or(u64::MAX); - load.num_waiting_reqs - .saturating_add(load.num_running_reqs) - .saturating_add(recent_dispatches) - .try_into() - .unwrap_or(usize::MAX) - }) - .unwrap_or_else(|| { - self.local_active_by_worker_id - .get(worker.id.0.as_str()) - .copied() - .unwrap_or(usize::MAX) - }) - } - fn min_by_pressure_key( - &self, - candidates: Vec>, - compare: impl Fn(&Self, &PressureKey<'a>, &PressureKey<'a>) -> Ordering, - ) -> Option> { - let mut candidates = candidates.into_iter(); - let mut best = candidates.next()?; - let mut best_key = self.pressure_key(&best); - for candidate in candidates { - let key = self.pressure_key(&candidate); - if compare(self, &key, &best_key).is_lt() { - best = candidate; - best_key = key; - } - } - Some(best) - } -} - -struct PressureKey<'a> { - load: Option<&'a EngineReportedSchedulingLoad>, - local_active: usize, -} - fn range_fallback( range: &CandidateRange<'_>, legal: &[Arc], @@ -901,9 +733,9 @@ fn range_fallback( // Scoped to the pool actually ranked: an admitted-but-gated worker // missing native monitor data would otherwise downgrade the comparison // for the whole unqueued tier to router-local. - let loads = FreshLoadLookup::new(Some(snapshot), pool.iter()); + let loads = CandidateLoads::new(Some(snapshot), pool.iter()); loads - .min_by_pressure_key(pool, FreshLoadLookup::compare_prefill_keys) + .select_min_by_load_key(pool, CandidateLoads::compare_prefill_load_keys) .map(|worker| (worker, DecisionReason::RangeFallback)) } @@ -932,100 +764,12 @@ fn decode_domain_fallback( .filter(|worker| is_decode_admitted(worker, request_kv_tokens, snapshot)) .cloned() .collect::>(); - let loads = FreshLoadLookup::new(Some(snapshot), admitted.iter()); + let loads = CandidateLoads::new(Some(snapshot), admitted.iter()); loads - .min_by_pressure_key(admitted, FreshLoadLookup::compare_decode_keys) + .select_min_by_load_key(admitted, CandidateLoads::compare_decode_load_keys) .map(|worker| (worker, DecisionReason::RangeFallback)) } -/// Compares prefill pressure by queue time when available, then by the V3 load tuple. -pub(crate) fn compare_prefill_pressure( - left: &Arc, - right: &Arc, - snapshot: Option<&EngineReportedLoadSnapshot>, -) -> Ordering { - match snapshot.and_then(|snapshot| { - Some(( - snapshot.fresh_native_cache_load_for_url(&left.url)?, - snapshot.fresh_native_cache_load_for_url(&right.url)?, - )) - }) { - Some((left_load, right_load)) => { - compare_prefill_load(left_load, right_load).then_with(|| { - left.router_inflight_load() - .cmp(&right.router_inflight_load()) - }) - } - None => left - .router_inflight_load() - .cmp(&right.router_inflight_load()), - } -} - -fn prefill_pressure_key(load: &EngineReportedSchedulingLoad) -> (u64, u64, u64) { - ( - load.num_waiting_uncached_tokens, - load.num_waiting_reqs, - load.num_running_reqs, - ) -} - -fn compare_prefill_load( - left: &EngineReportedSchedulingLoad, - right: &EngineReportedSchedulingLoad, -) -> Ordering { - match ( - left.estimated_prefill_queue_ms, - right.estimated_prefill_queue_ms, - ) { - (Some(left_ms), Some(right_ms)) => left_ms - .total_cmp(&right_ms) - .then_with(|| prefill_pressure_key(left).cmp(&prefill_pressure_key(right))), - _ => prefill_pressure_key(left).cmp(&prefill_pressure_key(right)), - } -} - -/// Compares decode pressure from LoadStat without treating unknown capacity as zero. -pub(crate) fn compare_decode_pressure( - left: &Arc, - right: &Arc, - snapshot: Option<&EngineReportedLoadSnapshot>, -) -> Ordering { - match snapshot.and_then(|snapshot| { - Some(( - snapshot.fresh_native_cache_load_for_url(&left.url)?, - snapshot.fresh_native_cache_load_for_url(&right.url)?, - )) - }) { - Some((left_load, right_load)) => { - compare_decode_load(left_load, right_load).then_with(|| { - left.router_inflight_load() - .cmp(&right.router_inflight_load()) - }) - } - None => left - .router_inflight_load() - .cmp(&right.router_inflight_load()), - } -} - -fn compare_decode_load( - left: &EngineReportedSchedulingLoad, - right: &EngineReportedSchedulingLoad, -) -> Ordering { - let kv_usage = match (left.max_total_num_tokens, right.max_total_num_tokens) { - (left_cap, right_cap) if left_cap > 0 && right_cap > 0 => u128::from(left.num_used_tokens) - .saturating_mul(u128::from(right_cap)) - .cmp(&u128::from(right.num_used_tokens).saturating_mul(u128::from(left_cap))), - _ => Ordering::Equal, - }; - left.num_waiting_reqs - .cmp(&right.num_waiting_reqs) - .then_with(|| left.num_running_reqs.cmp(&right.num_running_reqs)) - .then(kv_usage) - .then_with(|| left.num_used_tokens.cmp(&right.num_used_tokens)) -} - fn pressure_guard_prefers_backup( primary: &Arc, backup: &Arc, @@ -1199,22 +943,6 @@ mod tests { assert_eq!(decision.load_snapshot_version, explicit.version); } - #[test] - fn prefill_pressure_uses_waiting_then_running_requests() { - let busy = worker("busy"); - let idle = worker("idle"); - let loads = snapshot(&[(&busy, 1, 8, 10, 100), (&idle, 9, 2, 90, 100)]); - assert!(compare_prefill_pressure(&busy, &idle, Some(&loads)).is_gt()); - } - - #[test] - fn missing_snapshot_uses_local_active_load() { - let left = worker("left"); - let right = worker("right"); - let _guard = left.load_guard(); - assert!(compare_prefill_pressure(&left, &right, None).is_gt()); - } - #[test] fn complete_monitor_pressure_guard_overrides_a_near_cache_gain() { let congested = worker("congested"); diff --git a/experimental/sgl-router/src/policies/cache_aware.rs b/experimental/sgl-router/src/policies/cache_aware.rs index 6ad349b3f5..c24f1e6595 100644 --- a/experimental/sgl-router/src/policies/cache_aware.rs +++ b/experimental/sgl-router/src/policies/cache_aware.rs @@ -4,7 +4,7 @@ //! Builds bounded cache-aware candidates from ingress Indexer results. use crate::config::AffinityConfig; -use crate::policies::admission::FreshLoadLookup; +use crate::policies::admission::CandidateLoads; use crate::policies::power_of_two::PowerOfTwoChoicesPolicy; use crate::policies::{ CacheCandidate, CacheCandidateProposal, Policy, PrefillProposal, ProposalKind, @@ -88,7 +88,7 @@ impl CacheAwarePolicy { if limit == 0 { return None; } - let loads = FreshLoadLookup::new( + let loads = CandidateLoads::new( ctx.load_snapshot(), candidates.iter().map(|candidate| &candidate.worker), ); @@ -139,12 +139,12 @@ impl CacheAwarePolicy { fn compare_candidate_seed( left: &CacheCandidate, right: &CacheCandidate, - loads: &FreshLoadLookup<'_>, + loads: &CandidateLoads<'_>, ) -> Ordering { right .matched_prefix_tokens .cmp(&left.matched_prefix_tokens) - .then_with(|| loads.compare_prefill_pressure(&left.worker, &right.worker)) + .then_with(|| loads.compare_prefill_engines(&left.worker, &right.worker)) .then_with(|| left.worker.id.0.cmp(&right.worker.id.0)) } diff --git a/experimental/sgl-router/src/policies/decode.rs b/experimental/sgl-router/src/policies/decode.rs index 2edb82877c..85e8cb744a 100644 --- a/experimental/sgl-router/src/policies/decode.rs +++ b/experimental/sgl-router/src/policies/decode.rs @@ -5,7 +5,7 @@ use crate::config::DecodePolicyKind; use crate::policies::admission::{ - compare_decode_pressure, resolve_decode, CandidateDomain, DecisionReason, FinalDecision, + compare_decode_engines, resolve_decode, CandidateDomain, DecisionReason, FinalDecision, RoutingStage, }; use crate::policies::registry::select_decode_with_affinity; @@ -126,7 +126,7 @@ impl DecodePolicy for DecodePowerOfTwoPolicy { let left = &domain.workers[i]; let right = &domain.workers[j]; let (primary, backup) = - if compare_decode_pressure(left, right, ctx.load_snapshot()).is_gt() { + if compare_decode_engines(left, right, ctx.load_snapshot()).is_gt() { (Arc::clone(right), Arc::clone(left)) } else { (Arc::clone(left), Arc::clone(right)) diff --git a/experimental/sgl-router/src/policies/load_based.rs b/experimental/sgl-router/src/policies/load_based.rs index 0ea3b8fe87..1feca422d4 100644 --- a/experimental/sgl-router/src/policies/load_based.rs +++ b/experimental/sgl-router/src/policies/load_based.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors // SPDX-License-Identifier: Apache-2.0 -use crate::policies::admission::FreshLoadLookup; +use crate::policies::admission::CandidateLoads; use crate::policies::scoring::ScoringPolicy; use crate::policies::SelectionContext; use crate::workers::Worker; @@ -34,7 +34,7 @@ impl ScoringPolicy for LoadBasedPolicy { /// Purely a preference: "everybody is busy" is not a reason to refuse to /// route, so this term never constrains. Capacity is `--filter`'s job. fn scores(&self, workers: &[Arc], ctx: &SelectionContext<'_>) -> Vec { - let lookup = FreshLoadLookup::new(ctx.load_snapshot(), workers.iter()); + let lookup = CandidateLoads::new(ctx.load_snapshot(), workers.iter()); let loads: Vec = workers.iter().map(|w| lookup.score_load(w)).collect(); let lo = loads.iter().min().copied().unwrap_or(0); let span = (loads.iter().max().copied().unwrap_or(0) - lo) as f32; diff --git a/experimental/sgl-router/src/policies/mod.rs b/experimental/sgl-router/src/policies/mod.rs index 6ce57db5e9..f59c3551cf 100644 --- a/experimental/sgl-router/src/policies/mod.rs +++ b/experimental/sgl-router/src/policies/mod.rs @@ -9,7 +9,6 @@ pub mod dp_rank; pub mod factory; pub mod load_based; pub mod power_of_two; -pub mod prefix_provider; pub mod random; pub mod registry; pub mod round_robin; @@ -22,6 +21,7 @@ use crate::discovery::ModelId; use crate::policies::buckets::{BucketRequest, BucketSelector}; use crate::policies::scoring::{EligibilityFilter, ScoringPolicy}; use crate::server::metrics::MetricsRegistry; +pub(crate) use crate::state::kv_events::PrefixLookupResult; use crate::state::load_monitor::engine_reported_load::EngineReportedLoadSnapshot; use crate::tokenizer::{adapter, TokenizerRegistry}; use crate::workers::Worker; @@ -37,16 +37,6 @@ pub struct RequestTokens { pub rendered_from_chat: bool, } -/// External indexer answer prepared by the async ingress path for the -/// synchronous cache-aware policy. -pub struct ExternalPrefixSignal { - pub outcome: sgl_kv_indexer::PrefixOutcome, - pub query_blocks: usize, - /// The query's block hashes when the local tree answered, so routing can - /// record the placement without rehashing. - pub block_hashes: Option>, -} - /// Whether the caller pre-tokenized the prompt (`input_ids` present and not /// null). Such a request is never re-rendered: its ids drive routing and the /// body is forwarded untouched, malformed values included, for the engine to @@ -176,7 +166,7 @@ pub struct SelectionContext<'a> { candidate_range_id: &'a str, input_tokens: Option, request_tokens: Option<&'a [u32]>, - external_prefix: Option<&'a ExternalPrefixSignal>, + external_prefix: Option<&'a PrefixLookupResult>, load_snapshot: Option<&'a EngineReportedLoadSnapshot>, prefill_cache_bucket: Option<(&'a BucketSelector, BucketRequest)>, affinity_lookup_enabled: bool, @@ -246,10 +236,7 @@ impl<'a> SelectionContext<'a> { self } - pub fn with_external_prefix( - mut self, - external_prefix: Option<&'a ExternalPrefixSignal>, - ) -> Self { + pub fn with_external_prefix(mut self, external_prefix: Option<&'a PrefixLookupResult>) -> Self { self.external_prefix = external_prefix; self } @@ -312,7 +299,7 @@ impl<'a> SelectionContext<'a> { self.request_tokens } - pub fn external_prefix(&self) -> Option<&ExternalPrefixSignal> { + pub fn external_prefix(&self) -> Option<&PrefixLookupResult> { self.external_prefix } @@ -597,7 +584,7 @@ mod tests { use crate::config::{AffinityConfig, SessionAffinityMode}; use crate::discovery::{WorkerId, WorkerMode, WorkerSpec}; use crate::policies::admission::{ - resolve_cache_candidates, resolve_prefill, CandidateRange, DecisionReason, FreshLoadLookup, + resolve_cache_candidates, resolve_prefill, CandidateRange, DecisionReason, }; use crate::policies::cache_aware::CacheAwarePolicy; use crate::policies::power_of_two::PowerOfTwoChoicesPolicy; @@ -891,24 +878,13 @@ mod tests { assert_eq!(returned.kind, ProposalKind::SessionAffinity); } - #[test] - fn decode_pressure_tie_is_not_broken_by_worker_id() { - let a = worker("a"); - let z = worker("z"); - assert_eq!( - admission::compare_decode_pressure(&a, &z, None), - std::cmp::Ordering::Equal, - "P2 must preserve random sampling when observable pressure is equal" - ); - } - #[test] fn cache_affinity_uses_longest_routable_prefix_holder() { let model = ModelId("model".into()); let hot = worker("hot"); let other = worker("other"); let workers = vec![Arc::clone(&hot), Arc::clone(&other)]; - let signal = ExternalPrefixSignal { + let signal = PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches: vec![ sgl_kv_indexer::PrefixMatch { @@ -952,7 +928,7 @@ mod tests { let hot = worker("hot"); let warm = worker("warm"); let workers = vec![Arc::clone(&hot), Arc::clone(&warm)]; - let signal = ExternalPrefixSignal { + let signal = PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches: vec![ sgl_kv_indexer::PrefixMatch { @@ -1018,7 +994,7 @@ mod tests { address: worker.url.clone(), }) .collect(); - let signal = ExternalPrefixSignal { + let signal = PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches, best_prefix_blocks: workers.len() as u32, @@ -1075,7 +1051,7 @@ mod tests { address: worker.url.clone(), }) .collect(); - let signal = ExternalPrefixSignal { + let signal = PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches, best_prefix_blocks: 4, @@ -1116,7 +1092,7 @@ mod tests { let half = worker("half"); let below_ratio = worker("below-ratio"); let workers = vec![Arc::clone(&half), Arc::clone(&below_ratio)]; - let signal = ExternalPrefixSignal { + let signal = PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches: vec![ sgl_kv_indexer::PrefixMatch { @@ -1163,7 +1139,7 @@ mod tests { let model = ModelId("model".into()); let weak = worker("weak"); let workers = vec![Arc::clone(&weak)]; - let signal = ExternalPrefixSignal { + let signal = PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches: vec![sgl_kv_indexer::PrefixMatch { matched_prefix_blocks: 3, @@ -1195,7 +1171,7 @@ mod tests { let model = ModelId("model".into()); let holder = worker("holder"); let workers = vec![Arc::clone(&holder)]; - let signal = ExternalPrefixSignal { + let signal = PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches: vec![sgl_kv_indexer::PrefixMatch { matched_prefix_blocks: 2_048, @@ -1269,45 +1245,6 @@ mod tests { ) } - #[test] - fn mixed_freshness_uses_one_captured_local_level_for_the_candidate_set() { - let aggregate_idle = worker("aggregate-idle"); - let aggregate_busy = worker("aggregate-busy"); - let stale = worker("stale"); - aggregate_idle - .active_requests - .store(5, std::sync::atomic::Ordering::Relaxed); - aggregate_busy - .active_requests - .store(1, std::sync::atomic::Ordering::Relaxed); - let snapshot = snapshot(&[ - ( - &aggregate_idle, - TestEngineLoad { - num_waiting_reqs: 0, - ..TestEngineLoad::default() - }, - ), - ( - &aggregate_busy, - TestEngineLoad { - num_waiting_reqs: 1_000, - ..TestEngineLoad::default() - }, - ), - ]); - - let lookup = - FreshLoadLookup::new(Some(&snapshot), [&aggregate_idle, &aggregate_busy, &stale]); - assert!(lookup.get(&aggregate_idle.id).is_some()); - assert!(lookup.get(&stale.id).is_none()); - assert_eq!( - lookup.compare_prefill_pressure(&aggregate_idle, &aggregate_busy), - std::cmp::Ordering::Greater, - "one stale member makes the complete candidate set compare by the captured local level" - ); - } - fn cache_candidate( worker: &Arc, matched_prefix_tokens: u64, diff --git a/experimental/sgl-router/src/policies/power_of_two.rs b/experimental/sgl-router/src/policies/power_of_two.rs index 2d130bbbb6..efcae9d9ae 100644 --- a/experimental/sgl-router/src/policies/power_of_two.rs +++ b/experimental/sgl-router/src/policies/power_of_two.rs @@ -14,7 +14,7 @@ //! never a sort. use crate::config::DEFAULT_MIN_LOAD_CHOICES; -use crate::policies::admission::{compare_prefill_pressure, queue_gate_admits}; +use crate::policies::admission::{compare_prefill_engines, queue_gate_admits}; use crate::policies::{Policy, ProposalKind, SelectionContext, SelectionProposal}; use crate::state::load_monitor::engine_reported_load::EngineReportedLoadSnapshot; use crate::workers::Worker; @@ -128,7 +128,7 @@ fn sample_pool<'w>( /// two-worker fleet, so a fixed scan order would pin every fallback /// dispatch to the first worker. /// -/// Both tiers scan linearly and never sort. `compare_prefill_pressure` +/// Both tiers scan linearly and never sort. `compare_prefill_engines` /// is only a pairwise comparison: two workers that both publish /// `estimated_prefill_queue_ms` are ordered on that estimate, and any /// other pair on the waiting-token tuple, so it is not a total order @@ -164,12 +164,12 @@ fn best_two_of_sample( // Only a strict improvement displaces the incumbent, so a tie keeps // whichever member the draw presented first. if best.is_none_or(|current| { - compare_prefill_pressure(&pool[index], &pool[current], snapshot).is_lt() + compare_prefill_engines(&pool[index], &pool[current], snapshot).is_lt() }) { runner_up = best; best = Some(index); } else if runner_up.is_none_or(|current| { - compare_prefill_pressure(&pool[index], &pool[current], snapshot).is_lt() + compare_prefill_engines(&pool[index], &pool[current], snapshot).is_lt() }) { runner_up = Some(index); } @@ -231,7 +231,7 @@ mod tests { /// A fleet where only some workers publish `estimated_prefill_queue_ms`. /// An idle worker has no throughput delta to derive one from, so this is /// the steady state, not an edge case - and it makes - /// `compare_prefill_pressure` intransitive: a slow worker with a shallow + /// `compare_prefill_engines` intransitive: a slow worker with a shallow /// queue loses to a fast worker with a deep one on the estimate, while /// both are ordered against an estimate-less worker on waiting tokens. fn mixed_estimate_snapshot(workers: &[Arc]) -> EngineReportedLoadSnapshot { @@ -265,7 +265,7 @@ mod tests { ) } - /// `compare_prefill_pressure` is a pairwise comparison, not a total + /// `compare_prefill_engines` is a pairwise comparison, not a total /// order, so the k-way minimum must be a linear scan. Sorting a sample /// this size panics with "user-provided comparison function does not /// correctly implement a total order" - every request that samples a diff --git a/experimental/sgl-router/src/policies/selection.rs b/experimental/sgl-router/src/policies/selection.rs index 99b9646bad..987ce1dbfa 100644 --- a/experimental/sgl-router/src/policies/selection.rs +++ b/experimental/sgl-router/src/policies/selection.rs @@ -36,7 +36,7 @@ use crate::policies::decode::{ build_decode_policy, resolve_decode_with_capacity_fallback, DecodeSelectionContext, }; use crate::policies::{ - ExternalPrefixSignal, Policy, PrefillProposal, ProposalKind, SelectionContext, + Policy, PrefillProposal, PrefixLookupResult, ProposalKind, SelectionContext, }; use crate::server::metrics::{CacheAwareDecision, MetricsRegistry, PolicySelectionFailureReason}; use crate::state::load_monitor::engine_reported_load::EngineReportedLoadSnapshot; @@ -59,7 +59,7 @@ pub(crate) struct PrefillSelectionInputs<'a> { /// `request_input_tokens` only for a batch. pub request_sequence_tokens: u64, pub request_tokens: Option<&'a [u32]>, - pub external_prefix: Option<&'a ExternalPrefixSignal>, + pub external_prefix: Option<&'a PrefixLookupResult>, /// Required whenever `policy.uses_shared_prefill_admission()`; the /// per-domain rung panics without it. `Policy::needs_load_snapshot` /// defaults to `uses_shared_prefill_admission`, which is what keeps the @@ -707,7 +707,7 @@ mod tests { use crate::policies::buckets::BucketSelector; use crate::policies::cache_aware::CacheAwarePolicy; use crate::policies::power_of_two::PowerOfTwoChoicesPolicy; - use crate::policies::{ExternalPrefixSignal, Policy, ProposalKind, SelectionProposal}; + use crate::policies::{Policy, PrefixLookupResult, ProposalKind, SelectionProposal}; use crate::server::metrics::{ CacheAwareDecision, MetricsRegistry, PolicySelectionFailureReason, }; @@ -785,8 +785,8 @@ mod tests { } /// An indexer hit placing `matched_prefix_blocks` on each named worker. - fn prefix_signal(matches: &[(&Arc, u32)], query_blocks: usize) -> ExternalPrefixSignal { - ExternalPrefixSignal { + fn prefix_signal(matches: &[(&Arc, u32)], query_blocks: usize) -> PrefixLookupResult { + PrefixLookupResult { outcome: sgl_kv_indexer::PrefixOutcome::Matched { matches: matches .iter() diff --git a/experimental/sgl-router/src/policies/session_aware.rs b/experimental/sgl-router/src/policies/session_aware.rs index 5197d1dae3..af4174e479 100644 --- a/experimental/sgl-router/src/policies/session_aware.rs +++ b/experimental/sgl-router/src/policies/session_aware.rs @@ -5,7 +5,7 @@ use crate::config::{AffinityConfig, SessionAffinityMode}; use crate::discovery::WorkerId; -use crate::policies::admission::compare_prefill_pressure; +use crate::policies::admission::compare_prefill_engines; use crate::policies::power_of_two::PowerOfTwoChoicesPolicy; use crate::policies::{GuardHints, Policy, ProposalKind, SelectionContext, SelectionProposal}; use crate::state::load_monitor::router_inflight_load::JanitorHandle; @@ -201,7 +201,7 @@ fn sampled_backup_excluding( }; let left = &workers[first]; let right = &workers[second]; - if compare_prefill_pressure(left, right, ctx.load_snapshot()).is_gt() { + if compare_prefill_engines(left, right, ctx.load_snapshot()).is_gt() { Some(Arc::clone(right)) } else { Some(Arc::clone(left)) diff --git a/experimental/sgl-router/src/policies_reorg/cache_aware.rs b/experimental/sgl-router/src/policies_reorg/cache_aware.rs index ba82a0b3c0..4d56de62d6 100644 --- a/experimental/sgl-router/src/policies_reorg/cache_aware.rs +++ b/experimental/sgl-router/src/policies_reorg/cache_aware.rs @@ -14,10 +14,11 @@ use sgl_kv_indexer::{PrefixIndex, PrefixIndexError, PrefixOutcome}; use tokio::sync::OnceCell; use crate::config::{AffinityConfig, AffinityMode}; -use crate::policies::admission::FreshLoadLookup; -use crate::policies::prefix_provider::RadixTreePrefixProvider; -use crate::policies::ExternalPrefixSignal; -use crate::state::kv_events::{compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle}; +use crate::state::kv_events::{ + compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, PrefixLookupResult, + RadixTreePrefixProvider, +}; +use crate::state::load_monitor::engine_ranking::CandidateLoads; use crate::state::load_monitor::engine_reported_load::{ EngineReportedLoadSnapshot, EngineReportedLoadTable, }; @@ -28,7 +29,7 @@ use super::affinity; use super::power_of_two::PowerOfTwoPolicy; use super::{Pick, PickError, PickRequest, Policy, Rejection, Stage}; -type Signal = Option>; +type Signal = Option>; type Lookup = Arc>; /// Local radix tree or remote indexer. Groups sharing an index namespace share @@ -74,7 +75,7 @@ impl CacheSource { return Ok(None); } match index.match_prefix(hashes).await { - Ok(outcome) => Ok(Some(Arc::new(ExternalPrefixSignal { + Ok(outcome) => Ok(Some(Arc::new(PrefixLookupResult { outcome, query_blocks, block_hashes: None, @@ -105,7 +106,7 @@ impl fmt::Debug for PrefixMemo { impl PrefixMemo { /// The local tree's answer, once a policy has looked it up. - pub fn local_signal(&self) -> Option> { + pub fn local_signal(&self) -> Option> { let cells = self.cells.lock().unwrap_or_else(|e| e.into_inner()); cells .iter() @@ -133,10 +134,10 @@ struct Candidate<'a> { } /// Less uncached work first, then lower prefill pressure, then worker id. -fn rank(loads: &FreshLoadLookup<'_>, left: &Candidate<'_>, right: &Candidate<'_>) -> Ordering { +fn rank(loads: &CandidateLoads<'_>, left: &Candidate<'_>, right: &Candidate<'_>) -> Ordering { left.uncached_tokens .cmp(&right.uncached_tokens) - .then_with(|| loads.compare_prefill_pressure(left.engine, right.engine)) + .then_with(|| loads.compare_prefill_engines(left.engine, right.engine)) .then_with(|| left.engine.id.0.cmp(&right.engine.id.0)) } @@ -185,10 +186,10 @@ impl CacheAwarePolicy { &self, engines: &'e [Arc], request: &PickRequest<'_>, - signal: Option<&ExternalPrefixSignal>, + signal: Option<&PrefixLookupResult>, load: &EngineReportedLoadSnapshot, ) -> Vec> { - let Some(ExternalPrefixSignal { + let Some(PrefixLookupResult { outcome: PrefixOutcome::Matched { matches, .. }, query_blocks, .. @@ -229,7 +230,7 @@ impl CacheAwarePolicy { .len() .min(config.cache_candidate_max_workers) .min(config.cache_candidate_min_workers.max(proportional)); - let loads = FreshLoadLookup::new(Some(load), candidates.iter().map(|c| c.engine)); + let loads = CandidateLoads::new(Some(load), candidates.iter().map(|c| c.engine)); candidates.sort_by(|left, right| rank(&loads, left, right)); candidates.truncate(limit); candidates diff --git a/experimental/sgl-router/src/policies_reorg/factory.rs b/experimental/sgl-router/src/policies_reorg/factory.rs index e0bb799031..f25b07f3d5 100644 --- a/experimental/sgl-router/src/policies_reorg/factory.rs +++ b/experimental/sgl-router/src/policies_reorg/factory.rs @@ -7,7 +7,7 @@ use anyhow::{bail, ensure, Result}; use crate::buckets_reorg::{Bucket, BucketGroups, BucketResolver, EngineGroup}; use crate::config::{DecodePolicyKind, FilterKind, ModelConfig, PolicyKind, SessionAffinityMode}; -use crate::policies::prefix_provider::RadixTreePrefixProvider; +use crate::state::kv_events::RadixTreePrefixProvider; use crate::state::{ kv_events::KvEventIndex, load_monitor::router_inflight_load::JanitorHandle, AffinityStore, }; diff --git a/experimental/sgl-router/src/policies_reorg/power_of_two.rs b/experimental/sgl-router/src/policies_reorg/power_of_two.rs index 03d5bb38a9..1ac7d94fb3 100644 --- a/experimental/sgl-router/src/policies_reorg/power_of_two.rs +++ b/experimental/sgl-router/src/policies_reorg/power_of_two.rs @@ -7,14 +7,14 @@ use std::time::Instant; use futures::future::BoxFuture; use rand::Rng; -use crate::policies::admission::{compare_decode_pressure, compare_prefill_pressure}; +use crate::state::load_monitor::engine_ranking::{compare_decode_engines, compare_prefill_engines}; use crate::state::load_monitor::engine_reported_load::EngineReportedLoadTable; use crate::workers::Worker; use super::admission::{AdmissionLimits, Decision, EngineAdmission, EngineMetrics}; use super::{Pick, PickError, PickRequest, Policy, Rejection, Stage}; -/// Samples two distinct engines and selects the one with lower stage pressure. +/// Samples two distinct engines and selects the better-ranked one for its stage. /// Checks admission only on the selected engine; rejection never resamples. #[derive(Debug)] pub struct PowerOfTwoPolicy { @@ -54,13 +54,13 @@ impl Policy for PowerOfTwoPolicy { j += 1; } let (left, right) = (&engines[i], &engines[j]); - let pressure = match request.stage { + let ordering = match request.stage { Stage::Plain | Stage::Prefill => { - compare_prefill_pressure(left, right, Some(&load)) + compare_prefill_engines(left, right, Some(&load)) } - Stage::Decode => compare_decode_pressure(left, right, Some(&load)), + Stage::Decode => compare_decode_engines(left, right, Some(&load)), }; - Arc::clone(if pressure.is_gt() { right } else { left }) + Arc::clone(if ordering.is_gt() { right } else { left }) } }; let metrics = EngineMetrics::observe(&engine, &load); diff --git a/experimental/sgl-router/src/server/app_context.rs b/experimental/sgl-router/src/server/app_context.rs index 6c3f5123c2..a8d63405ae 100644 --- a/experimental/sgl-router/src/server/app_context.rs +++ b/experimental/sgl-router/src/server/app_context.rs @@ -6,12 +6,13 @@ use crate::config::Config; use crate::discovery::ModelId; use crate::policies::buckets::BucketSelector; -use crate::policies::prefix_provider::RadixTreePrefixProvider; use crate::policies::PolicyRegistry; use crate::proxy::Proxy; use crate::server::inflight::InflightHttp; use crate::server::metrics::MetricsRegistry; -use crate::state::kv_events::{BlockSizeOracle, KvEventIndex, KvIndexMetrics}; +use crate::state::kv_events::{ + BlockSizeOracle, KvEventIndex, KvIndexMetrics, RadixTreePrefixProvider, +}; use crate::state::load_monitor::engine_reported_load::EngineReportedLoadTable; use crate::state::load_monitor::router_inflight_load::RouterInflightLoadRegistry; use crate::tokenizer::TokenizerRegistry; diff --git a/experimental/sgl-router/src/server/routes/chat.rs b/experimental/sgl-router/src/server/routes/chat.rs index 2eb4f41900..0e6466e0e4 100644 --- a/experimental/sgl-router/src/server/routes/chat.rs +++ b/experimental/sgl-router/src/server/routes/chat.rs @@ -12,7 +12,7 @@ use crate::policies::registry::{PdPoolResolver, PdResolveError}; use crate::policies::selection::{ select_decode_peer, select_prefill_worker, DecodeSelectionInputs, PrefillSelectionInputs, }; -use crate::policies::{ExternalPrefixSignal, Policy}; +use crate::policies::{Policy, PrefixLookupResult}; use crate::server::app_context::{AppContext, ChatRouting}; use crate::server::error::ApiError; use crate::server::metrics::PolicySelectionFailureReason; @@ -229,11 +229,7 @@ async fn select_workers( /// Credit the chosen prefill with the prompt's prefix until KV events confirm /// it. Called only once the whole selection succeeded, so a request that is /// never dispatched credits nobody. -fn record_prefill_route( - ctx: &AppContext, - signal: Option<&ExternalPrefixSignal>, - prefill_url: &str, -) { +fn record_prefill_route(ctx: &AppContext, signal: Option<&PrefixLookupResult>, prefill_url: &str) { if let (Some(provider), Some(signal)) = (&ctx.radix_tree_prefix_provider, signal) { provider.record_route(signal, prefill_url); } @@ -287,7 +283,7 @@ fn capture_load_snapshot( } struct RoutingContext<'a> { - prefix_matches: Option, + prefix_matches: Option, load_snapshot: Option, ttft_slo_ms: Option, tps_slo: Option, @@ -414,7 +410,7 @@ fn pick_decode_worker( async fn lookup_prefix_matches( ctx: &AppContext, request: &PreparedRequest, -) -> Result, ApiError> { +) -> Result, ApiError> { let signal = match ( ctx.prefix_index.as_ref(), request.tokens.as_ref(), @@ -433,7 +429,7 @@ async fn lookup_prefix_matches( } else { resolve_prefix_query(index.match_prefix(hashes).await, &request.model.0)? }; - Some(ExternalPrefixSignal { + Some(PrefixLookupResult { outcome, query_blocks, block_hashes: None, diff --git a/experimental/sgl-router/src/state/kv_events/mod.rs b/experimental/sgl-router/src/state/kv_events/mod.rs index 19d60ea0de..a92e7ee004 100644 --- a/experimental/sgl-router/src/state/kv_events/mod.rs +++ b/experimental/sgl-router/src/state/kv_events/mod.rs @@ -26,6 +26,7 @@ pub mod discovery; pub mod hash; pub mod index; pub mod pending; +pub mod prefix_provider; pub mod subscriber; pub mod tally; pub mod tree; @@ -38,6 +39,7 @@ pub use discovery::{fetch_event_config, EventConfig}; pub use hash::{compute_block_hashes, compute_block_hashes_bigram, sha256_to_i64}; pub use index::{KvEventIndex, KvIndexMetrics}; pub use pending::PendingPrefixes; +pub use prefix_provider::{PrefixLookupResult, RadixTreePrefixProvider}; pub use subscriber::{KvEventSubscriberRegistry, SubKind, WorkerEvent}; pub use tally::{EventKind, EventTally, ReplayOutcome, TallyRow}; pub use tree::{ diff --git a/experimental/sgl-router/src/policies/prefix_provider.rs b/experimental/sgl-router/src/state/kv_events/prefix_provider.rs similarity index 85% rename from experimental/sgl-router/src/policies/prefix_provider.rs rename to experimental/sgl-router/src/state/kv_events/prefix_provider.rs index 74e6d7680b..139b7d5921 100644 --- a/experimental/sgl-router/src/policies/prefix_provider.rs +++ b/experimental/sgl-router/src/state/kv_events/prefix_provider.rs @@ -1,14 +1,21 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors // SPDX-License-Identifier: Apache-2.0 -use super::ExternalPrefixSignal; -use crate::state::kv_events::{ - compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, HashTree, -}; +use super::{compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, HashTree}; use sgl_kv_indexer::{PrefixMatch, PrefixOutcome}; use std::collections::BTreeMap; use std::sync::Arc; +/// Prefix lookup result from the local radix tree or remote KV indexer, +/// consumed by cache-aware routing. +pub struct PrefixLookupResult { + pub outcome: PrefixOutcome, + pub query_blocks: usize, + /// The query's block hashes when the local tree answered, so routing can + /// record the placement without rehashing. + pub block_hashes: Option>, +} + #[derive(Clone, Debug)] pub struct RadixTreePrefixProvider { tree: Arc, @@ -23,7 +30,7 @@ impl RadixTreePrefixProvider { } } - pub fn match_request_tokens(&self, tokens: &[u32]) -> Option { + pub fn match_request_tokens(&self, tokens: &[u32]) -> Option { let hashes = self.block_hashes(tokens)?; let mut depth_by_url = BTreeMap::::new(); @@ -55,7 +62,7 @@ impl RadixTreePrefixProvider { } }; // Empty is still returned so routing can record where this prompt went. - Some(ExternalPrefixSignal { + Some(PrefixLookupResult { outcome, query_blocks: hashes.len(), block_hashes: self.tree.pending().is_enabled().then(|| hashes.into()), @@ -63,7 +70,7 @@ impl RadixTreePrefixProvider { } /// Remember that `signal`'s prompt was just routed to `url`. - pub fn record_route(&self, signal: &ExternalPrefixSignal, url: &str) { + pub fn record_route(&self, signal: &PrefixLookupResult, url: &str) { if let Some(hashes) = &signal.block_hashes { self.tree.pending().record(url, hashes); } diff --git a/experimental/sgl-router/src/state/load_monitor/engine_ranking.rs b/experimental/sgl-router/src/state/load_monitor/engine_ranking.rs new file mode 100644 index 0000000000..5bdab7d6f8 --- /dev/null +++ b/experimental/sgl-router/src/state/load_monitor/engine_ranking.rs @@ -0,0 +1,362 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Stage-specific engine ranking over one captured load snapshot. + +use super::engine_reported_load::{ + EngineReportedLoadSnapshot, EngineReportedSchedulingLoad, EngineReportedWorkerLoad, +}; +use crate::workers::Worker; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::sync::Arc; + +/// Ranks prefill engines by queue time when available, then by the V3 load tuple. +pub(crate) fn compare_prefill_engines( + left: &Arc, + right: &Arc, + snapshot: Option<&EngineReportedLoadSnapshot>, +) -> Ordering { + match snapshot.and_then(|snapshot| { + Some(( + snapshot.fresh_native_cache_load_for_url(&left.url)?, + snapshot.fresh_native_cache_load_for_url(&right.url)?, + )) + }) { + Some((left_load, right_load)) => { + compare_prefill_load(left_load, right_load).then_with(|| { + left.router_inflight_load() + .cmp(&right.router_inflight_load()) + }) + } + None => left + .router_inflight_load() + .cmp(&right.router_inflight_load()), + } +} + +fn prefill_load_key(load: &EngineReportedSchedulingLoad) -> (u64, u64, u64) { + ( + load.num_waiting_uncached_tokens, + load.num_waiting_reqs, + load.num_running_reqs, + ) +} + +fn compare_prefill_load( + left: &EngineReportedSchedulingLoad, + right: &EngineReportedSchedulingLoad, +) -> Ordering { + match ( + left.estimated_prefill_queue_ms, + right.estimated_prefill_queue_ms, + ) { + (Some(left_ms), Some(right_ms)) => left_ms + .total_cmp(&right_ms) + .then_with(|| prefill_load_key(left).cmp(&prefill_load_key(right))), + _ => prefill_load_key(left).cmp(&prefill_load_key(right)), + } +} + +/// Ranks decode engines by LoadStat without treating unknown capacity as zero. +pub(crate) fn compare_decode_engines( + left: &Arc, + right: &Arc, + snapshot: Option<&EngineReportedLoadSnapshot>, +) -> Ordering { + match snapshot.and_then(|snapshot| { + Some(( + snapshot.fresh_native_cache_load_for_url(&left.url)?, + snapshot.fresh_native_cache_load_for_url(&right.url)?, + )) + }) { + Some((left_load, right_load)) => { + compare_decode_load(left_load, right_load).then_with(|| { + left.router_inflight_load() + .cmp(&right.router_inflight_load()) + }) + } + None => left + .router_inflight_load() + .cmp(&right.router_inflight_load()), + } +} + +fn compare_decode_load( + left: &EngineReportedSchedulingLoad, + right: &EngineReportedSchedulingLoad, +) -> Ordering { + let kv_usage = match (left.max_total_num_tokens, right.max_total_num_tokens) { + (left_cap, right_cap) if left_cap > 0 && right_cap > 0 => u128::from(left.num_used_tokens) + .saturating_mul(u128::from(right_cap)) + .cmp(&u128::from(right.num_used_tokens).saturating_mul(u128::from(left_cap))), + _ => Ordering::Equal, + }; + left.num_waiting_reqs + .cmp(&right.num_waiting_reqs) + .then_with(|| left.num_running_reqs.cmp(&right.num_running_reqs)) + .then(kv_usage) + .then_with(|| left.num_used_tokens.cmp(&right.num_used_tokens)) +} + +/// Constant-time request view over one captured load snapshot. +/// +/// External values are compared only when every candidate is present. Mixed +/// candidate sets use Router-local active load to preserve ordering. +/// +/// The key-level helpers (`comparable_get`, `compare_*_keys`, +/// `select_min_by_load_key`) are `pub(crate)` only for legacy admission. +pub(crate) struct CandidateLoads<'a> { + by_worker_id: HashMap, + basic_by_worker_id: HashMap, + local_active_by_worker_id: HashMap, + compare_engine: bool, + compare_basic_engine: bool, +} + +impl<'a> CandidateLoads<'a> { + pub(crate) fn new<'w>( + snapshot: Option<&'a EngineReportedLoadSnapshot>, + workers: impl IntoIterator>, + ) -> Self { + let workers: Vec<&Arc> = workers.into_iter().collect(); + let local_active_by_worker_id: HashMap = workers + .iter() + .map(|worker| (worker.id.0.clone(), worker.router_inflight_load())) + .collect(); + let by_worker_id = snapshot + .into_iter() + .flat_map(|snapshot| { + workers.iter().filter_map(move |worker| { + snapshot + .fresh_native_cache_load_for_url(&worker.url) + .map(|load| (worker.id.0.clone(), load)) + }) + }) + .collect::>(); + let basic_by_worker_id = snapshot + .into_iter() + .flat_map(|snapshot| { + workers.iter().filter_map(move |worker| { + snapshot + .fresh_load_for_url(&worker.url) + .map(|load| (worker.id.0.clone(), load)) + }) + }) + .collect::>(); + let compare_engine = !local_active_by_worker_id.is_empty() + && by_worker_id.len() == local_active_by_worker_id.len(); + let compare_basic_engine = !local_active_by_worker_id.is_empty() + && basic_by_worker_id.len() == local_active_by_worker_id.len(); + Self { + by_worker_id, + basic_by_worker_id, + local_active_by_worker_id, + compare_engine, + compare_basic_engine, + } + } + + pub(crate) fn get( + &self, + worker_id: &crate::discovery::WorkerId, + ) -> Option<&'a EngineReportedSchedulingLoad> { + self.by_worker_id.get(worker_id.0.as_str()).copied() + } + + pub(crate) fn comparable_get( + &self, + worker_id: &crate::discovery::WorkerId, + ) -> Option<&'a EngineReportedSchedulingLoad> { + self.compare_engine.then(|| self.get(worker_id)).flatten() + } + + fn load_key(&self, worker: &Arc) -> LoadKey<'a> { + LoadKey { + load: self.comparable_get(&worker.id), + local_active: self + .local_active_by_worker_id + .get(worker.id.0.as_str()) + .copied() + .unwrap_or(usize::MAX), + } + } + + pub(crate) fn compare_prefill_load_keys( + &self, + left: &LoadKey<'a>, + right: &LoadKey<'a>, + ) -> Ordering { + match (left.load, right.load) { + (Some(left_load), Some(right_load)) => compare_prefill_load(left_load, right_load) + .then_with(|| left.local_active.cmp(&right.local_active)), + _ => left.local_active.cmp(&right.local_active), + } + } + + pub(crate) fn compare_decode_load_keys( + &self, + left: &LoadKey<'a>, + right: &LoadKey<'a>, + ) -> Ordering { + match (left.load, right.load) { + (Some(left_load), Some(right_load)) => compare_decode_load(left_load, right_load) + .then_with(|| left.local_active.cmp(&right.local_active)), + _ => left.local_active.cmp(&right.local_active), + } + } + + pub(crate) fn compare_prefill_engines( + &self, + left: &Arc, + right: &Arc, + ) -> Ordering { + self.compare_prefill_load_keys(&self.load_key(left), &self.load_key(right)) + } + + pub(crate) fn prefill_pressure_source(&self) -> &'static str { + if self.compare_engine + && self + .by_worker_id + .values() + .all(|load| load.estimated_prefill_queue_ms.is_some()) + { + "estimated_prefill_queue_ms" + } else if self.compare_engine { + "native_queue_tokens" + } else { + "router_local" + } + } + + /// Returns a queue depth consistent with admission for this request. + /// + /// A fully covered candidate set uses `waiting + running`; otherwise the + /// whole set uses Router-local active load. Dispatches after the snapshot + /// are added to the reported value. + pub(crate) fn score_load(&self, worker: &Arc) -> usize { + self.compare_basic_engine + .then(|| self.basic_by_worker_id.get(worker.id.0.as_str()).copied()) + .flatten() + .map(|load| { + let recent_dispatches = worker + .slots_acquired_since(load.captured_at) + .try_into() + .unwrap_or(u64::MAX); + load.num_waiting_reqs + .saturating_add(load.num_running_reqs) + .saturating_add(recent_dispatches) + .try_into() + .unwrap_or(usize::MAX) + }) + .unwrap_or_else(|| { + self.local_active_by_worker_id + .get(worker.id.0.as_str()) + .copied() + .unwrap_or(usize::MAX) + }) + } + pub(crate) fn select_min_by_load_key( + &self, + candidates: Vec>, + compare: impl Fn(&Self, &LoadKey<'a>, &LoadKey<'a>) -> Ordering, + ) -> Option> { + let mut candidates = candidates.into_iter(); + let mut best = candidates.next()?; + let mut best_key = self.load_key(&best); + for candidate in candidates { + let key = self.load_key(&candidate); + if compare(self, &key, &best_key).is_lt() { + best = candidate; + best_key = key; + } + } + Some(best) + } +} + +pub(crate) struct LoadKey<'a> { + load: Option<&'a EngineReportedSchedulingLoad>, + local_active: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; + use std::time::Instant; + + fn worker(id: &str) -> Arc { + Arc::new(Worker::new(WorkerSpec { + id: WorkerId(id.into()), + url: format!("http://{id}:30000"), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("model".into())], + ..Default::default() + })) + } + + /// `(worker, running, waiting)`; waiting doubles as uncached tokens. + fn snapshot(entries: &[(&Arc, u64, u64)]) -> EngineReportedLoadSnapshot { + EngineReportedLoadSnapshot::from_native_cache_workers( + 7, + entries + .iter() + .map(|(worker, running, waiting)| { + ( + worker.url.clone(), + EngineReportedSchedulingLoad { + num_running_reqs: *running, + num_waiting_reqs: *waiting, + num_waiting_uncached_tokens: *waiting, + num_used_tokens: 0, + num_total_tokens: 0, + max_total_num_tokens: 100, + max_running_requests: 64, + prefill_throughput_tokens_per_s: None, + estimated_prefill_queue_ms: None, + captured_at: Instant::now(), + }, + ) + }) + .collect(), + ) + } + + #[test] + fn prefill_ranking_uses_waiting_then_running_requests() { + let busy = worker("busy"); + let idle = worker("idle"); + let loads = snapshot(&[(&busy, 1, 8), (&idle, 9, 2)]); + assert!(compare_prefill_engines(&busy, &idle, Some(&loads)).is_gt()); + } + + #[test] + fn missing_snapshot_uses_local_active_load() { + let left = worker("left"); + let right = worker("right"); + let _guard = left.load_guard(); + assert!(compare_prefill_engines(&left, &right, None).is_gt()); + } + + #[test] + fn decode_ranking_tie_is_not_broken_by_worker_id() { + let (a, z) = (worker("a"), worker("z")); + assert_eq!(compare_decode_engines(&a, &z, None), Ordering::Equal); + } + + #[test] + fn one_stale_candidate_makes_the_set_compare_by_local_load() { + let idle = worker("idle"); + let busy = worker("busy"); + let stale = worker("stale"); + let _idle_guards: Vec<_> = (0..5).map(|_| idle.load_guard()).collect(); + let _busy_guard = busy.load_guard(); + let snapshot = snapshot(&[(&idle, 0, 0), (&busy, 0, 1_000)]); + + let loads = CandidateLoads::new(Some(&snapshot), [&idle, &busy, &stale]); + assert!(loads.get(&idle.id).is_some()); + assert!(loads.get(&stale.id).is_none()); + assert!(loads.compare_prefill_engines(&idle, &busy).is_gt()); + } +} diff --git a/experimental/sgl-router/src/state/load_monitor/mod.rs b/experimental/sgl-router/src/state/load_monitor/mod.rs index 3d5aeee188..5c76445a6f 100644 --- a/experimental/sgl-router/src/state/load_monitor/mod.rs +++ b/experimental/sgl-router/src/state/load_monitor/mod.rs @@ -3,5 +3,6 @@ //! Load signals: engine-reported load and router-local in-flight accounting. +pub mod engine_ranking; pub mod engine_reported_load; pub mod router_inflight_load; diff --git a/experimental/sgl-router/tests/component/policies/cache_prefix_provider.rs b/experimental/sgl-router/tests/component/policies/cache_prefix_provider.rs index 45756b67f6..1d4458b89e 100644 --- a/experimental/sgl-router/tests/component/policies/cache_prefix_provider.rs +++ b/experimental/sgl-router/tests/component/policies/cache_prefix_provider.rs @@ -5,8 +5,9 @@ use std::collections::HashMap; use std::sync::Arc; use sgl_kv_indexer::PrefixOutcome; -use sgl_router::policies::prefix_provider::RadixTreePrefixProvider; -use sgl_router::state::kv_events::{compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId}; +use sgl_router::state::kv_events::{ + compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId, RadixTreePrefixProvider, +}; #[test] fn radix_tree_reports_contiguous_prefix_depth_per_worker() { diff --git a/experimental/sgl-router/tests/component/policies_reorg_cache_aware.rs b/experimental/sgl-router/tests/component/policies_reorg_cache_aware.rs index 4337aaf01d..060769797f 100644 --- a/experimental/sgl-router/tests/component/policies_reorg_cache_aware.rs +++ b/experimental/sgl-router/tests/component/policies_reorg_cache_aware.rs @@ -9,13 +9,13 @@ use sgl_kv_indexer::{PrefixIndex, PrefixIndexError, PrefixMatch, PrefixOutcome}; use sgl_router::buckets_reorg::{Bucket, BucketGroups, BucketResolver, EngineGroup}; use sgl_router::config::{AffinityConfig, AffinityMode}; use sgl_router::discovery::{ModelId, WorkerId, WorkerSpec}; -use sgl_router::policies::prefix_provider::RadixTreePrefixProvider; use sgl_router::policies_reorg::admission::{Decision, EngineAdmission, EngineMetrics}; use sgl_router::policies_reorg::cache_aware::{CacheAwarePolicy, CacheSource, PrefixMemo}; use sgl_router::policies_reorg::power_of_two::PowerOfTwoPolicy; use sgl_router::policies_reorg::{PickError, PickRequest, Policy, Stage}; use sgl_router::state::kv_events::{ compute_block_hashes, compute_block_hashes_bigram, BlockSizeOracle, HashTree, KvWorkerId, + RadixTreePrefixProvider, }; use sgl_router::state::load_monitor::engine_reported_load::{ EngineReportedLoadTable, LoadStat, NativeCacheRankLoad, diff --git a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs index afa779303c..377fde4a8a 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs @@ -633,10 +633,9 @@ async fn invalid_policy_signal_stops_bucket_iteration() { #[tokio::test] async fn cache_aware_routes_tokenized_prompt_and_rechecks_the_next_bucket() { use sgl_router::config::AffinityConfig; - use sgl_router::policies::prefix_provider::RadixTreePrefixProvider; use sgl_router::policies_reorg::cache_aware::{CacheAwarePolicy, CacheSource}; use sgl_router::state::kv_events::{ - compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId, + compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId, RadixTreePrefixProvider, }; use sgl_router::state::load_monitor::engine_reported_load::EngineReportedLoadTable; diff --git a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs index 8d94b4791c..1e333d2cfd 100644 --- a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs +++ b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs @@ -16,13 +16,14 @@ use sgl_router::config::{ }; use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; use sgl_router::policies::factory::build_registry; -use sgl_router::policies::prefix_provider::RadixTreePrefixProvider; use sgl_router::policies::PolicyRegistry; use sgl_router::policies_reorg::factory::build_resolver; use sgl_router::proxy::Proxy; use sgl_router::server::app::build_router; use sgl_router::server::app_context::{AppContext, ChatRouting}; -use sgl_router::state::kv_events::{BlockSizeOracle, HashTree, KvEventIndex}; +use sgl_router::state::kv_events::{ + BlockSizeOracle, HashTree, KvEventIndex, RadixTreePrefixProvider, +}; use sgl_router::tokenizer::TokenizerRegistry; use sgl_router::workers::WorkerRegistry; diff --git a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs index bbc2f21165..c3e23b71bf 100644 --- a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs +++ b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs @@ -12,12 +12,13 @@ use serde_json::json; use sgl_router::config::{Config, PolicyKind, StickyConfig, StickyFallbackKind}; use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; use sgl_router::policies::factory::build_registry; -use sgl_router::policies::prefix_provider::RadixTreePrefixProvider; use sgl_router::policies::request_tokens_for; use sgl_router::proxy::Proxy; use sgl_router::server::app::build_router; use sgl_router::server::app_context::AppContext; -use sgl_router::state::kv_events::{compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId}; +use sgl_router::state::kv_events::{ + compute_block_hashes, BlockSizeOracle, HashTree, KvWorkerId, RadixTreePrefixProvider, +}; use sgl_router::tokenizer::TokenizerRegistry; use sgl_router::workers::{EngineProfile, WireProtocol, WorkerRegistry}; use tower::ServiceExt;