Files

593 lines
21 KiB
Rust

// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
use crate::config::{
Config, FilterKind, ModelConfig, PolicyKind, ScoreTermKind, StickyFallbackKind,
};
use crate::discovery::ModelId;
use crate::policies::{
cache_aware::CacheAwarePolicy,
load_based::LoadBasedPolicy,
power_of_two::PowerOfTwoChoicesPolicy,
random::RandomPolicy,
round_robin::RoundRobinPolicy,
scoring::{
admission::Overloaded, prefix_cache, prefix_cache::PrefixCachePolicy, FusedScorePolicy,
Pipeline, ScorePolicy,
},
session_aware::SessionAwarePolicy,
sticky::StickyPolicy,
Policy, PolicyRegistry,
};
use crate::state::kv_events::{BlockSizeOracle, HashTree};
use anyhow::{anyhow, Result};
use std::sync::Arc;
use std::time::Duration;
/// Build a dependency-free policy for keyless sticky requests and new pins.
fn build_sticky_fallback(kind: StickyFallbackKind) -> Arc<dyn Policy> {
match kind {
StickyFallbackKind::RoundRobin => Arc::new(RoundRobinPolicy::new()),
StickyFallbackKind::Random => Arc::new(RandomPolicy::new()),
StickyFallbackKind::PowerOfTwo => Arc::new(PowerOfTwoChoicesPolicy::new()),
StickyFallbackKind::LoadBased => Arc::new(LoadBasedPolicy::new()),
}
}
/// Construct a [`StickyPolicy`] from a model's `sticky` config (or
/// defaults). Shared by `build_policy` and the test shim so the duration
/// conversion + fallback wiring live in one place.
fn build_sticky(model: &ModelConfig) -> Arc<dyn Policy> {
let s = model.sticky.clone().unwrap_or_default();
Arc::new(StickyPolicy::new(
Duration::from_secs(s.idle_secs),
Duration::from_secs(s.eviction_interval_secs),
build_sticky_fallback(s.fallback_policy),
))
}
/// Constructs a policy for one model.
pub fn build_policy(
model: &ModelConfig,
tree: Arc<HashTree>,
block_size_oracle: Arc<BlockSizeOracle>,
) -> Result<Arc<dyn Policy>> {
validate_eligibility(model)?;
let inner = build_kind(model.policy, model, &tree, &block_size_oracle)?;
let Some(elig) = model.eligibility.as_ref().filter(|e| !e.filters.is_empty()) else {
return Ok(inner);
};
let mut filters = Vec::with_capacity(elig.filters.len());
for &kind in &elig.filters {
filters.push(build_filter(kind, model, &tree, &block_size_oracle));
}
Ok(Arc::new(Pipeline::new(filters, inner)?))
}
/// Reject configurations that can bypass CLI validation when constructed in code.
fn validate_eligibility(model: &ModelConfig) -> Result<()> {
let Some(eligibility) = model.eligibility.as_ref().filter(|e| !e.filters.is_empty()) else {
return Ok(());
};
if model.policy == PolicyKind::Sticky {
return Err(anyhow!(
"eligibility filters cannot be combined with sticky policy"
));
}
if eligibility.filters.contains(&FilterKind::Overloaded)
&& eligibility.max_in_flight.unwrap_or(0) == 0
{
return Err(anyhow!("max_in_flight must be greater than 0"));
}
Ok(())
}
/// Build one policy kind with the shared model dependencies.
fn build_kind(
kind: PolicyKind,
model: &ModelConfig,
tree: &Arc<HashTree>,
block_size_oracle: &Arc<BlockSizeOracle>,
) -> Result<Arc<dyn Policy>> {
let (tree, block_size_oracle) = (Arc::clone(tree), Arc::clone(block_size_oracle));
Ok(match kind {
PolicyKind::RoundRobin => Arc::new(RoundRobinPolicy::new()),
PolicyKind::Random => Arc::new(RandomPolicy::new()),
PolicyKind::PowerOfTwo => Arc::new(PowerOfTwoChoicesPolicy::new()),
PolicyKind::LoadBased => Arc::new(LoadBasedPolicy::new()),
PolicyKind::SessionAware => Arc::new(SessionAwarePolicy::new(
model.affinity.clone().unwrap_or_default(),
)),
PolicyKind::CacheAware => Arc::new(CacheAwarePolicy::new(
model.affinity.clone().unwrap_or_default(),
)),
PolicyKind::Sticky => build_sticky(model),
PolicyKind::FusedScore => build_fused(model, &tree, &block_size_oracle)?,
PolicyKind::ScorePolicy => build_score_policy(model, &tree, &block_size_oracle)?,
})
}
/// Builds one hard admission filter with the shared model dependencies.
fn build_filter(
kind: FilterKind,
model: &ModelConfig,
tree: &Arc<HashTree>,
block_size_oracle: &Arc<BlockSizeOracle>,
) -> Arc<dyn Policy> {
match kind {
FilterKind::Overloaded => {
let cap = (model.eligibility.as_ref())
.and_then(|e| e.max_in_flight)
.unwrap_or(usize::MAX);
Arc::new(Overloaded::new(cap))
}
FilterKind::PrefixCache => {
let share = (model.eligibility.as_ref())
.and_then(|e| e.min_prefix_share)
.unwrap_or(0.0);
Arc::new(
PrefixCachePolicy::new(
Arc::clone(tree),
Arc::clone(block_size_oracle),
prefix_cache::DEFAULT_WEIGHT,
)
.with_min_share(share),
)
}
}
}
/// Builds one soft scoring term for `--policy fused_score`.
fn build_score(
kind: ScoreTermKind,
tree: &Arc<HashTree>,
block_size_oracle: &Arc<BlockSizeOracle>,
) -> Arc<dyn Policy> {
match kind {
ScoreTermKind::Random => Arc::new(RandomPolicy::new()),
ScoreTermKind::LoadBased => Arc::new(LoadBasedPolicy::new()),
ScoreTermKind::PrefixCache => Arc::new(PrefixCachePolicy::new(
Arc::clone(tree),
Arc::clone(block_size_oracle),
prefix_cache::DEFAULT_WEIGHT,
)),
}
}
/// Builds `--policy fused_score`.
fn build_fused(
model: &ModelConfig,
tree: &Arc<HashTree>,
oracle: &Arc<BlockSizeOracle>,
) -> Result<Arc<dyn Policy>> {
let spec = model.fused.as_deref().unwrap_or_default();
if spec.is_empty() {
return Err(anyhow!(
"--policy fused_score needs at least one --fuse term"
));
}
let mut terms: Vec<(Arc<dyn Policy>, Option<f32>)> = Vec::with_capacity(spec.len());
for t in spec {
terms.push((build_score(t.kind, tree, oracle), t.weight));
}
Ok(Arc::new(FusedScorePolicy::new(terms)?))
}
/// Builds the top-level `score_policy`.
fn build_score_policy(
model: &ModelConfig,
tree: &Arc<HashTree>,
oracle: &Arc<BlockSizeOracle>,
) -> Result<Arc<dyn Policy>> {
Ok(Arc::new(ScorePolicy::new(build_fused(
model, tree, oracle,
)?)))
}
/// Builds a policy with test defaults.
#[cfg(test)]
pub fn build_policy_kind_only(kind: PolicyKind) -> Result<Arc<dyn Policy>> {
Ok(match kind {
PolicyKind::RoundRobin => Arc::new(RoundRobinPolicy::new()),
PolicyKind::Random => Arc::new(RandomPolicy::new()),
PolicyKind::PowerOfTwo => Arc::new(PowerOfTwoChoicesPolicy::new()),
PolicyKind::LoadBased => Arc::new(LoadBasedPolicy::new()),
PolicyKind::SessionAware => Arc::new(SessionAwarePolicy::new(
crate::config::AffinityConfig::default(),
)),
PolicyKind::CacheAware => Arc::new(CacheAwarePolicy::new(
crate::config::AffinityConfig::default(),
)),
PolicyKind::Sticky => {
let s = crate::config::StickyConfig::default();
Arc::new(StickyPolicy::new(
Duration::from_secs(s.idle_secs),
Duration::from_secs(s.eviction_interval_secs),
build_sticky_fallback(s.fallback_policy),
))
}
PolicyKind::FusedScore | PolicyKind::ScorePolicy => {
return Err(anyhow!("--policy {kind} needs --fuse terms from the model"))
}
})
}
pub fn build_registry(
cfg: &Config,
tree: Arc<HashTree>,
block_size_oracle: Arc<BlockSizeOracle>,
) -> Result<PolicyRegistry> {
let reg = PolicyRegistry::default();
let m = &cfg.model;
reg.insert(
ModelId(m.id.clone()),
build_policy(m, Arc::clone(&tree), Arc::clone(&block_size_oracle))?,
);
Ok(reg)
}
/// Builds a registry with empty cache-aware dependencies.
pub fn build_registry_with_defaults(cfg: &Config) -> Result<PolicyRegistry> {
build_registry(cfg, Arc::new(HashTree::new()), BlockSizeOracle::new())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::{
Config, DiscoveryBackend, InflightLoadConfig, ModelConfig, ProxyConfig, ServerConfig,
StaticUrlsDiscoveryConfig,
};
use crate::config::{
EligibilityConfig, FilterKind, PolicyKind, ScoreTermKind, StickyFallbackKind,
};
use crate::discovery::{WorkerId, WorkerMode, WorkerSpec};
use crate::policies::SelectionContext;
use crate::workers::Worker;
fn worker(id: &str) -> Arc<Worker> {
Arc::new(Worker::new(WorkerSpec {
id: WorkerId(id.into()),
url: format!("http://{id}:30000"),
mode: WorkerMode::Plain,
model_ids: vec![ModelId("modelA".into())],
..Default::default()
}))
}
#[test]
fn filter_overloaded_wires_through_the_factory() {
let mut cfg = cfg_with_model("modelA", PolicyKind::LoadBased);
cfg.model.eligibility = Some(EligibilityConfig {
filters: vec![FilterKind::Overloaded],
max_in_flight: Some(2),
min_prefix_share: None,
});
let reg = build_registry_with_defaults(&cfg).unwrap();
let p = reg.get(&ModelId("modelA".into())).unwrap();
let ws = vec![worker("w0"), worker("w1")];
let model = ModelId("modelA".into());
let ctx = SelectionContext::new(&model, None);
let _one = ws[1].load_guard();
assert_eq!(
p.select(&ws, &ctx).unwrap().id,
ws[0].id,
"load still ranks"
);
let _fill: Vec<_> = (ws.iter())
.flat_map(|w| (0..2).map(|_| w.load_guard()))
.collect();
assert!(
p.select(&ws, &ctx).is_none(),
"every worker is over the cap"
);
}
#[test]
fn a_filter_must_actually_constrain() {
let mut cfg = cfg_with_model("modelA", PolicyKind::LoadBased);
let built = |c: &Config| {
build_registry_with_defaults(c).map(|r| r.get(&ModelId("modelA".into())).unwrap())
};
assert!(
!built(&cfg).unwrap().can_filter(),
"no floor, so the term stays a pure preference",
);
cfg.model.eligibility = Some(EligibilityConfig {
filters: vec![FilterKind::PrefixCache],
max_in_flight: None,
min_prefix_share: Some(0.6),
});
assert!(
built(&cfg).unwrap().needs_request_tokens(),
"the floor reads the prompt"
);
cfg.model.eligibility = Some(EligibilityConfig {
filters: vec![FilterKind::PrefixCache],
max_in_flight: None,
min_prefix_share: Some(0.6),
});
let filter = build_filter(
FilterKind::PrefixCache,
&cfg.model,
&Arc::new(HashTree::new()),
&BlockSizeOracle::new(),
);
assert!(
filter.can_filter(),
"a configured prefix-cache floor is a filter",
);
}
fn cfg_with_model(id: &str, policy: PolicyKind) -> Config {
Config {
server: ServerConfig {
host: "0".into(),
port: 0,
..Default::default()
},
observability: Default::default(),
model: ModelConfig {
id: id.into(),
tokenizer_path: Some("/tmp/x".into()),
disable_input_ids_forwarding: false,
tokenizer: Default::default(),
policy,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
circuit_breaker: None,
cache_aware: None,
sticky: None,
affinity: None,
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
}),
proxy: ProxyConfig::default(),
router_inflight_load: InflightLoadConfig::default(),
}
}
#[test]
fn only_prefix_reading_policies_need_request_tokens() {
fn fuse(model: &mut ModelConfig, kinds: &[ScoreTermKind]) {
model.policy = PolicyKind::FusedScore;
model.fused = Some(
(kinds.iter())
.map(|&kind| crate::config::FusedTerm { kind, weight: None })
.collect(),
);
}
type Mutate = fn(&mut ModelConfig);
let cases: [(Mutate, bool); 8] = [
(|_| {}, false),
(|m| m.policy = PolicyKind::PowerOfTwo, false),
(|m| m.policy = PolicyKind::SessionAware, false),
(|m| m.policy = PolicyKind::CacheAware, true),
(|m| fuse(m, &crate::config::DEFAULT_FUSE), true),
(|m| fuse(m, &[ScoreTermKind::LoadBased]), false),
(|m| fuse(m, &[ScoreTermKind::PrefixCache]), true),
(
|m| {
m.eligibility = Some(EligibilityConfig {
filters: vec![FilterKind::PrefixCache],
min_prefix_share: Some(0.6),
..Default::default()
})
},
true,
),
];
for (i, (mutate, needs)) in cases.into_iter().enumerate() {
let mut cfg = cfg_with_model("modelA", PolicyKind::RoundRobin);
mutate(&mut cfg.model);
let registry = build_registry_with_defaults(&cfg).unwrap();
assert_eq!(registry.needs_request_tokens(), needs, "case {i}");
}
}
#[test]
fn build_policy_kind_only_covers_all_variants() {
for (kind, needs_load_snapshot, needs_dispatch_timestamps) in [
(PolicyKind::RoundRobin, false, false),
(PolicyKind::Random, false, false),
(PolicyKind::PowerOfTwo, true, false),
(PolicyKind::LoadBased, true, true),
(PolicyKind::SessionAware, true, false),
(PolicyKind::CacheAware, true, false),
(PolicyKind::Sticky, false, false),
] {
let policy = build_policy_kind_only(kind).unwrap();
assert_eq!(
policy.needs_load_snapshot(),
needs_load_snapshot,
"{kind:?}"
);
assert_eq!(
policy.needs_dispatch_timestamps(),
needs_dispatch_timestamps,
"{kind:?}"
);
}
assert!(build_policy_kind_only(PolicyKind::FusedScore).is_err());
assert!(build_policy_kind_only(PolicyKind::ScorePolicy).is_err());
}
#[test]
fn prefix_cache_builds_as_a_score_term() {
let p = build_score(
ScoreTermKind::PrefixCache,
&Arc::new(HashTree::new()),
&BlockSizeOracle::new(),
);
assert!(p.can_fuse(), "prefix_cache must be usable as a --fuse term");
assert!(!p.needs_load_snapshot());
}
#[test]
fn fused_score_builds_score_terms_and_rejects_an_empty_config() {
let mut cfg = cfg_with_model("m", PolicyKind::FusedScore);
let term = |kind, weight| crate::config::FusedTerm { kind, weight };
for weight in [None, Some(2.5)] {
cfg.model.fused = Some(vec![term(ScoreTermKind::PrefixCache, weight)]);
assert!(build_registry_with_defaults(&cfg).is_ok(), "{weight:?}");
}
cfg.model.fused = Some(vec![]);
assert!(build_registry_with_defaults(&cfg)
.unwrap_err()
.to_string()
.contains("at least one --fuse term"));
}
/// `score_policy` uses its own top-level factory branch.
#[test]
fn score_policy_builds_via_its_own_factory_branch() {
let mut cfg = cfg_with_model("modelA", PolicyKind::ScorePolicy);
cfg.model.fused = Some(vec![crate::config::FusedTerm {
kind: ScoreTermKind::PrefixCache,
weight: Some(2.0),
}]);
let registry = build_registry_with_defaults(&cfg).expect("score policy builds");
let policy = registry
.get(&ModelId("modelA".into()))
.expect("configured model has a policy");
assert!(
policy.can_fuse(),
"the score policy exposes score semantics"
);
assert!(
policy.uses_shared_prefill_admission(),
"top-level score_policy participates in the shared hard admission layer"
);
}
#[test]
fn score_policy_with_filter_builds_one_outer_pipeline() {
let mut cfg = cfg_with_model("modelA", PolicyKind::ScorePolicy);
cfg.model.fused = Some(vec![crate::config::FusedTerm {
kind: ScoreTermKind::LoadBased,
weight: Some(1.0),
}]);
cfg.model.eligibility = Some(EligibilityConfig {
filters: vec![FilterKind::Overloaded],
max_in_flight: Some(2),
min_prefix_share: None,
});
let registry = build_registry_with_defaults(&cfg)
.expect("a score policy keeps eligibility outside its scoring terms");
let policy = registry
.get(&ModelId("modelA".into()))
.expect("configured model has a policy");
assert!(policy.uses_shared_prefill_admission());
}
#[test]
fn factory_rejects_missing_or_zero_overloaded_capacity() {
for max_in_flight in [None, Some(0)] {
let mut cfg = cfg_with_model("modelA", PolicyKind::FusedScore);
cfg.model.fused = Some(vec![crate::config::FusedTerm {
kind: ScoreTermKind::LoadBased,
weight: None,
}]);
cfg.model.eligibility = Some(EligibilityConfig {
filters: vec![FilterKind::Overloaded],
max_in_flight,
min_prefix_share: None,
});
let error = build_registry_with_defaults(&cfg)
.expect_err("overloaded capacity must be positive at construction");
let message = error.to_string();
assert!(message.contains("max_in_flight must be greater than 0"));
}
}
#[test]
fn factory_rejects_sticky_eligibility_pipeline() {
let mut cfg = cfg_with_model("modelA", PolicyKind::Sticky);
cfg.model.eligibility = Some(EligibilityConfig {
filters: vec![FilterKind::Overloaded],
max_in_flight: Some(2),
min_prefix_share: None,
});
let error = build_registry_with_defaults(&cfg)
.expect_err("sticky assignments cannot be wrapped by eligibility filters");
let message = error.to_string();
assert!(message.contains("eligibility filters cannot be combined with sticky"));
}
#[test]
fn registry_assigns_configured_model() {
let cfg = cfg_with_model("qwen", PolicyKind::RoundRobin);
let tree = Arc::new(HashTree::new());
let reg = build_registry(&cfg, tree, BlockSizeOracle::new()).unwrap();
assert!(reg.get(&ModelId("qwen".into())).is_some());
assert!(reg.get(&ModelId("missing".into())).is_none());
}
#[test]
fn load_based_builds_via_factory() {
let cfg = cfg_with_model("modelA", PolicyKind::LoadBased);
let tree = Arc::new(HashTree::new());
let reg = build_registry(&cfg, tree, BlockSizeOracle::new()).unwrap();
let p = reg.get(&ModelId("modelA".into())).unwrap();
let dbg = format!("{p:?}");
assert!(
dbg.contains("LoadBasedPolicy"),
"expected LoadBasedPolicy debug repr, got: {dbg}",
);
assert!(p.needs_load_snapshot());
}
#[test]
fn sticky_builds_via_factory() {
let cfg = cfg_with_model("modelA", PolicyKind::Sticky);
let tree = Arc::new(HashTree::new());
let reg = build_registry(&cfg, tree, BlockSizeOracle::new()).unwrap();
let p = reg.get(&ModelId("modelA".into())).unwrap();
let dbg = format!("{p:?}");
assert!(
dbg.contains("StickyPolicy"),
"expected StickyPolicy debug repr, got: {dbg}",
);
}
#[test]
fn sticky_fallback_builder_covers_all_typed_choices() {
let workers = vec![worker("w0"), worker("w1")];
let model = ModelId("modelA".into());
let ctx = SelectionContext::new(&model, None);
for kind in [
StickyFallbackKind::RoundRobin,
StickyFallbackKind::Random,
StickyFallbackKind::PowerOfTwo,
StickyFallbackKind::LoadBased,
] {
assert!(
build_sticky_fallback(kind).select(&workers, &ctx).is_some(),
"{kind:?}"
);
}
}
}