// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors // SPDX-License-Identifier: Apache-2.0 use std::sync::{Arc, Mutex}; use futures::future::BoxFuture; use sgl_router::buckets_reorg::{ Bucket, BucketGroups, BucketRequest, BucketResolver, EngineGroup, TokenLimits, }; use sgl_router::discovery::{ModelId, WorkerId, WorkerSpec}; use sgl_router::policies_reorg::admission::{ AdmissionLimits, Decision, EngineAdmission, EngineMetrics, }; use sgl_router::policies_reorg::{Pick, PickError, PickRequest, Policy, Rejection, Stage}; use sgl_router::workers::{Worker, WorkerRegistry}; #[derive(Debug)] struct TestPolicy { admission: Arc, result: Option>, miss: bool, invalid: bool, calls: Mutex>, } impl Default for TestPolicy { fn default() -> Self { Self { admission: Arc::new(AdmissionLimits::default()), result: None, miss: false, invalid: false, calls: Mutex::new(Vec::new()), } } } impl Policy for TestPolicy { fn pick<'a>( &'a self, engines: &'a [Arc], request: &'a PickRequest<'a>, ) -> BoxFuture<'a, Result> { Box::pin(async move { self.calls.lock().unwrap().push(request.bucket.to_owned()); if self.invalid { return Err(PickError::InvalidSignal("test signal".into())); } if self.miss { return Err(PickError::NoCandidates); } if engines.is_empty() { return Err(PickError::NoCandidates); } let engine = self.result.clone().unwrap_or_else(|| engines[0].clone()); if let Decision::Reject(reason) = self.admission.check(&engine, &EngineMetrics::default())? { return Err(PickError::AdmissionRejected(Rejection { engine: engine.id.clone(), reason, })); } Ok(Pick { engine, reason: "test", }) }) } } #[derive(Debug)] struct Reject(&'static str); impl EngineAdmission for Reject { fn check(&self, engine: &Worker, _: &EngineMetrics) -> Result { Ok(if engine.id.0 == self.0 { Decision::Reject("full".into()) } else { Decision::Allow }) } } fn spec(id: &str, mode: Stage, model: &str) -> WorkerSpec { WorkerSpec { id: WorkerId(id.into()), url: format!("http://{id}"), mode, model_ids: vec![ModelId(model.into())], bootstrap_port: None, } } fn registry() -> Arc { let workers = Arc::new(WorkerRegistry::default()); for (id, mode, model) in [ ("a", Stage::Plain, "m"), ("b", Stage::Plain, "m"), ("unhealthy", Stage::Plain, "m"), ("other", Stage::Plain, "other"), ("p", Stage::Prefill, "pd"), ("d", Stage::Decode, "pd"), ] { workers.add(spec(id, mode, model)).unwrap(); } let unhealthy = workers.get(&WorkerId("unhealthy".into())).unwrap(); for _ in 0..3 { unhealthy.breaker.record_failure(); } workers } fn group(members: &[&str], policy: Arc) -> EngineGroup { EngineGroup { worker_ids: Some(members.iter().map(|id| WorkerId((*id).into())).collect()), policy, } } fn bucket(id: &str, max: Option, policy: Arc) -> Bucket { let mut bucket = Bucket::new(id, BucketGroups::Plain(EngineGroup::new(policy))); bucket.limits.max = max; bucket } #[tokio::test] async fn groups_isolate_model_health_stage_and_membership() { let workers = registry(); let model = ModelId("m".into()); let request = PickRequest::new(&model, Stage::Plain, 10); let group = EngineGroup::new(Arc::new(TestPolicy::default())); assert_eq!( group.pick(&workers, &request).await.unwrap().engine.id.0, "a" ); workers.remove(&WorkerId("a".into())); assert_eq!( group.pick(&workers, &request).await.unwrap().engine.id.0, "b" ); workers.remove(&WorkerId("b".into())); assert!(matches!( group.pick(&workers, &request).await, Err(PickError::NoCandidates) )); let pd = ModelId("pd".into()); for (stage, expected) in [(Stage::Prefill, "p"), (Stage::Decode, "d")] { let request = PickRequest::new(&pd, stage, 10); let group = self::group( &["p", "d", "other", "unhealthy"], Arc::new(TestPolicy::default()), ); assert_eq!( group.pick(&workers, &request).await.unwrap().engine.id.0, expected ); } let empty = self::group(&[], Arc::new(TestPolicy::default())); assert!(matches!( empty.pick(&workers, &request).await, Err(PickError::NoCandidates) )); } #[test] fn resolve_orders_all_length_fits_by_capacity_rank_and_id() { let policy = Arc::new(TestPolicy::default()); let mut z = bucket("z", Some(20), policy.clone()); z.rank = 1; let mut a = bucket("a", Some(20), policy.clone()); a.rank = 1; let mut later = bucket("later", Some(20), policy.clone()); later.rank = 2; let mut min = bucket("min", Some(15), policy.clone()); min.limits.min = Some(11); let resolver = BucketResolver::new(vec![ bucket("catch-all", None, policy.clone()), bucket("too-small", Some(9), policy), z, later, a, min, ]) .unwrap(); assert_eq!( resolver .resolve(10, None, None, None) .unwrap() .iter() .map(|bucket| bucket.id.as_str()) .collect::>(), ["a", "z", "later", "catch-all"] ); assert_eq!(resolver.resolve(11, None, None, None).unwrap()[0].id, "min"); assert_eq!(resolver.resolve(15, None, None, None).unwrap()[0].id, "min"); assert_eq!(resolver.resolve(20, None, None, None).unwrap()[0].id, "a"); assert_eq!( resolver.resolve(21, None, None, None).unwrap()[0].id, "catch-all" ); } #[test] fn context_capacity_checks_peak_when_known_and_input_otherwise() { let policy = Arc::new(TestPolicy::default()); let mut short = bucket("short", None, policy.clone()); short.max_context_tokens = Some(20); let mut long = bucket("long", None, policy); long.max_context_tokens = Some(30); let resolver = BucketResolver::new(vec![long, short]).unwrap(); assert_eq!( resolver.resolve(10, None, None, None).unwrap()[0].id, "short" ); assert_eq!( resolver.resolve(10, Some(20), None, None).unwrap()[0].id, "short" ); assert_eq!( resolver.resolve(10, Some(21), None, None).unwrap()[0].id, "long" ); assert!(resolver .resolve(10, Some(31), None, None) .unwrap() .is_empty()); assert!(resolver.resolve(31, None, None, None).unwrap().is_empty()); assert!(matches!( resolver.resolve(10, Some(9), None, None), Err(PickError::InvalidSignal(_)) )); assert!(BucketResolver::default() .resolve(1, None, None, None) .unwrap() .is_empty()); } #[tokio::test] async fn selected_pd_bucket_owns_both_memberships_and_policies() { let workers = registry(); workers.add(spec("p2", Stage::Prefill, "pd")).unwrap(); workers.add(spec("d2", Stage::Decode, "pd")).unwrap(); let model = ModelId("pd".into()); let prefill_policy = Arc::new(TestPolicy::default()); let decode_policy = Arc::new(TestPolicy::default()); let resolver = BucketResolver::new(vec![Bucket::new( "shared", BucketGroups::Pd { prefill: group(&["p2", "d", "a"], prefill_policy.clone()), decode: group(&["d2", "p", "other"], decode_policy.clone()), }, )]) .unwrap(); let bucket = resolver.resolve(10, Some(20), None, None).unwrap()[0]; let request = BucketRequest { prefix: None, model: &model, input_tokens: 10, expected_peak_tokens: Some(20), token_ids: None, session_key: None, routing_key: None, }; let picks = bucket.pick_engines(&workers, &request).await.unwrap(); assert_eq!(picks.prefill.engine.id.0, "p2"); assert_eq!(picks.decode.unwrap().engine.id.0, "d2"); assert_eq!(*prefill_policy.calls.lock().unwrap(), ["shared"]); assert_eq!(*decode_policy.calls.lock().unwrap(), ["shared"]); } #[tokio::test] async fn resolver_includes_empty_groups_without_invoking_policies() { let workers = registry(); let model = ModelId("m".into()); let policy = Arc::new(TestPolicy::default()); let mut empty = Bucket::new( "empty", BucketGroups::Plain(group(&["missing"], policy.clone())), ); empty.limits = TokenLimits { min: None, max: Some(10), }; let resolver = BucketResolver::new(vec![empty, bucket("available", Some(20), policy.clone())]).unwrap(); let buckets = resolver.resolve(10, None, None, None).unwrap(); assert_eq!( buckets .iter() .map(|bucket| bucket.id.as_str()) .collect::>(), ["empty", "available"] ); let bucket = buckets[0]; let BucketGroups::Plain(group) = &bucket.groups else { panic!("expected plain") }; let request = PickRequest { bucket: &bucket.id, ..PickRequest::new(&model, Stage::Plain, 10) }; assert!(matches!( group.pick(&workers, &request).await, Err(PickError::NoCandidates) )); assert!(policy.calls.lock().unwrap().is_empty()); } #[tokio::test] async fn group_propagates_rejections_misses_and_invalid_signals() { let workers = registry(); let model = ModelId("m".into()); let request = PickRequest::new(&model, Stage::Plain, 10); let rejected = group( &["a"], Arc::new(TestPolicy { admission: Arc::new(Reject("a")), ..Default::default() }), ); assert!(matches!( rejected.pick(&workers, &request).await, Err(PickError::AdmissionRejected(_)) )); let miss = group( &["a"], Arc::new(TestPolicy { miss: true, ..Default::default() }), ); assert!(matches!( miss.pick(&workers, &request).await, Err(PickError::NoCandidates) )); let invalid = group( &["a"], Arc::new(TestPolicy { invalid: true, ..Default::default() }), ); assert!(matches!( invalid.pick(&workers, &request).await, Err(PickError::InvalidSignal(_)) )); } #[tokio::test] async fn group_rejects_foreign_pick_even_with_same_worker_id() { let workers = registry(); let model = ModelId("m".into()); let request = PickRequest::new(&model, Stage::Plain, 10); let group = group( &["a"], Arc::new(TestPolicy { result: Some(Arc::new(Worker::new(spec("a", Stage::Plain, "m")))), ..Default::default() }), ); assert!(matches!( group.pick(&workers, &request).await, Err(PickError::OutsideCandidates(_)) )); } #[tokio::test] async fn selected_engine_rejection_does_not_try_an_alternative() { let model = ModelId("m".into()); let request = PickRequest::new(&model, Stage::Plain, 10); let engines = [ Arc::new(Worker::new(spec("a", Stage::Plain, "m"))), Arc::new(Worker::new(spec("b", Stage::Plain, "m"))), ]; let policy = TestPolicy { admission: Arc::new(Reject("a")), ..Default::default() }; assert!(matches!( policy.pick(&engines, &request).await, Err(PickError::AdmissionRejected(_)) )); assert!(matches!( policy.pick(&[], &request).await, Err(PickError::NoCandidates) )); } #[tokio::test] async fn bucket_scopes_plain_pick_and_preserves_request_facts() { #[derive(Debug)] struct InspectRequest; impl Policy for InspectRequest { fn pick<'a>( &'a self, engines: &'a [Arc], request: &'a PickRequest<'a>, ) -> BoxFuture<'a, Result> { Box::pin(async move { assert_eq!(request.model.0, "m"); assert_eq!(request.bucket, "plain-bucket"); assert_eq!(request.stage, Stage::Plain); assert_eq!(request.input_tokens, 2); assert_eq!(request.expected_peak_tokens, Some(12)); assert_eq!(request.token_ids, Some([7, 9].as_slice())); assert_eq!(request.session_key, Some("session")); assert_eq!(request.routing_key, Some("routing")); Ok(Pick { engine: engines[0].clone(), reason: "inspected", }) }) } } let workers = registry(); let model = ModelId("m".into()); let bucket = Bucket::new( "plain-bucket", BucketGroups::Plain(group(&["b"], Arc::new(InspectRequest))), ); let request = BucketRequest { prefix: None, model: &model, input_tokens: 2, expected_peak_tokens: Some(12), token_ids: Some(&[7, 9]), session_key: Some("session"), routing_key: Some("routing"), }; let picks = bucket.pick_engines(&workers, &request).await.unwrap(); assert_eq!(picks.prefill.engine.id.0, "b"); assert_eq!(picks.prefill.reason, "inspected"); assert!(picks.decode.is_none()); } #[tokio::test] async fn power_of_two_checks_selected_engine_and_propagates_rejection_without_fallback() { use sgl_router::policies_reorg::power_of_two::PowerOfTwoPolicy; use sgl_router::state::load_monitor::engine_reported_load::EngineReportedLoadTable; #[derive(Debug)] struct Check { calls: Mutex>, reject: bool, invalid: bool, } impl EngineAdmission for Check { fn check(&self, engine: &Worker, _: &EngineMetrics) -> Result { self.calls.lock().unwrap().push(engine.id.clone()); if self.invalid { Err(PickError::InvalidSignal("admission input".into())) } else if self.reject { Ok(Decision::Reject("full".into())) } else { Ok(Decision::Allow) } } } let model = ModelId("m".into()); let request = PickRequest::new(&model, Stage::Plain, 10); let engine = Arc::new(Worker::new(spec("a", Stage::Plain, "m"))); let other = Arc::new(Worker::new(spec("b", Stage::Plain, "m"))); let _busy = other.load_guard(); for engines in [vec![engine.clone()], vec![other.clone(), engine.clone()]] { for (reject, invalid) in [(false, false), (true, false), (false, true)] { let check = Arc::new(Check { calls: Mutex::new(Vec::new()), reject, invalid, }); let mut policy = PowerOfTwoPolicy::new(EngineReportedLoadTable::new()); policy.admission = check.clone(); assert!(matches!( policy.pick(&[], &request).await, Err(PickError::NoCandidates) )); assert!(check.calls.lock().unwrap().is_empty()); let result = policy.pick(&engines, &request).await; if invalid { assert!(matches!(result, Err(PickError::InvalidSignal(_)))); } else if reject { assert!(matches!(result, Err(PickError::AdmissionRejected(reason)) if reason.engine == engine.id && reason.reason == "full")); } else { assert!(Arc::ptr_eq(&result.unwrap().engine, &engine)); } assert_eq!( check.calls.lock().unwrap().as_slice(), std::slice::from_ref(&engine.id) ); } } }