[sgl-router] refactor - SLO ordering for bucket selection (#40292)

This commit is contained in:
Kan Wu
2026-09-22 11:16:10 +08:00
committed by GitHub
parent 27f796ca6c
commit 59a723ef1e
8 changed files with 422 additions and 37 deletions
@@ -186,17 +186,20 @@ fn resolve_orders_all_length_fits_by_capacity_rank_and_id() {
.unwrap();
assert_eq!(
resolver
.resolve(10, None)
.resolve(10, None, None, None)
.unwrap()
.iter()
.map(|bucket| bucket.id.as_str())
.collect::<Vec<_>>(),
["a", "z", "later", "catch-all"]
);
assert_eq!(resolver.resolve(11, None).unwrap()[0].id, "min");
assert_eq!(resolver.resolve(15, None).unwrap()[0].id, "min");
assert_eq!(resolver.resolve(20, None).unwrap()[0].id, "a");
assert_eq!(resolver.resolve(21, None).unwrap()[0].id, "catch-all");
assert_eq!(resolver.resolve(11, None, None, None).unwrap()[0].id, "min");
assert_eq!(resolver.resolve(15, None, None, None).unwrap()[0].id, "min");
assert_eq!(resolver.resolve(20, None, None, None).unwrap()[0].id, "a");
assert_eq!(
resolver.resolve(21, None, None, None).unwrap()[0].id,
"catch-all"
);
}
#[test]
@@ -207,17 +210,29 @@ fn context_capacity_checks_peak_when_known_and_input_otherwise() {
let mut long = bucket("long", None, policy);
long.max_context_tokens = Some(30);
let resolver = BucketResolver::new(vec![long, short]).unwrap();
assert_eq!(resolver.resolve(10, None).unwrap()[0].id, "short");
assert_eq!(resolver.resolve(10, Some(20)).unwrap()[0].id, "short");
assert_eq!(resolver.resolve(10, Some(21)).unwrap()[0].id, "long");
assert!(resolver.resolve(10, Some(31)).unwrap().is_empty());
assert!(resolver.resolve(31, None).unwrap().is_empty());
assert_eq!(
resolver.resolve(10, None, None, None).unwrap()[0].id,
"short"
);
assert_eq!(
resolver.resolve(10, Some(20), None, None).unwrap()[0].id,
"short"
);
assert_eq!(
resolver.resolve(10, Some(21), None, None).unwrap()[0].id,
"long"
);
assert!(resolver
.resolve(10, Some(31), None, None)
.unwrap()
.is_empty());
assert!(resolver.resolve(31, None, None, None).unwrap().is_empty());
assert!(matches!(
resolver.resolve(10, Some(9)),
resolver.resolve(10, Some(9), None, None),
Err(PickError::InvalidSignal(_))
));
assert!(BucketResolver::default()
.resolve(1, None)
.resolve(1, None, None, None)
.unwrap()
.is_empty());
}
@@ -238,7 +253,7 @@ async fn selected_pd_bucket_owns_both_memberships_and_policies() {
},
)])
.unwrap();
let bucket = resolver.resolve(10, Some(20)).unwrap()[0];
let bucket = resolver.resolve(10, Some(20), None, None).unwrap()[0];
let request = BucketRequest {
prefix: None,
model: &model,
@@ -270,7 +285,7 @@ async fn resolver_includes_empty_groups_without_invoking_policies() {
};
let resolver =
BucketResolver::new(vec![empty, bucket("available", Some(20), policy.clone())]).unwrap();
let buckets = resolver.resolve(10, None).unwrap();
let buckets = resolver.resolve(10, None, None, None).unwrap();
assert_eq!(
buckets
.iter()