[sgl-router] refactor - SLO ordering for bucket selection (#40292)
This commit is contained in:
@@ -186,17 +186,20 @@ fn resolve_orders_all_length_fits_by_capacity_rank_and_id() {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resolver
|
||||
.resolve(10, None)
|
||||
.resolve(10, None, None, None)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|bucket| bucket.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["a", "z", "later", "catch-all"]
|
||||
);
|
||||
assert_eq!(resolver.resolve(11, None).unwrap()[0].id, "min");
|
||||
assert_eq!(resolver.resolve(15, None).unwrap()[0].id, "min");
|
||||
assert_eq!(resolver.resolve(20, None).unwrap()[0].id, "a");
|
||||
assert_eq!(resolver.resolve(21, None).unwrap()[0].id, "catch-all");
|
||||
assert_eq!(resolver.resolve(11, None, None, None).unwrap()[0].id, "min");
|
||||
assert_eq!(resolver.resolve(15, None, None, None).unwrap()[0].id, "min");
|
||||
assert_eq!(resolver.resolve(20, None, None, None).unwrap()[0].id, "a");
|
||||
assert_eq!(
|
||||
resolver.resolve(21, None, None, None).unwrap()[0].id,
|
||||
"catch-all"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -207,17 +210,29 @@ fn context_capacity_checks_peak_when_known_and_input_otherwise() {
|
||||
let mut long = bucket("long", None, policy);
|
||||
long.max_context_tokens = Some(30);
|
||||
let resolver = BucketResolver::new(vec![long, short]).unwrap();
|
||||
assert_eq!(resolver.resolve(10, None).unwrap()[0].id, "short");
|
||||
assert_eq!(resolver.resolve(10, Some(20)).unwrap()[0].id, "short");
|
||||
assert_eq!(resolver.resolve(10, Some(21)).unwrap()[0].id, "long");
|
||||
assert!(resolver.resolve(10, Some(31)).unwrap().is_empty());
|
||||
assert!(resolver.resolve(31, None).unwrap().is_empty());
|
||||
assert_eq!(
|
||||
resolver.resolve(10, None, None, None).unwrap()[0].id,
|
||||
"short"
|
||||
);
|
||||
assert_eq!(
|
||||
resolver.resolve(10, Some(20), None, None).unwrap()[0].id,
|
||||
"short"
|
||||
);
|
||||
assert_eq!(
|
||||
resolver.resolve(10, Some(21), None, None).unwrap()[0].id,
|
||||
"long"
|
||||
);
|
||||
assert!(resolver
|
||||
.resolve(10, Some(31), None, None)
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
assert!(resolver.resolve(31, None, None, None).unwrap().is_empty());
|
||||
assert!(matches!(
|
||||
resolver.resolve(10, Some(9)),
|
||||
resolver.resolve(10, Some(9), None, None),
|
||||
Err(PickError::InvalidSignal(_))
|
||||
));
|
||||
assert!(BucketResolver::default()
|
||||
.resolve(1, None)
|
||||
.resolve(1, None, None, None)
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
}
|
||||
@@ -238,7 +253,7 @@ async fn selected_pd_bucket_owns_both_memberships_and_policies() {
|
||||
},
|
||||
)])
|
||||
.unwrap();
|
||||
let bucket = resolver.resolve(10, Some(20)).unwrap()[0];
|
||||
let bucket = resolver.resolve(10, Some(20), None, None).unwrap()[0];
|
||||
let request = BucketRequest {
|
||||
prefix: None,
|
||||
model: &model,
|
||||
@@ -270,7 +285,7 @@ async fn resolver_includes_empty_groups_without_invoking_policies() {
|
||||
};
|
||||
let resolver =
|
||||
BucketResolver::new(vec![empty, bucket("available", Some(20), policy.clone())]).unwrap();
|
||||
let buckets = resolver.resolve(10, None).unwrap();
|
||||
let buckets = resolver.resolve(10, None, None, None).unwrap();
|
||||
assert_eq!(
|
||||
buckets
|
||||
.iter()
|
||||
|
||||
Reference in New Issue
Block a user