[sgl-router] refactor - cache-aware policy (#40366)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-09-21 13:53:57 +08:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 11ecdbf39f
commit fcb080bd40
9 changed files with 1168 additions and 14 deletions
@@ -184,7 +184,8 @@ fn resolve_orders_all_length_fits_by_capacity_rank_and_id() {
later,
a,
min,
]);
])
.unwrap();
assert_eq!(
resolver
.resolve(10, None)
@@ -207,7 +208,7 @@ fn context_capacity_checks_peak_when_known_and_input_otherwise() {
short.max_context_tokens = Some(20);
let mut long = bucket("long", None, policy);
long.max_context_tokens = Some(30);
let resolver = BucketResolver::new(vec![long, short]);
let resolver = BucketResolver::new(vec![long, short]).unwrap();
assert_eq!(resolver.resolve(10, None).unwrap()[0].id, "short");
assert_eq!(resolver.resolve(10, Some(20)).unwrap()[0].id, "short");
assert_eq!(resolver.resolve(10, Some(21)).unwrap()[0].id, "long");
@@ -237,9 +238,11 @@ async fn selected_pd_bucket_owns_both_memberships_and_policies() {
prefill: group(&["p2", "d", "a"], prefill_policy.clone()),
decode: group(&["d2", "p", "other"], decode_policy.clone()),
},
)]);
)])
.unwrap();
let bucket = resolver.resolve(10, Some(20)).unwrap()[0];
let request = BucketRequest {
prefix: None,
model: &model,
input_tokens: 10,
expected_peak_tokens: Some(20),
@@ -267,7 +270,8 @@ async fn resolver_includes_empty_groups_without_invoking_policies() {
min: None,
max: Some(10),
};
let resolver = BucketResolver::new(vec![empty, bucket("available", Some(20), policy.clone())]);
let resolver =
BucketResolver::new(vec![empty, bucket("available", Some(20), policy.clone())]).unwrap();
let buckets = resolver.resolve(10, None).unwrap();
assert_eq!(
buckets
@@ -406,6 +410,7 @@ async fn bucket_scopes_plain_pick_and_preserves_request_facts() {
BucketGroups::Plain(group(&["b"], Arc::new(InspectRequest))),
);
let request = BucketRequest {
prefix: None,
model: &model,
input_tokens: 2,
expected_peak_tokens: Some(12),