[sgl-router] refactor - cache-aware policy (#40366)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
11ecdbf39f
commit
fcb080bd40
@@ -184,7 +184,8 @@ fn resolve_orders_all_length_fits_by_capacity_rank_and_id() {
|
||||
later,
|
||||
a,
|
||||
min,
|
||||
]);
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resolver
|
||||
.resolve(10, None)
|
||||
@@ -207,7 +208,7 @@ fn context_capacity_checks_peak_when_known_and_input_otherwise() {
|
||||
short.max_context_tokens = Some(20);
|
||||
let mut long = bucket("long", None, policy);
|
||||
long.max_context_tokens = Some(30);
|
||||
let resolver = BucketResolver::new(vec![long, short]);
|
||||
let resolver = BucketResolver::new(vec![long, short]).unwrap();
|
||||
assert_eq!(resolver.resolve(10, None).unwrap()[0].id, "short");
|
||||
assert_eq!(resolver.resolve(10, Some(20)).unwrap()[0].id, "short");
|
||||
assert_eq!(resolver.resolve(10, Some(21)).unwrap()[0].id, "long");
|
||||
@@ -237,9 +238,11 @@ async fn selected_pd_bucket_owns_both_memberships_and_policies() {
|
||||
prefill: group(&["p2", "d", "a"], prefill_policy.clone()),
|
||||
decode: group(&["d2", "p", "other"], decode_policy.clone()),
|
||||
},
|
||||
)]);
|
||||
)])
|
||||
.unwrap();
|
||||
let bucket = resolver.resolve(10, Some(20)).unwrap()[0];
|
||||
let request = BucketRequest {
|
||||
prefix: None,
|
||||
model: &model,
|
||||
input_tokens: 10,
|
||||
expected_peak_tokens: Some(20),
|
||||
@@ -267,7 +270,8 @@ async fn resolver_includes_empty_groups_without_invoking_policies() {
|
||||
min: None,
|
||||
max: Some(10),
|
||||
};
|
||||
let resolver = BucketResolver::new(vec![empty, bucket("available", Some(20), policy.clone())]);
|
||||
let resolver =
|
||||
BucketResolver::new(vec![empty, bucket("available", Some(20), policy.clone())]).unwrap();
|
||||
let buckets = resolver.resolve(10, None).unwrap();
|
||||
assert_eq!(
|
||||
buckets
|
||||
@@ -406,6 +410,7 @@ async fn bucket_scopes_plain_pick_and_preserves_request_facts() {
|
||||
BucketGroups::Plain(group(&["b"], Arc::new(InspectRequest))),
|
||||
);
|
||||
let request = BucketRequest {
|
||||
prefix: None,
|
||||
model: &model,
|
||||
input_tokens: 2,
|
||||
expected_peak_tokens: Some(12),
|
||||
|
||||
Reference in New Issue
Block a user