[Router] Add --worker-queue-limit: stop sending cache-affinity traffic to a queueing worker (#39168)
Co-authored-by: Kangyan Zhou <kangyan.zhou@radixark.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Kangyan Zhou
Claude Opus 5
parent
3f8eb35ead
commit
c1f5b4736a
@@ -154,6 +154,7 @@ fn cache_candidate_uses_bucket_metadata_without_extend_range_filtering() {
|
||||
worker: cached,
|
||||
matched_prefix_tokens: 128,
|
||||
uncached_tokens: 128,
|
||||
matched_prefix_blocks: 8,
|
||||
candidate_range_id: "global".into(),
|
||||
max_pending_prefill_tokens: None,
|
||||
};
|
||||
|
||||
@@ -124,6 +124,7 @@ impl Policy for CacheCandidatesPolicy {
|
||||
worker: Arc::clone(&self.worker),
|
||||
matched_prefix_tokens: 1,
|
||||
uncached_tokens: 1,
|
||||
matched_prefix_blocks: 1,
|
||||
candidate_range_id: "global".into(),
|
||||
max_pending_prefill_tokens: None,
|
||||
}],
|
||||
|
||||
Reference in New Issue
Block a user