[Router] Add bucket-aware policy domains and native cache indexing (#38108)
Signed-off-by: Vincent Gao <vincentbo@linux.alibaba.com> Co-authored-by: inkcherry <mingzhi.liu@amd.com> Co-authored-by: yangbodong22011 <13137470+yangbodong22011@users.noreply.github.com>
This commit is contained in:
co-authored by
inkcherry
yangbodong22011
parent
a176ba2f7b
commit
5bebe7a033
@@ -13,7 +13,7 @@ use sgl_router::config::{
|
||||
|
||||
pub const MODEL: &str = "deepseek-v4-tiny";
|
||||
|
||||
/// A single-model `cache_aware_zmq` router. Discovery is a placeholder because
|
||||
/// A single-model native `cache_aware` router. Discovery is a placeholder because
|
||||
/// every caller installs its own `WorkerRegistry`.
|
||||
pub fn config() -> Config {
|
||||
Config {
|
||||
@@ -25,11 +25,13 @@ pub fn config() -> Config {
|
||||
model: ModelConfig {
|
||||
id: MODEL.into(),
|
||||
tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(),
|
||||
policy: PolicyKind::CacheAwareZmq,
|
||||
policy: PolicyKind::CacheAware,
|
||||
decode_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
circuit_breaker: None,
|
||||
cache_aware: Some(CacheAwareConfig::default()),
|
||||
sticky: None,
|
||||
affinity: None,
|
||||
sticky: None,
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user