[Router] Add load-aware prefill admission and bounded policy proposals (#37843)
Signed-off-by: Vincent Gao <vincentbo@linux.alibaba.com> Co-authored-by: Kangyan Zhou <zky314343421@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Kangyan Zhou
Claude Opus 4.8
Shangming Cai
parent
d50e9a9756
commit
ecd97de1fc
@@ -17,6 +17,7 @@ use sgl_kv_indexer::{
|
||||
server_builder, GrpcPrefixIndex, InMemoryKvIndexerBackend, KvIndexerService, PrefixIndexConfig,
|
||||
};
|
||||
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
|
||||
use sgl_router::policies::engine_load::EngineLoadTable;
|
||||
use sgl_router::policies::factory::build_registry;
|
||||
use sgl_router::policies::kv_events::{compute_block_hashes, BlockSizeOracle, HashTree};
|
||||
use sgl_router::policies::request_tokens_for;
|
||||
@@ -94,6 +95,7 @@ async fn external_indexer_routes_to_the_cached_worker() {
|
||||
Arc::new(HashTree::new()),
|
||||
Arc::clone(&tokenizers),
|
||||
Arc::clone(&oracle),
|
||||
EngineLoadTable::new(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user