[sgl-router] Rename chat encoder to chat formatter (#39459)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-09-16 08:46:11 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 5aaa18207c
commit ad94978adf
8 changed files with 117 additions and 113 deletions
@@ -3,8 +3,8 @@
//! Tokenize-once at ingress under the STICKY policy. The engine-tokenization
//! offload (`input_ids` forwarding) is a property of the MODEL — does it have a
//! chat encoder? — not of the routing policy, so a sticky-routed request on a
//! chat-encoder model must forward `input_ids` exactly like cache-aware does,
//! chat formatter? — not of the routing policy, so a sticky-routed request on a
//! chat-formatter model must forward `input_ids` exactly like cache-aware does,
//! while still pinning sessions O(1) by header.
//!
//! Asserts through the real chat handler + `MockWorker` backends:
@@ -17,7 +17,7 @@
//! routing is unchanged by the added tokenization).
//!
//! The model id contains `deepseek-v4` so the tokenizer registry auto-attaches
//! the built-in V4 chat encoder — the engine-equivalent path — without a
//! the built-in V4 chat formatter — the engine-equivalent path — without a
//! template fixture.
use axum::body::Body;
@@ -82,14 +82,14 @@ fn config() -> Config {
/// Build an `AppContext` running the sticky policy over the given workers.
/// The tokenizer registry is loaded from config (real tiny tokenizer + the
/// auto-attached V4 chat encoder) so the ingress can tokenize — the sticky
/// auto-attached V4 chat formatter) so the ingress can tokenize — the sticky
/// policy itself holds no tokenizer.
fn build_ctx(worker_urls: &[String]) -> Arc<AppContext> {
let cfg = config();
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
assert!(
tokenizers.has_chat_encoder(MODEL),
"deepseek-v4 model id must auto-attach the built-in chat encoder"
tokenizers.has_chat_formatter(MODEL),
"deepseek-v4 model id must auto-attach the built-in chat formatter"
);
let registry = Arc::new(WorkerRegistry::default());
for (i, url) in worker_urls.iter().enumerate() {