diff --git a/experimental/sgl-router/src/policies/mod.rs b/experimental/sgl-router/src/policies/mod.rs index cf54c2862..3237e64d8 100644 --- a/experimental/sgl-router/src/policies/mod.rs +++ b/experimental/sgl-router/src/policies/mod.rs @@ -46,12 +46,31 @@ pub struct ExternalPrefixSignal { pub query_blocks: usize, } -/// Tokenizes requests for routing, preferring chat rendering over raw text. +/// Whether the caller pre-tokenized the prompt (`input_ids` present and not +/// null). Such a request is never re-rendered: its ids drive routing and the +/// body is forwarded untouched, malformed values included, for the engine to +/// validate. +pub fn has_caller_input_ids(value: &serde_json::Value) -> bool { + value.get("input_ids").is_some_and(|v| !v.is_null()) +} + +/// Tokenizes a request for routing. Caller `input_ids` win; chat-rendered +/// tokens may also be forwarded to the engine (the chat route decides); raw +/// prompt tokens are routing-only. pub fn request_tokens_for( tokenizers: &TokenizerRegistry, model_id: &ModelId, value: &serde_json::Value, ) -> Option { + if has_caller_input_ids(value) { + // A flat u32 array (empty included) supplies routing tokens; anything + // else yields none, leaving validation to the engine. + let ids = serde::Deserialize::deserialize(&value["input_ids"]).ok()?; + return Some(RequestTokens { + ids, + rendered_from_chat: false, + }); + } if tokenizers.has_chat_formatter(&model_id.0) && value.get("messages").is_some_and(|m| m.is_array()) { diff --git a/experimental/sgl-router/src/server/routes/chat.rs b/experimental/sgl-router/src/server/routes/chat.rs index 2ac5c052d..65fe5806f 100644 --- a/experimental/sgl-router/src/server/routes/chat.rs +++ b/experimental/sgl-router/src/server/routes/chat.rs @@ -10,7 +10,9 @@ use crate::policies::registry::{PdPoolResolver, PdResolveError}; use crate::policies::selection::{ select_decode_peer, select_prefill_worker, DecodeSelectionInputs, PrefillSelectionInputs, }; -use crate::policies::{request_tokens_for, ExternalPrefixSignal, RequestTokens}; +use crate::policies::{ + has_caller_input_ids, request_tokens_for, ExternalPrefixSignal, RequestTokens, +}; use crate::proxy::sse::StreamEnd; use crate::server::app_context::AppContext; use crate::server::error::ApiError; @@ -1409,7 +1411,8 @@ fn build_outgoing_body( /// Forward generated IDs only for request shapes verified against the engine. /// The engine uses `input_ids` verbatim, bypassing its chat-template processing. /// -/// Exclude requests that may render differently with dynamo-render: +/// Preserve caller-provided IDs. Exclude requests that may render differently +/// with dynamo-render: /// - Non-leading system turns or consecutive users, which strict templates rewrite. /// - Historical `reasoning_content`, which may be injected into message content. /// - Tools and tool-call history, which the engine merges and normalizes @@ -1422,7 +1425,8 @@ fn build_outgoing_body( /// overrides and default kwargs cannot be inferred from the request. /// `--disable-input-ids-forwarding` gates forwarding separately for such fleets. fn input_ids_safe_to_forward(value: &serde_json::Value) -> bool { - if request_has_tools(value) + if has_caller_input_ids(value) + || request_has_tools(value) || request_has_non_text_content(value) || request_has_reasoning_content(value) || request_has_role_rewrites(value) @@ -1947,6 +1951,8 @@ mod tests { serde_json::json!({"messages":[{"role":"user","content":"hi"}],"task":"generate"}), serde_json::json!({"messages":[{"role":"user","content":"hi"}],"continue_final_message":true}), serde_json::json!({"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"partial"}]}), + serde_json::json!({"messages":[{"role":"user","content":"hi"}],"input_ids":[7, 8]}), + serde_json::json!({"messages":[{"role":"user","content":"hi"}],"input_ids":"bad"}), ]; for b in blockers { assert!( @@ -1964,6 +1970,7 @@ mod tests { "chat_template": null, "reasoning_effort": null, "chat_template_kwargs": null, + "input_ids": null, "continue_final_message": false }))); } diff --git a/experimental/sgl-router/src/tokenizer/chat_formatter.rs b/experimental/sgl-router/src/tokenizer/chat_formatter.rs index e36b60da3..72ef144ff 100644 --- a/experimental/sgl-router/src/tokenizer/chat_formatter.rs +++ b/experimental/sgl-router/src/tokenizer/chat_formatter.rs @@ -3,8 +3,9 @@ //! Chat rendering via dynamo-render for cache-aware routing and input ID forwarding. //! -//! Engine-specific request normalization is not replicated here. The forwarding -//! guard omits IDs for request shapes whose engine rendering has not been verified. +//! Mirrors SGLang reasoning controls, assistant continuations, and DeepSeek-V4 +//! task selection before rendering. Forwarding remains guarded until parity +//! has been verified for each request shape. use std::collections::HashMap; use std::sync::Arc; @@ -36,6 +37,9 @@ pub struct ChatFormatter { formatter: Arc, /// Template context defaults; request `chat_template_kwargs` override them. defaults: ChatTemplateKwargs, + /// Stripped from a separately tokenized continuation prefix, as SGLang does. + bos_token: Option, + is_deepseek_v4: bool, } impl ChatFormatter { @@ -119,12 +123,15 @@ impl ChatFormatter { if template.chat_template.is_none() { return Ok(None); } + let bos_token = template.bos_tok(); let PromptFormatter::OAI(formatter) = PromptFormatter::from_parts(template, ContextMixins::default(), true) .context("compile chat template")?; Ok(Some(Self { formatter, defaults, + bos_token, + is_deepseek_v4: false, })) } @@ -147,6 +154,15 @@ impl ChatFormatter { } }); let PromptFormatter::OAI(formatter) = deepseek_formatter_for(&model_type, &name)?; + // Same rule dynamo-render applies for the name fallback: `deepseek` + one + // separator + a `v4` segment. + let version = name.strip_prefix("deepseek").unwrap_or(""); + let version = version.strip_prefix(['-', '_', '.']).unwrap_or(version); + let is_deepseek_v4 = model_type + .as_deref() + .map_or(version.split(['-', '_', '.']).next() == Some("v4"), |t| { + t == "deepseek_v4" + }); // Engine defaults: chat mode (`SGLANG_DEFAULT_THINKING=false`) and no // reasoning-effort preamble; dynamo-render defaults to thinking at high effort. let defaults = HashMap::from([ @@ -156,27 +172,112 @@ impl ChatFormatter { Some(Self { formatter, defaults, + bos_token: Some("<|begin▁of▁sentence|>".into()), + is_deepseek_v4, }) } + /// Request kwargs plus the thinking/effort defaults SGLang derives from + /// `reasoning` / `reasoning_effort` (`protocol.py::normalize_reasoning_inputs`). fn template_kwargs(&self, request: &JsonValue) -> Result { let mut kwargs: ChatTemplateKwargs = match request.get("chat_template_kwargs") { None | Some(JsonValue::Null) => ChatTemplateKwargs::new(), Some(v) => serde_json::from_value(v.clone()).context("chat_template_kwargs")?, }; + // `reasoning.effort` overrides top-level `reasoning_effort`; `enabled` + // alone turns thinking on; any effort decides thinking by `!= "none"`. + // Explicit kwargs keep their values (the engine uses setdefault). + let reasoning = &request["reasoning"]; + let mut thinking = None; + if reasoning.is_object() { + let enabled = match reasoning + .get("enabled") + .filter(|v| !v.is_null()) + .or_else(|| reasoning.get("enable")) + { + Some(JsonValue::Bool(enabled)) => *enabled, + Some(JsonValue::String(s)) => { + ["1", "true", "yes", "y", "on"].contains(&s.trim().to_lowercase().as_str()) + } + _ => false, + }; + if enabled { + thinking = Some(true); + } + } + let effort = [ + reasoning.get("effort"), + reasoning.get("reasoning_effort"), + request.get("reasoning_effort"), + ] + .into_iter() + .flatten() + .find(|v| !v.is_null()) + .cloned(); + if let Some(effort) = &effort { + thinking = Some(effort != "none"); + } + if let Some(thinking) = thinking { + kwargs.entry("thinking".into()).or_insert(thinking.into()); + kwargs + .entry("enable_thinking".into()) + .or_insert(thinking.into()); + } + if let Some(mut effort) = effort { + // The engine's official V4 profile accepts only these; others map to + // no preamble. + if self.is_deepseek_v4 && !matches!(effort.as_str(), Some("low" | "high" | "max")) { + effort = "low".into(); + } + kwargs.entry("reasoning_effort".into()).or_insert(effort); + } for (key, value) in &self.defaults { kwargs.entry(key.clone()).or_insert_with(|| value.clone()); } Ok(kwargs) } - /// Render the prompt text dynamo-render produces for `request`. - pub fn render(&self, request: &JsonValue) -> Result { - anyhow::ensure!(request["messages"].is_array(), "messages must be an array"); + /// Rendered prompt plus the assistant continuation prefix SGLang tokenizes + /// separately (`_handle_last_assistant_message`). + fn render_parts(&self, request: &JsonValue) -> Result<(String, String)> { let kwargs = self.template_kwargs(request)?; - self.formatter - .render(&ChatRequest { request, kwargs }) - .context("render chat template") + let continuing = request["continue_final_message"] == true; + let mut messages: Vec = request["messages"] + .as_array() + .context("messages must be an array")? + .iter() + .map(engine_message) + .collect(); + let mut prefix = String::new(); + if let Some(last) = messages.last_mut().filter(|m| m["role"] == "assistant") { + if let Some(content) = last["content"].as_str() { + if continuing { + prefix = content.to_owned(); + messages.pop(); + } else { + *last = serde_json::json!({"role": "user", "content": content}); + } + } + } + if self.is_deepseek_v4 { + if let Some(task) = request.get("task").filter(|v| !v.is_null()).cloned() { + let message = messages + .iter_mut() + .rev() + .find(|m| matches!(m["role"].as_str(), Some("user" | "developer"))) + .context("task requires a user or developer message")?; + message["task"] = task; + } + } + let prompt = self + .formatter + .render(&ChatRequest { + request, + messages, + kwargs, + }) + .context("render chat template")?; + Ok((prompt, prefix)) } pub fn encode( @@ -184,8 +285,60 @@ impl ChatFormatter { tokenizer: &dynamo_tokenizers::Tokenizer, request: &JsonValue, ) -> Result> { - super::adapter::encode(tokenizer, &self.render(request)?) + let (prompt, prefix) = self.render_parts(request)?; + let mut ids = super::adapter::encode(tokenizer, &prompt)?; + if !prefix.is_empty() { + // SGLang encodes the assistant prefix separately and removes its leading BOS. + let mut suffix = super::adapter::encode(tokenizer, &prefix)?; + if let Some(bos) = self.bos_token.as_deref().filter(|s| !s.is_empty()) { + let bos = super::adapter::encode(tokenizer, bos)?; + if bos.len() == 1 && suffix.first() == bos.first() { + suffix.remove(0); + } + } + ids.extend(suffix); + } + Ok(ids) } + + /// Use `encode` for token ids to preserve continuation boundaries. + pub fn render(&self, request: &JsonValue) -> Result { + let (prompt, prefix) = self.render_parts(request)?; + Ok(prompt + &prefix) + } +} + +/// Message fields the engine's request schema keeps for non-user roles. +const GENERIC_MESSAGE_KEYS: [&str; 7] = [ + "role", + "content", + "tool_call_id", + "name", + "reasoning_content", + "tool_calls", + "tools", +]; + +/// Normalize a message as SGLang's pydantic dump does before rendering: roles +/// lowercased, unknown and null fields dropped, `user` reduced to role and +/// content, null content blanked. +fn engine_message(message: &JsonValue) -> JsonValue { + let role = message["role"].as_str().unwrap_or_default().to_lowercase(); + let mut out = serde_json::Map::new(); + if role != "user" { + for key in GENERIC_MESSAGE_KEYS { + if let Some(v) = message.get(key).filter(|v| !v.is_null()) { + out.insert(key.into(), v.clone()); + } + } + } + let content = match &message["content"] { + JsonValue::Null => "".into(), + content => content.clone(), + }; + out.insert("role".into(), role.into()); + out.insert("content".into(), content); + out.into() } /// `content` of an HF `AddedToken` object (`{"content": "", "lstrip": ...}`). @@ -199,10 +352,11 @@ fn added_token_content(token: &JsonValue) -> Option { struct ChatRequest<'a> { request: &'a JsonValue, + /// Normalized copy of `request["messages"]`. + messages: Vec, kwargs: ChatTemplateKwargs, } -/// Mirrors dynamo-render's own impl for its wire type: request fields pass through. impl OAIChatLikeRequest for ChatRequest<'_> { fn model(&self) -> String { self.request["model"] @@ -211,7 +365,7 @@ impl OAIChatLikeRequest for ChatRequest<'_> { .to_owned() } fn messages(&self) -> Value { - Value::from_serialize(&self.request["messages"]) + Value::from_serialize(&self.messages) } fn tools(&self) -> Option { let tools = self.request.get("tools")?; @@ -221,13 +375,20 @@ impl OAIChatLikeRequest for ChatRequest<'_> { if tools.as_array().is_none_or(|t| t.is_empty()) { return None; } - may_be_fix_tool_schema(tools.clone()) + let mut tools = tools.clone(); + // SGLang renders only the named tool for a function `tool_choice`. + if let Some(name) = self.request["tool_choice"]["function"]["name"].as_str() { + tools + .as_array_mut()? + .retain(|tool| tool["function"]["name"] == name); + } + may_be_fix_tool_schema(tools) } fn tool_choice(&self) -> Option { self.request.get("tool_choice").map(Value::from_serialize) } fn reasoning_effort(&self) -> Option { - self.request + self.kwargs .get("reasoning_effort") .map(Value::from_serialize) } @@ -468,13 +629,139 @@ mod tests { ); } - /// Request kwargs override the chat-mode default. + /// The engine's V4 encoder reads only `chat_template_kwargs.thinking` + /// (`serving_chat.py`); `enable_thinking` alone leaves chat mode, while + /// `reasoning_effort` sets both keys through the normalization above. #[test] fn v4_thinking_kwarg_overrides_chat_default() { let mut req = request(json!([{"role":"user","content":"ABCD"}])); req["chat_template_kwargs"] = json!({"thinking": true}); let out = deepseek_v4().render(&req).unwrap(); assert!(out.ends_with("<|Assistant|>"), "got: {out}"); + req["chat_template_kwargs"] = json!({"enable_thinking": true}); + let out = deepseek_v4().render(&req).unwrap(); + assert!(out.ends_with("<|Assistant|>"), "got: {out}"); + req["chat_template_kwargs"] = JsonValue::Null; + req["reasoning_effort"] = json!("high"); + let out = deepseek_v4().render(&req).unwrap(); + assert!(out.contains("<|Assistant|>"), "got: {out}"); + } + + #[test] + fn request_controls_reach_dynamo() { + let jinja = jinja( + json!({"chat_template": "{{ tools | tojson }} {{ thinking }} {{ reasoning_effort }}"}), + ); + for formatter in [jinja, deepseek_v4()] { + let mut req = request(json!([{"role":"user","content":"hi"}])); + req["tools"] = json!([ + {"type":"function","function":{"name":"first"}}, + {"type":"function","function":{"name":"second"}} + ]); + req["tool_choice"] = json!({"type":"function","function":{"name":"second"}}); + req["reasoning"] = json!({"effort":"high"}); + let out = formatter.render(&req).unwrap(); + assert!(out.contains("second") && !out.contains("first")); + assert!(out.contains("high") || out.contains("Reasoning Effort:")); + req["tool_choice"] = json!("none"); + req["reasoning_effort"] = json!("none"); + req["reasoning"] = JsonValue::Null; + let out = formatter.render(&req).unwrap(); + assert!(!out.contains("first") && !out.contains("second")); + assert!(out.contains("False none") || out.ends_with("")); + } + } + + /// Mirrors `protocol.py::normalize_reasoning_inputs`: effort decides both + /// thinking keys via setdefault, so explicit kwargs win. + #[test] + fn reasoning_effort_sets_thinking_defaults_with_explicit_kwargs_winning() { + let enc = jinja(json!({ + "chat_template": "{{ reasoning_effort }}:{{ thinking }}:{{ enable_thinking }}" + })); + let mut req = request(json!([{"role": "user", "content": "hi"}])); + req["reasoning_effort"] = json!("none"); + assert_eq!(enc.render(&req).unwrap(), "none:False:False"); + req["reasoning_effort"] = json!("high"); + assert_eq!(enc.render(&req).unwrap(), "high:True:True"); + req["chat_template_kwargs"] = json!({"enable_thinking": false}); + assert_eq!(enc.render(&req).unwrap(), "high:True:False"); + req["reasoning"] = json!({"effort": "low"}); + assert_eq!(enc.render(&req).unwrap(), "low:True:False"); + req["reasoning"] = json!({"enabled": "yes"}); + req["reasoning_effort"] = JsonValue::Null; + req["chat_template_kwargs"] = JsonValue::Null; + assert_eq!(enc.render(&req).unwrap(), ":True:True"); + } + + /// Messages reach the template shaped like the engine's request schema. + #[test] + fn messages_match_engine_schema() { + let enc = jinja(json!({"chat_template": "{{ messages | tojson }}"})); + let out = enc + .render(&request(json!([ + {"role": "User", "content": "hi", "name": "bob", "extra": 1}, + {"role": "assistant", "name": "a", "tool_calls": null, "tool_call_id": "c1"}, + {"role": "user", "content": "again"} + ]))) + .unwrap(); + let rendered: JsonValue = serde_json::from_str(&out).unwrap(); + assert_eq!( + rendered, + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "", "name": "a", "tool_call_id": "c1"}, + {"role": "user", "content": "again"} + ]) + ); + } + + #[test] + fn v4_task_uses_dynamo_task_tokens() { + let mut req = request(json!([{"role":"user","content":"example.com"}])); + for (task, suffix) in [ + ("domain", "<|domain|>"), + ("action", "<|Assistant|><|action|>"), + ] { + req["task"] = json!(task); + assert_eq!( + deepseek_v4().render(&req).unwrap(), + format!("<|begin▁of▁sentence|><|User|>example.com{suffix}") + ); + } + req["messages"] = json!([{"role":"system","content":"hi"}]); + assert!(deepseek_v4().render(&req).is_err()); + } + + #[test] + fn continuation_preserves_token_boundaries() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg: JsonValue = + serde_json::from_str(include_str!("../../tests/fixtures/tiny_tokenizer.json")).unwrap(); + cfg["model"]["vocab"]["ab"] = json!(257); + cfg["model"]["merges"] = json!(["a b"]); + let path = dir.path().join("tokenizer.json"); + std::fs::write(&path, cfg.to_string()).unwrap(); + let tokenizer = super::super::adapter::load(path.to_str().unwrap()).unwrap(); + let formatter = jinja(json!({"chat_template":"a", "bos_token":"<|endoftext|>"})); + let mut req = request(json!([ + {"role":"user","content":"hi"}, {"role":"assistant","content":"b"} + ])); + req["continue_final_message"] = json!(true); + assert_eq!(formatter.encode(&tokenizer, &req).unwrap(), vec![97, 98]); + assert_eq!( + super::super::adapter::encode(&tokenizer, "ab").unwrap(), + vec![257] + ); + req["messages"][1]["content"] = json!("<|endoftext|>b"); + assert_eq!(formatter.encode(&tokenizer, &req).unwrap(), vec![97, 98]); + req["continue_final_message"] = json!(false); + req["messages"][1]["content"] = json!("b"); + let out = deepseek_v4().render(&req).unwrap(); + assert_eq!( + out, + "<|begin▁of▁sentence|><|User|>hi\n\nb<|Assistant|>" + ); } /// The engine never renders `response_format` into the prompt. diff --git a/experimental/sgl-router/tests/component/tokenizer/mod.rs b/experimental/sgl-router/tests/component/tokenizer/mod.rs index 09c530d86..1ac2e5859 100644 --- a/experimental/sgl-router/tests/component/tokenizer/mod.rs +++ b/experimental/sgl-router/tests/component/tokenizer/mod.rs @@ -2,3 +2,4 @@ // SPDX-License-Identifier: Apache-2.0 mod parity; +mod render_parity; diff --git a/experimental/sgl-router/tests/component/tokenizer/render_parity.rs b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs new file mode 100644 index 000000000..f69220729 --- /dev/null +++ b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Compare Dynamo prompt IDs with SGLang-generated fixtures using cached tokenizers. +//! CI skips this matrix unless model snapshots are available. + +use serde::Deserialize; +use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, ModelConfig, ObservabilityConfig, PolicyKind, + ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, +}; +use sgl_router::discovery::ModelId; +use sgl_router::policies::request_tokens_for; +use sgl_router::tokenizer::{adapter, chat_formatter::ChatFormatter, TokenizerRegistry}; +use std::path::PathBuf; + +#[derive(Deserialize)] +struct Fixture { + model_id: String, + cases: Vec, +} + +#[derive(Deserialize)] +struct Case { + shape: String, + request: serde_json::Value, + expected_token_ids: Vec, +} + +/// String-to-array conversion is a known parity gap, so these templates must +/// opt out of forwarding. This fixture needs no cached model files. +#[test] +fn array_only_template_content_parity() { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("../../fixtures/array_content_rendering.json")).unwrap(); + let formatter = ChatFormatter::from_tokenizer_config( + serde_json::json!({"chat_template": fixture["chat_template"]}), + None, + ) + .unwrap() + .unwrap(); + let tokenizer = adapter::load("tests/fixtures/tiny_tokenizer.json").unwrap(); + for case in fixture["cases"].as_array().unwrap() { + let request = + serde_json::json!({"messages": [{"role": "user", "content": case["content"]}]}); + let ids = formatter.encode(&tokenizer, &request).unwrap(); + assert_eq!( + serde_json::json!(ids) == case["engine_token_ids"], + case["content"].is_array(), + "{case}" + ); + } +} + +/// Replace every `YYYY-MM-DD` with a placeholder. +/// +/// Templates that call `strftime_now` render the day the prompt is built, so a +/// fixture captured earlier differs from today's render in the date alone. That +/// is not drift: the engine consumes forwarded IDs verbatim and never re-renders. +/// Masking keeps the rest of the prompt under exact comparison, and dates that +/// come from the request render the same on both sides, so masking them is a +/// no-op. +fn mask_dates(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(c) = rest.chars().next() { + if starts_with_iso_date(rest) { + out.push_str(""); + rest = &rest[10..]; + } else { + out.push(c); + rest = &rest[c.len_utf8()..]; + } + } + out +} + +fn starts_with_iso_date(text: &str) -> bool { + let b = text.as_bytes(); + b.len() >= 10 + && b[..4].iter().all(u8::is_ascii_digit) + && b[4] == b'-' + && b[5..7].iter().all(u8::is_ascii_digit) + && b[7] == b'-' + && b[8..10].iter().all(u8::is_ascii_digit) +} + +fn snapshot_tokenizer(model_id: &str) -> Option { + let hf_home = std::env::var("HF_HOME") + .ok() + .map(PathBuf::from) + .or_else(|| dirs::home_dir().map(|h| h.join(".cache/huggingface")))?; + let snapshots = hf_home + .join("hub") + .join(format!("models--{}", model_id.replace('/', "--"))) + .join("snapshots"); + std::fs::read_dir(snapshots) + .ok()? + .flatten() + .map(|e| e.path().join("tokenizer.json")) + .find(|p| p.is_file()) +} + +fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry { + let cfg = Config { + server: ServerConfig { + host: "0".into(), + port: 0, + ..Default::default() + }, + observability: ObservabilityConfig::default(), + model: ModelConfig { + id: model_id.into(), + tokenizer_path: tokenizer_path.to_str().unwrap().into(), + disable_input_ids_forwarding: false, + policy: PolicyKind::RoundRobin, + decode_policy: Default::default(), + bucket_config: None, + circuit_breaker: None, + cache_aware: None, + affinity: None, + sticky: None, + fused: None, + eligibility: None, + sampling_overrides: Default::default(), + }, + discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + }; + TokenizerRegistry::load_from_config(&cfg).unwrap() +} + +#[test] +fn chat_render_parity_matrix() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/chat_render_parity"); + let mut checked = 0; + for entry in std::fs::read_dir(&root).unwrap().flatten() { + let raw = std::fs::read_to_string(entry.path()).unwrap(); + let fixture: Fixture = serde_json::from_str(&raw).unwrap(); + let Some(tokenizer_path) = snapshot_tokenizer(&fixture.model_id) else { + eprintln!("skip {}: snapshot not cached", fixture.model_id); + continue; + }; + let reg = registry(&fixture.model_id, tokenizer_path); + assert!( + reg.has_chat_formatter(&fixture.model_id), + "{}: no chat formatter resolved", + fixture.model_id + ); + for case in &fixture.cases { + let tokens = + request_tokens_for(®, &ModelId(fixture.model_id.clone()), &case.request) + .unwrap_or_else(|| panic!("{}/{}: no tokens", fixture.model_id, case.shape)); + assert!( + tokens.rendered_from_chat, + "{}/{}: fell back to raw text", + fixture.model_id, case.shape + ); + if tokens.ids != case.expected_token_ids { + // Decode with special tokens kept, so a difference in them still fails. + let tokenizer = reg.get(&fixture.model_id).unwrap(); + let rendered = adapter::decode_complete(&tokenizer, &tokens.ids, false).unwrap(); + let expected = + adapter::decode_complete(&tokenizer, &case.expected_token_ids, false).unwrap(); + assert_eq!( + mask_dates(&rendered), + mask_dates(&expected), + "DRIFT on {}/{}", + fixture.model_id, + case.shape + ); + eprintln!( + "{}/{}: date drift only; fixture captured on another day", + fixture.model_id, case.shape + ); + } + checked += 1; + } + } + assert!( + checked > 0, + "no cached model snapshots; parity was not checked" + ); + eprintln!("chat render parity: {checked} cases checked"); +} diff --git a/experimental/sgl-router/tests/fixtures/array_content_rendering.json b/experimental/sgl-router/tests/fixtures/array_content_rendering.json new file mode 100644 index 000000000..968d6ed13 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/array_content_rendering.json @@ -0,0 +1,4 @@ +{"chat_template": "{% for message in messages %}{{ message.role }}:{% for part in message.content %}{% if part.type == 'text' %}{{ part.text }}{% elif part.type == 'image' %}{% endif %}{% endfor %};{% endfor %}{% if add_generation_prompt %}assistant:{% endif %}", "cases": [ +{"content": "hello", "engine_token_ids": [117, 115, 101, 114, 58, 59, 97, 115, 115, 105, 115, 116, 97, 110, 116, 58]}, +{"content": [{"type": "text", "text": "hello"}], "engine_token_ids": [117, 115, 101, 114, 58, 104, 101, 108, 108, 111, 59, 97, 115, 115, 105, 115, 116, 97, 110, 116, 58]} +]} diff --git a/experimental/sgl-router/tests/fixtures/chat_render_parity/deepseek-v4-flash.json b/experimental/sgl-router/tests/fixtures/chat_render_parity/deepseek-v4-flash.json new file mode 100644 index 000000000..6ac2014ee --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/chat_render_parity/deepseek-v4-flash.json @@ -0,0 +1,7 @@ +{"model_id": "deepseek-ai/DeepSeek-V4-Flash", "cases": [ +{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [0, 128803, 63006, 19346, 295, 834, 10175, 16, 128804, 128822]}, +{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [0, 3476, 477, 259, 10935, 16, 128803, 3085, 344, 223, 20, 13, 20, 33, 128804, 128822]}, +{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. "}]}, "expected_token_ids": [0, 3476, 477, 259, 10935, 16, 128803, 23166, 128804, 128822, 19923, 3, 1730, 588, 342, 1694, 33, 1, 128803, 52, 3319, 90902, 294, 270, 2831, 28, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 128804, 128822]}, +{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [0, 128803, 50012, 943, 436, 16, 128804, 128822]}, +{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [0, 128803, 50012, 943, 436, 16, 128804, 128821]} +]} diff --git a/experimental/sgl-router/tests/fixtures/chat_render_parity/glm-5.2.json b/experimental/sgl-router/tests/fixtures/chat_render_parity/glm-5.2.json new file mode 100644 index 000000000..7613286bf --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/chat_render_parity/glm-5.2.json @@ -0,0 +1,7 @@ +{"model_id": "zai-org/GLM-5.2-FP8", "cases": [ +{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154827, 45494, 15576, 304, 825, 11646, 13, 154828, 154841]}, +{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154826, 2610, 525, 50205, 13, 154827, 3838, 374, 220, 17, 10, 17, 30, 154828, 154841]}, +{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. "}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154826, 2610, 525, 50205, 13, 154827, 13041, 154828, 154841, 154842, 9703, 0, 2585, 646, 358, 1492, 30, 154827, 84140, 1242, 963, 315, 279, 3119, 25, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 220, 154828, 154841]}, +{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [154822, 154824, 154827, 38479, 911, 432, 13, 154828, 154841, 154842]}, +{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154827, 38479, 911, 432, 13, 154828, 154841]} +]} diff --git a/experimental/sgl-router/tests/fixtures/chat_render_parity/gpt-oss-20b.json b/experimental/sgl-router/tests/fixtures/chat_render_parity/gpt-oss-20b.json new file mode 100644 index 000000000..c7cdc1d6e --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/chat_render_parity/gpt-oss-20b.json @@ -0,0 +1,7 @@ +{"model_id": "openai/gpt-oss-20b", "cases": [ +{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 62316, 5911, 306, 1001, 21872, 13, 200007, 200006, 173781]}, +{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 77944, 200008, 2, 68406, 279, 3575, 553, 28410, 364, 200007, 200006, 1428, 200008, 4827, 382, 220, 17, 10, 17, 30, 200007, 200006, 173781]}, +{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. "}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 77944, 200008, 2, 68406, 279, 3575, 553, 28410, 364, 200007, 200006, 1428, 200008, 12194, 200007, 200006, 173781, 200005, 17196, 200008, 13225, 0, 3253, 665, 357, 1652, 30, 200007, 200006, 1428, 200008, 198128, 328, 290, 3496, 25, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 220, 200007, 200006, 173781]}, +{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 42421, 1078, 480, 13, 200007, 200006, 173781]}, +{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 42421, 1078, 480, 13, 200007, 200006, 173781]} +]} diff --git a/experimental/sgl-router/tests/fixtures/chat_render_parity/minimax-m3.json b/experimental/sgl-router/tests/fixtures/chat_render_parity/minimax-m3.json new file mode 100644 index 000000000..231a47311 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/chat_render_parity/minimax-m3.json @@ -0,0 +1,7 @@ +{"model_id": "MiniMaxAI/MiniMax-M3", "cases": [ +{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 66938, 13182, 296, 841, 14997, 46, 200020, 10, 200019, 1361, 10]}, +{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 6000, 101, 46, 200020, 10, 200019, 3995, 10, 3376, 355, 32, 50, 43, 50, 63, 200020, 10, 200019, 1361, 10]}, +{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. "}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 6000, 101, 46, 200020, 10, 200019, 3995, 10, 22700, 200020, 10, 200019, 1361, 10, 200060, 19739, 33, 2329, 566, 343, 1576, 63, 200020, 10, 200019, 3995, 10, 96530, 300, 275, 2748, 58, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 32, 200020, 10, 200019, 1361, 10]}, +{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 38460, 894, 412, 46, 200020, 10, 200019, 1361, 10]}, +{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 38460, 894, 412, 46, 200020, 10, 200019, 1361, 10]} +]} diff --git a/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3-8b.json b/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3-8b.json new file mode 100644 index 000000000..2618a006a --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3-8b.json @@ -0,0 +1,7 @@ +{"model_id": "Qwen/Qwen3-8B", "cases": [ +{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [151644, 872, 198, 45764, 15588, 304, 825, 11652, 13, 151645, 198, 151644, 77091, 198]}, +{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [151644, 8948, 198, 2610, 525, 50537, 13, 151645, 198, 151644, 872, 198, 3838, 374, 220, 17, 10, 17, 30, 151645, 198, 151644, 77091, 198]}, +{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. "}]}, "expected_token_ids": [151644, 8948, 198, 2610, 525, 50537, 13, 151645, 198, 151644, 872, 198, 13048, 151645, 198, 151644, 77091, 198, 9707, 0, 2585, 646, 358, 1492, 30, 151645, 198, 151644, 872, 198, 84836, 1242, 963, 315, 279, 3119, 25, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 220, 151645, 198, 151644, 77091, 198]}, +{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [151644, 872, 198, 38687, 911, 432, 13, 151645, 198, 151644, 77091, 198, 151667, 271, 151668, 271]}, +{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [151644, 872, 198, 38687, 911, 432, 13, 151645, 198, 151644, 77091, 198]} +]} diff --git a/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.5-27b.json b/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.5-27b.json new file mode 100644 index 000000000..9a85f2e5d --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.5-27b.json @@ -0,0 +1,7 @@ +{"model_id": "Qwen/Qwen3.5-27B", "cases": [ +{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [248045, 846, 198, 44240, 15131, 303, 799, 11316, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}, +{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [248045, 8678, 198, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 3710, 369, 220, 17, 10, 17, 30, 248046, 198, 248045, 74455, 198, 248068, 198]}, +{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. "}]}, "expected_token_ids": [248045, 8678, 198, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 12675, 248046, 198, 248045, 74455, 198, 9419, 0, 2500, 628, 353, 1438, 30, 248046, 198, 248045, 846, 198, 81911, 168699, 314, 279, 3019, 25, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}, +{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271]}, +{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 198]} +]} diff --git a/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.8-27b.json b/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.8-27b.json new file mode 100644 index 000000000..3c82526ab --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.8-27b.json @@ -0,0 +1,7 @@ +{"model_id": "Qwen/Qwen3.8-27B", "cases": [ +{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 44240, 15131, 303, 799, 11316, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}, +{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 271, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 3710, 369, 220, 17, 10, 17, 30, 248046, 198, 248045, 74455, 198, 248068, 198]}, +{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. 第一步,收集数据。Then we iterate. "}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 271, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 12675, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271, 9419, 0, 2500, 628, 353, 1438, 30, 248046, 198, 248045, 846, 198, 81911, 168699, 314, 279, 3019, 25, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}, +{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271]}, +{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 198]} +]} diff --git a/experimental/sgl-router/tests/proxy/cache_aware_input_ids.rs b/experimental/sgl-router/tests/proxy/cache_aware_input_ids.rs index d4b7cda17..59d0666ca 100644 --- a/experimental/sgl-router/tests/proxy/cache_aware_input_ids.rs +++ b/experimental/sgl-router/tests/proxy/cache_aware_input_ids.rs @@ -168,3 +168,63 @@ async fn multimodal_request_omits_input_ids() { "multimodal requests must not forward input_ids; got {body}" ); } + +/// Caller-supplied `input_ids` are never re-rendered or replaced: a flat u32 +/// array (empty included) drives routing, anything else yields no routing +/// tokens, and the body reaches the engine byte-for-byte for validation. +#[tokio::test] +async fn caller_input_ids_are_used_for_routing_and_preserved() { + let mock = MockWorker::start(vec![]).await; + let ctx = build_ctx(mock.url.clone()); + for (ids, expected) in [ + (json!([7, 8]), Some(vec![7, 8])), + (json!([]), Some(vec![])), + (json!([7, -1]), None), + (json!("bad"), None), + ] { + let request = json!({ + "model": MODEL, + "messages": [{"role": "user", "content": "hi"}], + "input_ids": ids, + }); + let tokens = sgl_router::policies::request_tokens_for( + &ctx.tokenizers, + &ModelId(MODEL.into()), + &request, + ); + assert!(!tokens.as_ref().is_some_and(|t| t.rendered_from_chat)); + assert_eq!(tokens.map(|t| t.ids), expected, "input_ids: {ids}"); + assert_eq!( + send(Arc::clone(&ctx), request.clone()).await, + StatusCode::OK + ); + assert_eq!(captured(&mock), request, "body must be forwarded untouched"); + } + // Bypasses are not rendering failures. + assert!(!ctx + .metrics + .render() + .contains("sgl_router_ingress_tokenize_errors_total{")); +} + +/// `input_ids: null` is the same as absent: the router renders and forwards. +#[tokio::test] +async fn null_input_ids_keep_normal_rendering() { + let mock = MockWorker::start(vec![]).await; + let ctx = build_ctx(mock.url.clone()); + let status = send( + ctx, + json!({ + "model": MODEL, + "messages": [{"role": "user", "content": "hello there friend"}], + "input_ids": null, + }), + ) + .await; + assert_eq!(status, StatusCode::OK); + let body = captured(&mock); + assert!( + body["input_ids"].as_array().is_some_and(|a| !a.is_empty()), + "null input_ids must not suppress rendering; got {body}" + ); +} diff --git a/experimental/sgl-router/tests/scripts/generate_array_content_fixture.py b/experimental/sgl-router/tests/scripts/generate_array_content_fixture.py new file mode 100644 index 000000000..1fc60d1a5 --- /dev/null +++ b/experimental/sgl-router/tests/scripts/generate_array_content_fixture.py @@ -0,0 +1,50 @@ +"""Regenerate the array-only template regression using SGLang's content processor. + +Run: PYTHONPATH=../../python python tests/scripts/generate_array_content_fixture.py +""" + +import json +from pathlib import Path + +from transformers import PreTrainedTokenizerFast + +from sglang.srt.parser.jinja_template_utils import ( + detect_jinja_template_content_format, + process_content_for_template_format, +) + +ROOT = Path(__file__).resolve().parents[1] / "fixtures" +TEMPLATE = ( + "{% for message in messages %}{{ message.role }}:" + "{% for part in message.content %}" + "{% if part.type == 'text' %}{{ part.text }}" + "{% elif part.type == 'image' %}{% endif %}" + "{% endfor %};{% endfor %}" + "{% if add_generation_prompt %}assistant:{% endif %}" +) + + +def main(): + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=str(ROOT / "tiny_tokenizer.json") + ) + tokenizer.chat_template = TEMPLATE + content_format = detect_jinja_template_content_format(TEMPLATE) + assert content_format == "openai" + cases = [] + for content in ["hello", [{"type": "text", "text": "hello"}]]: + message = process_content_for_template_format( + {"role": "user", "content": content}, content_format, [], [], [], [] + ) + ids = tokenizer.apply_chat_template( + [message], return_dict=False, add_generation_prompt=True + ) + cases.append(json.dumps({"content": content, "engine_token_ids": ids})) + cases_json = ",\n".join(cases) + (ROOT / "array_content_rendering.json").write_text( + f'{{"chat_template": {json.dumps(TEMPLATE)}, "cases": [\n{cases_json}\n]}}\n' + ) + + +if __name__ == "__main__": + main() diff --git a/experimental/sgl-router/tests/scripts/generate_chat_render_parity.py b/experimental/sgl-router/tests/scripts/generate_chat_render_parity.py new file mode 100644 index 000000000..cec6c3c19 --- /dev/null +++ b/experimental/sgl-router/tests/scripts/generate_chat_render_parity.py @@ -0,0 +1,158 @@ +"""Generate reference prompt IDs with SGLang helpers and cached model tokenizers. + +Run: python tests/scripts/generate_chat_render_parity.py +""" + +import copy +import json +import pathlib +import sys + +from transformers.utils.hub import cached_file + +from sglang.srt.entrypoints.openai import encoding_dsv4 +from sglang.srt.entrypoints.openai.chat_encoding import ( + resolve_dsv4_reasoning_effort_profile, +) +from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.entrypoints.openai.serving_chat import ( + ThinkingMode, + normalize_assistant_tool_call_arguments, + normalize_tool_content, +) +from sglang.srt.parser.jinja_template_utils import ( + detect_jinja_template_content_format, + process_content_for_template_format, +) +from sglang.srt.utils.hf_transformers_utils import get_tokenizer + +ROOT = pathlib.Path(__file__).resolve().parents[1] / "fixtures" / "chat_render_parity" + +MODELS = { + "qwen3-8b": "Qwen/Qwen3-8B", + "qwen3.5-27b": "Qwen/Qwen3.5-27B", + "qwen3.8-27b": "Qwen/Qwen3.8-27B", + "gpt-oss-20b": "openai/gpt-oss-20b", + "glm-5.2": "zai-org/GLM-5.2-FP8", + "minimax-m3": "MiniMaxAI/MiniMax-M3", + "deepseek-v4-flash": "deepseek-ai/DeepSeek-V4-Flash", +} + +LONG = "Résumé of the plan: " + "第一步,收集数据。Then we iterate. " * 8 + +SHAPES = { + "user_only": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, + "system_user": { + "messages": [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "What is 2+2?"}, + ] + }, + "multi_turn": { + "messages": [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "user", "content": LONG}, + ] + }, + "thinking_off": { + "messages": [{"role": "user", "content": "Think about it."}], + "chat_template_kwargs": {"enable_thinking": False, "thinking": False}, + }, + "thinking_on": { + "messages": [{"role": "user", "content": "Think about it."}], + "chat_template_kwargs": {"enable_thinking": True, "thinking": True}, + }, +} + + +def snapshot_dir(model_id): + return pathlib.Path( + cached_file(model_id, "config.json", local_files_only=True) + ).parent + + +def engine_messages(request, content_format): + messages = [m.model_dump() for m in request.messages] + for message in messages: + normalize_assistant_tool_call_arguments(message) + out = [] + for msg in copy.deepcopy(messages): + if msg.get("content") is None: + msg["content"] = "" + processed = process_content_for_template_format( + msg, content_format, [], [], [], [] + ) + processed["content"] = normalize_tool_content( + processed["role"], processed.get("content") + ) + out.append(processed) + return out + + +def engine_prompt_ids(model_id, tok, request): + """Mirror `_apply_jinja_template` for a text-only request without tools.""" + snapshot = snapshot_dir(model_id) + model_type = json.load(open(snapshot / "config.json")).get("model_type") + if model_type == "deepseek_v4": + messages = engine_messages(request, "string") + if messages[0]["role"] != "system": + messages.insert(0, {"role": "system", "content": ""}) + thinking = (request.chat_template_kwargs or {}).get("thinking", False) + text = encoding_dsv4.encode_messages( + messages, + thinking_mode=ThinkingMode.THINKING if thinking else ThinkingMode.CHAT, + reasoning_effort=None, + reasoning_effort_profile=resolve_dsv4_reasoning_effort_profile( + model_path=str(snapshot) + ), + ) + return tok.encode(text) + + template = tok.chat_template + if not isinstance(template, str): + raise RuntimeError(f"{model_id}: named template dict is not supported here") + messages = engine_messages(request, detect_jinja_template_content_format(template)) + extra = {} + if request.reasoning_effort is not None: + extra["reasoning_effort"] = request.reasoning_effort + if request.chat_template_kwargs: + extra.update(request.chat_template_kwargs) + rendered = tok.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + tools=None, + return_dict=False, + **extra, + ) + encode_kwargs = {"add_special_tokens": False} if len(tok.encode("")) > 0 else {} + return tok.encode(rendered, **encode_kwargs) + + +def main(): + ROOT.mkdir(parents=True, exist_ok=True) + for slug, model_id in MODELS.items(): + try: + snapshot = snapshot_dir(model_id) + except Exception as e: + print(f"skip {model_id}: {e}", file=sys.stderr) + continue + tok = get_tokenizer(str(snapshot)) + cases = [] + for shape, body in SHAPES.items(): + request = ChatCompletionRequest(model=model_id, **copy.deepcopy(body)) + ids = engine_prompt_ids(model_id, tok, request) + cases.append({"shape": shape, "request": body, "expected_token_ids": ids}) + out = ROOT / f"{slug}.json" + lines = [json.dumps(case, ensure_ascii=False) for case in cases] + out.write_text( + '{"model_id": %s, "cases": [\n%s\n]}\n' + % (json.dumps(model_id), ",\n".join(lines)) + ) + print(f"wrote {out} ({len(cases)} cases)") + + +if __name__ == "__main__": + main()