[router] Improve SGLang chat render parity (#39133)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>

experimental/sgl-router/src/policies/mod.rs

@@ -46,12 +46,31 @@ pub struct ExternalPrefixSignal {
     pub query_blocks: usize,
 }
 
-/// Tokenizes requests for routing, preferring chat rendering over raw text.
+/// Whether the caller pre-tokenized the prompt (`input_ids` present and not
+/// null). Such a request is never re-rendered: its ids drive routing and the
+/// body is forwarded untouched, malformed values included, for the engine to
+/// validate.
+pub fn has_caller_input_ids(value: &serde_json::Value) -> bool {
+    value.get("input_ids").is_some_and(|v| !v.is_null())
+}
+
+/// Tokenizes a request for routing. Caller `input_ids` win; chat-rendered
+/// tokens may also be forwarded to the engine (the chat route decides); raw
+/// prompt tokens are routing-only.
 pub fn request_tokens_for(
     tokenizers: &TokenizerRegistry,
     model_id: &ModelId,
     value: &serde_json::Value,
 ) -> Option<RequestTokens> {
+    if has_caller_input_ids(value) {
+        // A flat u32 array (empty included) supplies routing tokens; anything
+        // else yields none, leaving validation to the engine.
+        let ids = serde::Deserialize::deserialize(&value["input_ids"]).ok()?;
+        return Some(RequestTokens {
+            ids,
+            rendered_from_chat: false,
+        });
+    }
     if tokenizers.has_chat_formatter(&model_id.0)
         && value.get("messages").is_some_and(|m| m.is_array())
     {

experimental/sgl-router/src/server/routes/chat.rs

@@ -10,7 +10,9 @@ use crate::policies::registry::{PdPoolResolver, PdResolveError};
 use crate::policies::selection::{
     select_decode_peer, select_prefill_worker, DecodeSelectionInputs, PrefillSelectionInputs,
 };
-use crate::policies::{request_tokens_for, ExternalPrefixSignal, RequestTokens};
+use crate::policies::{
+    has_caller_input_ids, request_tokens_for, ExternalPrefixSignal, RequestTokens,
+};
 use crate::proxy::sse::StreamEnd;
 use crate::server::app_context::AppContext;
 use crate::server::error::ApiError;
@@ -1409,7 +1411,8 @@ fn build_outgoing_body(
 /// Forward generated IDs only for request shapes verified against the engine.
 /// The engine uses `input_ids` verbatim, bypassing its chat-template processing.
 ///
-/// Exclude requests that may render differently with dynamo-render:
+/// Preserve caller-provided IDs. Exclude requests that may render differently
+/// with dynamo-render:
 /// - Non-leading system turns or consecutive users, which strict templates rewrite.
 /// - Historical `reasoning_content`, which may be injected into message content.
 /// - Tools and tool-call history, which the engine merges and normalizes
@@ -1422,7 +1425,8 @@ fn build_outgoing_body(
 /// overrides and default kwargs cannot be inferred from the request.
 /// `--disable-input-ids-forwarding` gates forwarding separately for such fleets.
 fn input_ids_safe_to_forward(value: &serde_json::Value) -> bool {
-    if request_has_tools(value)
+    if has_caller_input_ids(value)
+        || request_has_tools(value)
         || request_has_non_text_content(value)
         || request_has_reasoning_content(value)
         || request_has_role_rewrites(value)
@@ -1947,6 +1951,8 @@ mod tests {
             serde_json::json!({"messages":[{"role":"user","content":"hi"}],"task":"generate"}),
             serde_json::json!({"messages":[{"role":"user","content":"hi"}],"continue_final_message":true}),
             serde_json::json!({"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"partial"}]}),
+            serde_json::json!({"messages":[{"role":"user","content":"hi"}],"input_ids":[7, 8]}),
+            serde_json::json!({"messages":[{"role":"user","content":"hi"}],"input_ids":"bad"}),
         ];
         for b in blockers {
             assert!(
@@ -1964,6 +1970,7 @@ mod tests {
             "chat_template": null,
             "reasoning_effort": null,
             "chat_template_kwargs": null,
+            "input_ids": null,
             "continue_final_message": false
         })));
     }

experimental/sgl-router/src/tokenizer/chat_formatter.rs

@@ -3,8 +3,9 @@
 
 //! Chat rendering via dynamo-render for cache-aware routing and input ID forwarding.
 //!
-//! Engine-specific request normalization is not replicated here. The forwarding
-//! guard omits IDs for request shapes whose engine rendering has not been verified.
+//! Mirrors SGLang reasoning controls, assistant continuations, and DeepSeek-V4
+//! task selection before rendering. Forwarding remains guarded until parity
+//! has been verified for each request shape.
 
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -36,6 +37,9 @@ pub struct ChatFormatter {
     formatter: Arc<dyn OAIPromptFormatter>,
     /// Template context defaults; request `chat_template_kwargs` override them.
     defaults: ChatTemplateKwargs,
+    /// Stripped from a separately tokenized continuation prefix, as SGLang does.
+    bos_token: Option<String>,
+    is_deepseek_v4: bool,
 }
 
 impl ChatFormatter {
@@ -119,12 +123,15 @@ impl ChatFormatter {
         if template.chat_template.is_none() {
             return Ok(None);
         }
+        let bos_token = template.bos_tok();
         let PromptFormatter::OAI(formatter) =
             PromptFormatter::from_parts(template, ContextMixins::default(), true)
                 .context("compile chat template")?;
         Ok(Some(Self {
             formatter,
             defaults,
+            bos_token,
+            is_deepseek_v4: false,
         }))
     }
 
@@ -147,6 +154,15 @@ impl ChatFormatter {
             }
         });
         let PromptFormatter::OAI(formatter) = deepseek_formatter_for(&model_type, &name)?;
+        // Same rule dynamo-render applies for the name fallback: `deepseek` + one
+        // separator + a `v4` segment.
+        let version = name.strip_prefix("deepseek").unwrap_or("");
+        let version = version.strip_prefix(['-', '_', '.']).unwrap_or(version);
+        let is_deepseek_v4 = model_type
+            .as_deref()
+            .map_or(version.split(['-', '_', '.']).next() == Some("v4"), |t| {
+                t == "deepseek_v4"
+            });
         // Engine defaults: chat mode (`SGLANG_DEFAULT_THINKING=false`) and no
         // reasoning-effort preamble; dynamo-render defaults to thinking at high effort.
         let defaults = HashMap::from([
@@ -156,27 +172,112 @@ impl ChatFormatter {
         Some(Self {
             formatter,
             defaults,
+            bos_token: Some("<｜begin▁of▁sentence｜>".into()),
+            is_deepseek_v4,
         })
     }
 
+    /// Request kwargs plus the thinking/effort defaults SGLang derives from
+    /// `reasoning` / `reasoning_effort` (`protocol.py::normalize_reasoning_inputs`).
     fn template_kwargs(&self, request: &JsonValue) -> Result<ChatTemplateKwargs> {
         let mut kwargs: ChatTemplateKwargs = match request.get("chat_template_kwargs") {
             None | Some(JsonValue::Null) => ChatTemplateKwargs::new(),
             Some(v) => serde_json::from_value(v.clone()).context("chat_template_kwargs")?,
         };
+        // `reasoning.effort` overrides top-level `reasoning_effort`; `enabled`
+        // alone turns thinking on; any effort decides thinking by `!= "none"`.
+        // Explicit kwargs keep their values (the engine uses setdefault).
+        let reasoning = &request["reasoning"];
+        let mut thinking = None;
+        if reasoning.is_object() {
+            let enabled = match reasoning
+                .get("enabled")
+                .filter(|v| !v.is_null())
+                .or_else(|| reasoning.get("enable"))
+            {
+                Some(JsonValue::Bool(enabled)) => *enabled,
+                Some(JsonValue::String(s)) => {
+                    ["1", "true", "yes", "y", "on"].contains(&s.trim().to_lowercase().as_str())
+                }
+                _ => false,
+            };
+            if enabled {
+                thinking = Some(true);
+            }
+        }
+        let effort = [
+            reasoning.get("effort"),
+            reasoning.get("reasoning_effort"),
+            request.get("reasoning_effort"),
+        ]
+        .into_iter()
+        .flatten()
+        .find(|v| !v.is_null())
+        .cloned();
+        if let Some(effort) = &effort {
+            thinking = Some(effort != "none");
+        }
+        if let Some(thinking) = thinking {
+            kwargs.entry("thinking".into()).or_insert(thinking.into());
+            kwargs
+                .entry("enable_thinking".into())
+                .or_insert(thinking.into());
+        }
+        if let Some(mut effort) = effort {
+            // The engine's official V4 profile accepts only these; others map to
+            // no preamble.
+            if self.is_deepseek_v4 && !matches!(effort.as_str(), Some("low" | "high" | "max")) {
+                effort = "low".into();
+            }
+            kwargs.entry("reasoning_effort".into()).or_insert(effort);
+        }
         for (key, value) in &self.defaults {
             kwargs.entry(key.clone()).or_insert_with(|| value.clone());
         }
         Ok(kwargs)
     }
 
-    /// Render the prompt text dynamo-render produces for `request`.
-    pub fn render(&self, request: &JsonValue) -> Result<String> {
-        anyhow::ensure!(request["messages"].is_array(), "messages must be an array");
+    /// Rendered prompt plus the assistant continuation prefix SGLang tokenizes
+    /// separately (`_handle_last_assistant_message`).
+    fn render_parts(&self, request: &JsonValue) -> Result<(String, String)> {
         let kwargs = self.template_kwargs(request)?;
-        self.formatter
-            .render(&ChatRequest { request, kwargs })
-            .context("render chat template")
+        let continuing = request["continue_final_message"] == true;
+        let mut messages: Vec<JsonValue> = request["messages"]
+            .as_array()
+            .context("messages must be an array")?
+            .iter()
+            .map(engine_message)
+            .collect();
+        let mut prefix = String::new();
+        if let Some(last) = messages.last_mut().filter(|m| m["role"] == "assistant") {
+            if let Some(content) = last["content"].as_str() {
+                if continuing {
+                    prefix = content.to_owned();
+                    messages.pop();
+                } else {
+                    *last = serde_json::json!({"role": "user", "content": content});
+                }
+            }
+        }
+        if self.is_deepseek_v4 {
+            if let Some(task) = request.get("task").filter(|v| !v.is_null()).cloned() {
+                let message = messages
+                    .iter_mut()
+                    .rev()
+                    .find(|m| matches!(m["role"].as_str(), Some("user" | "developer")))
+                    .context("task requires a user or developer message")?;
+                message["task"] = task;
+            }
+        }
+        let prompt = self
+            .formatter
+            .render(&ChatRequest {
+                request,
+                messages,
+                kwargs,
+            })
+            .context("render chat template")?;
+        Ok((prompt, prefix))
     }
 
     pub fn encode(
@@ -184,8 +285,60 @@ impl ChatFormatter {
         tokenizer: &dynamo_tokenizers::Tokenizer,
         request: &JsonValue,
     ) -> Result<Vec<u32>> {
-        super::adapter::encode(tokenizer, &self.render(request)?)
+        let (prompt, prefix) = self.render_parts(request)?;
+        let mut ids = super::adapter::encode(tokenizer, &prompt)?;
+        if !prefix.is_empty() {
+            // SGLang encodes the assistant prefix separately and removes its leading BOS.
+            let mut suffix = super::adapter::encode(tokenizer, &prefix)?;
+            if let Some(bos) = self.bos_token.as_deref().filter(|s| !s.is_empty()) {
+                let bos = super::adapter::encode(tokenizer, bos)?;
+                if bos.len() == 1 && suffix.first() == bos.first() {
+                    suffix.remove(0);
+                }
+            }
+            ids.extend(suffix);
+        }
+        Ok(ids)
     }
+
+    /// Use `encode` for token ids to preserve continuation boundaries.
+    pub fn render(&self, request: &JsonValue) -> Result<String> {
+        let (prompt, prefix) = self.render_parts(request)?;
+        Ok(prompt + &prefix)
+    }
+}
+
+/// Message fields the engine's request schema keeps for non-user roles.
+const GENERIC_MESSAGE_KEYS: [&str; 7] = [
+    "role",
+    "content",
+    "tool_call_id",
+    "name",
+    "reasoning_content",
+    "tool_calls",
+    "tools",
+];
+
+/// Normalize a message as SGLang's pydantic dump does before rendering: roles
+/// lowercased, unknown and null fields dropped, `user` reduced to role and
+/// content, null content blanked.
+fn engine_message(message: &JsonValue) -> JsonValue {
+    let role = message["role"].as_str().unwrap_or_default().to_lowercase();
+    let mut out = serde_json::Map::new();
+    if role != "user" {
+        for key in GENERIC_MESSAGE_KEYS {
+            if let Some(v) = message.get(key).filter(|v| !v.is_null()) {
+                out.insert(key.into(), v.clone());
+            }
+        }
+    }
+    let content = match &message["content"] {
+        JsonValue::Null => "".into(),
+        content => content.clone(),
+    };
+    out.insert("role".into(), role.into());
+    out.insert("content".into(), content);
+    out.into()
 }
 
 /// `content` of an HF `AddedToken` object (`{"content": "<s>", "lstrip": ...}`).
@@ -199,10 +352,11 @@ fn added_token_content(token: &JsonValue) -> Option<String> {
 
 struct ChatRequest<'a> {
     request: &'a JsonValue,
+    /// Normalized copy of `request["messages"]`.
+    messages: Vec<JsonValue>,
     kwargs: ChatTemplateKwargs,
 }
 
-/// Mirrors dynamo-render's own impl for its wire type: request fields pass through.
 impl OAIChatLikeRequest for ChatRequest<'_> {
     fn model(&self) -> String {
         self.request["model"]
@@ -211,7 +365,7 @@ impl OAIChatLikeRequest for ChatRequest<'_> {
             .to_owned()
     }
     fn messages(&self) -> Value {
-        Value::from_serialize(&self.request["messages"])
+        Value::from_serialize(&self.messages)
     }
     fn tools(&self) -> Option<Value> {
         let tools = self.request.get("tools")?;
@@ -221,13 +375,20 @@ impl OAIChatLikeRequest for ChatRequest<'_> {
         if tools.as_array().is_none_or(|t| t.is_empty()) {
             return None;
         }
-        may_be_fix_tool_schema(tools.clone())
+        let mut tools = tools.clone();
+        // SGLang renders only the named tool for a function `tool_choice`.
+        if let Some(name) = self.request["tool_choice"]["function"]["name"].as_str() {
+            tools
+                .as_array_mut()?
+                .retain(|tool| tool["function"]["name"] == name);
+        }
+        may_be_fix_tool_schema(tools)
     }
     fn tool_choice(&self) -> Option<Value> {
         self.request.get("tool_choice").map(Value::from_serialize)
     }
     fn reasoning_effort(&self) -> Option<Value> {
-        self.request
+        self.kwargs
             .get("reasoning_effort")
             .map(Value::from_serialize)
     }
@@ -468,13 +629,139 @@ mod tests {
         );
     }
 
-    /// Request kwargs override the chat-mode default.
+    /// The engine's V4 encoder reads only `chat_template_kwargs.thinking`
+    /// (`serving_chat.py`); `enable_thinking` alone leaves chat mode, while
+    /// `reasoning_effort` sets both keys through the normalization above.
     #[test]
     fn v4_thinking_kwarg_overrides_chat_default() {
         let mut req = request(json!([{"role":"user","content":"ABCD"}]));
         req["chat_template_kwargs"] = json!({"thinking": true});
         let out = deepseek_v4().render(&req).unwrap();
         assert!(out.ends_with("<｜Assistant｜><think>"), "got: {out}");
+        req["chat_template_kwargs"] = json!({"enable_thinking": true});
+        let out = deepseek_v4().render(&req).unwrap();
+        assert!(out.ends_with("<｜Assistant｜></think>"), "got: {out}");
+        req["chat_template_kwargs"] = JsonValue::Null;
+        req["reasoning_effort"] = json!("high");
+        let out = deepseek_v4().render(&req).unwrap();
+        assert!(out.contains("<｜Assistant｜><think>"), "got: {out}");
+    }
+
+    #[test]
+    fn request_controls_reach_dynamo() {
+        let jinja = jinja(
+            json!({"chat_template": "{{ tools | tojson }} {{ thinking }} {{ reasoning_effort }}"}),
+        );
+        for formatter in [jinja, deepseek_v4()] {
+            let mut req = request(json!([{"role":"user","content":"hi"}]));
+            req["tools"] = json!([
+                {"type":"function","function":{"name":"first"}},
+                {"type":"function","function":{"name":"second"}}
+            ]);
+            req["tool_choice"] = json!({"type":"function","function":{"name":"second"}});
+            req["reasoning"] = json!({"effort":"high"});
+            let out = formatter.render(&req).unwrap();
+            assert!(out.contains("second") && !out.contains("first"));
+            assert!(out.contains("high") || out.contains("Reasoning Effort:"));
+            req["tool_choice"] = json!("none");
+            req["reasoning_effort"] = json!("none");
+            req["reasoning"] = JsonValue::Null;
+            let out = formatter.render(&req).unwrap();
+            assert!(!out.contains("first") && !out.contains("second"));
+            assert!(out.contains("False none") || out.ends_with("</think>"));
+        }
+    }
+
+    /// Mirrors `protocol.py::normalize_reasoning_inputs`: effort decides both
+    /// thinking keys via setdefault, so explicit kwargs win.
+    #[test]
+    fn reasoning_effort_sets_thinking_defaults_with_explicit_kwargs_winning() {
+        let enc = jinja(json!({
+            "chat_template": "{{ reasoning_effort }}:{{ thinking }}:{{ enable_thinking }}"
+        }));
+        let mut req = request(json!([{"role": "user", "content": "hi"}]));
+        req["reasoning_effort"] = json!("none");
+        assert_eq!(enc.render(&req).unwrap(), "none:False:False");
+        req["reasoning_effort"] = json!("high");
+        assert_eq!(enc.render(&req).unwrap(), "high:True:True");
+        req["chat_template_kwargs"] = json!({"enable_thinking": false});
+        assert_eq!(enc.render(&req).unwrap(), "high:True:False");
+        req["reasoning"] = json!({"effort": "low"});
+        assert_eq!(enc.render(&req).unwrap(), "low:True:False");
+        req["reasoning"] = json!({"enabled": "yes"});
+        req["reasoning_effort"] = JsonValue::Null;
+        req["chat_template_kwargs"] = JsonValue::Null;
+        assert_eq!(enc.render(&req).unwrap(), ":True:True");
+    }
+
+    /// Messages reach the template shaped like the engine's request schema.
+    #[test]
+    fn messages_match_engine_schema() {
+        let enc = jinja(json!({"chat_template": "{{ messages | tojson }}"}));
+        let out = enc
+            .render(&request(json!([
+                {"role": "User", "content": "hi", "name": "bob", "extra": 1},
+                {"role": "assistant", "name": "a", "tool_calls": null, "tool_call_id": "c1"},
+                {"role": "user", "content": "again"}
+            ])))
+            .unwrap();
+        let rendered: JsonValue = serde_json::from_str(&out).unwrap();
+        assert_eq!(
+            rendered,
+            json!([
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": "", "name": "a", "tool_call_id": "c1"},
+                {"role": "user", "content": "again"}
+            ])
+        );
+    }
+
+    #[test]
+    fn v4_task_uses_dynamo_task_tokens() {
+        let mut req = request(json!([{"role":"user","content":"example.com"}]));
+        for (task, suffix) in [
+            ("domain", "<｜domain｜>"),
+            ("action", "<｜Assistant｜></think><｜action｜>"),
+        ] {
+            req["task"] = json!(task);
+            assert_eq!(
+                deepseek_v4().render(&req).unwrap(),
+                format!("<｜begin▁of▁sentence｜><｜User｜>example.com{suffix}")
+            );
+        }
+        req["messages"] = json!([{"role":"system","content":"hi"}]);
+        assert!(deepseek_v4().render(&req).is_err());
+    }
+
+    #[test]
+    fn continuation_preserves_token_boundaries() {
+        let dir = tempfile::tempdir().unwrap();
+        let mut cfg: JsonValue =
+            serde_json::from_str(include_str!("../../tests/fixtures/tiny_tokenizer.json")).unwrap();
+        cfg["model"]["vocab"]["ab"] = json!(257);
+        cfg["model"]["merges"] = json!(["a b"]);
+        let path = dir.path().join("tokenizer.json");
+        std::fs::write(&path, cfg.to_string()).unwrap();
+        let tokenizer = super::super::adapter::load(path.to_str().unwrap()).unwrap();
+        let formatter = jinja(json!({"chat_template":"a", "bos_token":"<|endoftext|>"}));
+        let mut req = request(json!([
+            {"role":"user","content":"hi"}, {"role":"assistant","content":"b"}
+        ]));
+        req["continue_final_message"] = json!(true);
+        assert_eq!(formatter.encode(&tokenizer, &req).unwrap(), vec![97, 98]);
+        assert_eq!(
+            super::super::adapter::encode(&tokenizer, "ab").unwrap(),
+            vec![257]
+        );
+        req["messages"][1]["content"] = json!("<|endoftext|>b");
+        assert_eq!(formatter.encode(&tokenizer, &req).unwrap(), vec![97, 98]);
+        req["continue_final_message"] = json!(false);
+        req["messages"][1]["content"] = json!("b");
+        let out = deepseek_v4().render(&req).unwrap();
+        assert_eq!(
+            out,
+            "<｜begin▁of▁sentence｜><｜User｜>hi\n\nb<｜Assistant｜></think>"
+        );
     }
 
     /// The engine never renders `response_format` into the prompt.

experimental/sgl-router/tests/component/tokenizer/mod.rs

@@ -2,3 +2,4 @@
 // SPDX-License-Identifier: Apache-2.0
 
 mod parity;
+mod render_parity;

experimental/sgl-router/tests/component/tokenizer/render_parity.rs

@@ -0,0 +1,188 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//! Compare Dynamo prompt IDs with SGLang-generated fixtures using cached tokenizers.
+//! CI skips this matrix unless model snapshots are available.
+
+use serde::Deserialize;
+use sgl_router::config::{
+    ActiveLoadConfig, Config, DiscoveryBackend, ModelConfig, ObservabilityConfig, PolicyKind,
+    ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig,
+};
+use sgl_router::discovery::ModelId;
+use sgl_router::policies::request_tokens_for;
+use sgl_router::tokenizer::{adapter, chat_formatter::ChatFormatter, TokenizerRegistry};
+use std::path::PathBuf;
+
+#[derive(Deserialize)]
+struct Fixture {
+    model_id: String,
+    cases: Vec<Case>,
+}
+
+#[derive(Deserialize)]
+struct Case {
+    shape: String,
+    request: serde_json::Value,
+    expected_token_ids: Vec<u32>,
+}
+
+/// String-to-array conversion is a known parity gap, so these templates must
+/// opt out of forwarding. This fixture needs no cached model files.
+#[test]
+fn array_only_template_content_parity() {
+    let fixture: serde_json::Value =
+        serde_json::from_str(include_str!("../../fixtures/array_content_rendering.json")).unwrap();
+    let formatter = ChatFormatter::from_tokenizer_config(
+        serde_json::json!({"chat_template": fixture["chat_template"]}),
+        None,
+    )
+    .unwrap()
+    .unwrap();
+    let tokenizer = adapter::load("tests/fixtures/tiny_tokenizer.json").unwrap();
+    for case in fixture["cases"].as_array().unwrap() {
+        let request =
+            serde_json::json!({"messages": [{"role": "user", "content": case["content"]}]});
+        let ids = formatter.encode(&tokenizer, &request).unwrap();
+        assert_eq!(
+            serde_json::json!(ids) == case["engine_token_ids"],
+            case["content"].is_array(),
+            "{case}"
+        );
+    }
+}
+
+/// Replace every `YYYY-MM-DD` with a placeholder.
+///
+/// Templates that call `strftime_now` render the day the prompt is built, so a
+/// fixture captured earlier differs from today's render in the date alone. That
+/// is not drift: the engine consumes forwarded IDs verbatim and never re-renders.
+/// Masking keeps the rest of the prompt under exact comparison, and dates that
+/// come from the request render the same on both sides, so masking them is a
+/// no-op.
+fn mask_dates(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    let mut rest = text;
+    while let Some(c) = rest.chars().next() {
+        if starts_with_iso_date(rest) {
+            out.push_str("<DATE>");
+            rest = &rest[10..];
+        } else {
+            out.push(c);
+            rest = &rest[c.len_utf8()..];
+        }
+    }
+    out
+}
+
+fn starts_with_iso_date(text: &str) -> bool {
+    let b = text.as_bytes();
+    b.len() >= 10
+        && b[..4].iter().all(u8::is_ascii_digit)
+        && b[4] == b'-'
+        && b[5..7].iter().all(u8::is_ascii_digit)
+        && b[7] == b'-'
+        && b[8..10].iter().all(u8::is_ascii_digit)
+}
+
+fn snapshot_tokenizer(model_id: &str) -> Option<PathBuf> {
+    let hf_home = std::env::var("HF_HOME")
+        .ok()
+        .map(PathBuf::from)
+        .or_else(|| dirs::home_dir().map(|h| h.join(".cache/huggingface")))?;
+    let snapshots = hf_home
+        .join("hub")
+        .join(format!("models--{}", model_id.replace('/', "--")))
+        .join("snapshots");
+    std::fs::read_dir(snapshots)
+        .ok()?
+        .flatten()
+        .map(|e| e.path().join("tokenizer.json"))
+        .find(|p| p.is_file())
+}
+
+fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry {
+    let cfg = Config {
+        server: ServerConfig {
+            host: "0".into(),
+            port: 0,
+            ..Default::default()
+        },
+        observability: ObservabilityConfig::default(),
+        model: ModelConfig {
+            id: model_id.into(),
+            tokenizer_path: tokenizer_path.to_str().unwrap().into(),
+            disable_input_ids_forwarding: false,
+            policy: PolicyKind::RoundRobin,
+            decode_policy: Default::default(),
+            bucket_config: None,
+            circuit_breaker: None,
+            cache_aware: None,
+            affinity: None,
+            sticky: None,
+            fused: None,
+            eligibility: None,
+            sampling_overrides: Default::default(),
+        },
+        discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
+            urls: vec!["http://placeholder:0".into()],
+        }),
+        proxy: ProxyConfig::default(),
+        active_load: ActiveLoadConfig::default(),
+    };
+    TokenizerRegistry::load_from_config(&cfg).unwrap()
+}
+
+#[test]
+fn chat_render_parity_matrix() {
+    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/chat_render_parity");
+    let mut checked = 0;
+    for entry in std::fs::read_dir(&root).unwrap().flatten() {
+        let raw = std::fs::read_to_string(entry.path()).unwrap();
+        let fixture: Fixture = serde_json::from_str(&raw).unwrap();
+        let Some(tokenizer_path) = snapshot_tokenizer(&fixture.model_id) else {
+            eprintln!("skip {}: snapshot not cached", fixture.model_id);
+            continue;
+        };
+        let reg = registry(&fixture.model_id, tokenizer_path);
+        assert!(
+            reg.has_chat_formatter(&fixture.model_id),
+            "{}: no chat formatter resolved",
+            fixture.model_id
+        );
+        for case in &fixture.cases {
+            let tokens =
+                request_tokens_for(&reg, &ModelId(fixture.model_id.clone()), &case.request)
+                    .unwrap_or_else(|| panic!("{}/{}: no tokens", fixture.model_id, case.shape));
+            assert!(
+                tokens.rendered_from_chat,
+                "{}/{}: fell back to raw text",
+                fixture.model_id, case.shape
+            );
+            if tokens.ids != case.expected_token_ids {
+                // Decode with special tokens kept, so a difference in them still fails.
+                let tokenizer = reg.get(&fixture.model_id).unwrap();
+                let rendered = adapter::decode_complete(&tokenizer, &tokens.ids, false).unwrap();
+                let expected =
+                    adapter::decode_complete(&tokenizer, &case.expected_token_ids, false).unwrap();
+                assert_eq!(
+                    mask_dates(&rendered),
+                    mask_dates(&expected),
+                    "DRIFT on {}/{}",
+                    fixture.model_id,
+                    case.shape
+                );
+                eprintln!(
+                    "{}/{}: date drift only; fixture captured on another day",
+                    fixture.model_id, case.shape
+                );
+            }
+            checked += 1;
+        }
+    }
+    assert!(
+        checked > 0,
+        "no cached model snapshots; parity was not checked"
+    );
+    eprintln!("chat render parity: {checked} cases checked");
+}

experimental/sgl-router/tests/fixtures/array_content_rendering.json

@@ -0,0 +1,4 @@
+{"chat_template": "{% for message in messages %}{{ message.role }}:{% for part in message.content %}{% if part.type == 'text' %}{{ part.text }}{% elif part.type == 'image' %}<image>{% endif %}{% endfor %};{% endfor %}{% if add_generation_prompt %}assistant:{% endif %}", "cases": [
+{"content": "hello", "engine_token_ids": [117, 115, 101, 114, 58, 59, 97, 115, 115, 105, 115, 116, 97, 110, 116, 58]},
+{"content": [{"type": "text", "text": "hello"}], "engine_token_ids": [117, 115, 101, 114, 58, 104, 101, 108, 108, 111, 59, 97, 115, 115, 105, 115, 116, 97, 110, 116, 58]}
+]}

experimental/sgl-router/tests/fixtures/chat_render_parity/deepseek-v4-flash.json

@@ -0,0 +1,7 @@
+{"model_id": "deepseek-ai/DeepSeek-V4-Flash", "cases": [
+{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [0, 128803, 63006, 19346, 295, 834, 10175, 16, 128804, 128822]},
+{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [0, 3476, 477, 259, 10935, 16, 128803, 3085, 344, 223, 20, 13, 20, 33, 128804, 128822]},
+{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [0, 3476, 477, 259, 10935, 16, 128803, 23166, 128804, 128822, 19923, 3, 1730, 588, 342, 1694, 33, 1, 128803, 52, 3319, 90902, 294, 270, 2831, 28, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 128804, 128822]},
+{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [0, 128803, 50012, 943, 436, 16, 128804, 128822]},
+{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [0, 128803, 50012, 943, 436, 16, 128804, 128821]}
+]}

experimental/sgl-router/tests/fixtures/chat_render_parity/glm-5.2.json

@@ -0,0 +1,7 @@
+{"model_id": "zai-org/GLM-5.2-FP8", "cases": [
+{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154827, 45494, 15576, 304, 825, 11646, 13, 154828, 154841]},
+{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154826, 2610, 525, 50205, 13, 154827, 3838, 374, 220, 17, 10, 17, 30, 154828, 154841]},
+{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154826, 2610, 525, 50205, 13, 154827, 13041, 154828, 154841, 154842, 9703, 0, 2585, 646, 358, 1492, 30, 154827, 84140, 1242, 963, 315, 279, 3119, 25, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 220, 154828, 154841]},
+{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [154822, 154824, 154827, 38479, 911, 432, 13, 154828, 154841, 154842]},
+{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154827, 38479, 911, 432, 13, 154828, 154841]}
+]}

experimental/sgl-router/tests/fixtures/chat_render_parity/gpt-oss-20b.json

@@ -0,0 +1,7 @@
+{"model_id": "openai/gpt-oss-20b", "cases": [
+{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 62316, 5911, 306, 1001, 21872, 13, 200007, 200006, 173781]},
+{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 77944, 200008, 2, 68406, 279, 3575, 553, 28410, 364, 200007, 200006, 1428, 200008, 4827, 382, 220, 17, 10, 17, 30, 200007, 200006, 173781]},
+{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 77944, 200008, 2, 68406, 279, 3575, 553, 28410, 364, 200007, 200006, 1428, 200008, 12194, 200007, 200006, 173781, 200005, 17196, 200008, 13225, 0, 3253, 665, 357, 1652, 30, 200007, 200006, 1428, 200008, 198128, 328, 290, 3496, 25, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 220, 200007, 200006, 173781]},
+{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 42421, 1078, 480, 13, 200007, 200006, 173781]},
+{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 42421, 1078, 480, 13, 200007, 200006, 173781]}
+]}

experimental/sgl-router/tests/fixtures/chat_render_parity/minimax-m3.json

@@ -0,0 +1,7 @@
+{"model_id": "MiniMaxAI/MiniMax-M3", "cases": [
+{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 66938, 13182, 296, 841, 14997, 46, 200020, 10, 200019, 1361, 10]},
+{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 6000, 101, 46, 200020, 10, 200019, 3995, 10, 3376, 355, 32, 50, 43, 50, 63, 200020, 10, 200019, 1361, 10]},
+{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 6000, 101, 46, 200020, 10, 200019, 3995, 10, 22700, 200020, 10, 200019, 1361, 10, 200060, 19739, 33, 2329, 566, 343, 1576, 63, 200020, 10, 200019, 3995, 10, 96530, 300, 275, 2748, 58, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 32, 200020, 10, 200019, 1361, 10]},
+{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 38460, 894, 412, 46, 200020, 10, 200019, 1361, 10]},
+{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 38460, 894, 412, 46, 200020, 10, 200019, 1361, 10]}
+]}

experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3-8b.json

@@ -0,0 +1,7 @@
+{"model_id": "Qwen/Qwen3-8B", "cases": [
+{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [151644, 872, 198, 45764, 15588, 304, 825, 11652, 13, 151645, 198, 151644, 77091, 198]},
+{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [151644, 8948, 198, 2610, 525, 50537, 13, 151645, 198, 151644, 872, 198, 3838, 374, 220, 17, 10, 17, 30, 151645, 198, 151644, 77091, 198]},
+{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [151644, 8948, 198, 2610, 525, 50537, 13, 151645, 198, 151644, 872, 198, 13048, 151645, 198, 151644, 77091, 198, 9707, 0, 2585, 646, 358, 1492, 30, 151645, 198, 151644, 872, 198, 84836, 1242, 963, 315, 279, 3119, 25, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 220, 151645, 198, 151644, 77091, 198]},
+{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [151644, 872, 198, 38687, 911, 432, 13, 151645, 198, 151644, 77091, 198, 151667, 271, 151668, 271]},
+{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [151644, 872, 198, 38687, 911, 432, 13, 151645, 198, 151644, 77091, 198]}
+]}

experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.5-27b.json

@@ -0,0 +1,7 @@
+{"model_id": "Qwen/Qwen3.5-27B", "cases": [
+{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [248045, 846, 198, 44240, 15131, 303, 799, 11316, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
+{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [248045, 8678, 198, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 3710, 369, 220, 17, 10, 17, 30, 248046, 198, 248045, 74455, 198, 248068, 198]},
+{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [248045, 8678, 198, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 12675, 248046, 198, 248045, 74455, 198, 9419, 0, 2500, 628, 353, 1438, 30, 248046, 198, 248045, 846, 198, 81911, 168699, 314, 279, 3019, 25, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
+{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271]},
+{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}
+]}

experimental/sgl-router/tests/fixtures/chat_render_parity/qwen3.8-27b.json

@@ -0,0 +1,7 @@
+{"model_id": "Qwen/Qwen3.8-27B", "cases": [
+{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 44240, 15131, 303, 799, 11316, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
+{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 271, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 3710, 369, 220, 17, 10, 17, 30, 248046, 198, 248045, 74455, 198, 248068, 198]},
+{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 271, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 12675, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271, 9419, 0, 2500, 628, 353, 1438, 30, 248046, 198, 248045, 846, 198, 81911, 168699, 314, 279, 3019, 25, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
+{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271]},
+{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}
+]}

experimental/sgl-router/tests/proxy/cache_aware_input_ids.rs

@@ -168,3 +168,63 @@ async fn multimodal_request_omits_input_ids() {
         "multimodal requests must not forward input_ids; got {body}"
     );
 }
+
+/// Caller-supplied `input_ids` are never re-rendered or replaced: a flat u32
+/// array (empty included) drives routing, anything else yields no routing
+/// tokens, and the body reaches the engine byte-for-byte for validation.
+#[tokio::test]
+async fn caller_input_ids_are_used_for_routing_and_preserved() {
+    let mock = MockWorker::start(vec![]).await;
+    let ctx = build_ctx(mock.url.clone());
+    for (ids, expected) in [
+        (json!([7, 8]), Some(vec![7, 8])),
+        (json!([]), Some(vec![])),
+        (json!([7, -1]), None),
+        (json!("bad"), None),
+    ] {
+        let request = json!({
+            "model": MODEL,
+            "messages": [{"role": "user", "content": "hi"}],
+            "input_ids": ids,
+        });
+        let tokens = sgl_router::policies::request_tokens_for(
+            &ctx.tokenizers,
+            &ModelId(MODEL.into()),
+            &request,
+        );
+        assert!(!tokens.as_ref().is_some_and(|t| t.rendered_from_chat));
+        assert_eq!(tokens.map(|t| t.ids), expected, "input_ids: {ids}");
+        assert_eq!(
+            send(Arc::clone(&ctx), request.clone()).await,
+            StatusCode::OK
+        );
+        assert_eq!(captured(&mock), request, "body must be forwarded untouched");
+    }
+    // Bypasses are not rendering failures.
+    assert!(!ctx
+        .metrics
+        .render()
+        .contains("sgl_router_ingress_tokenize_errors_total{"));
+}
+
+/// `input_ids: null` is the same as absent: the router renders and forwards.
+#[tokio::test]
+async fn null_input_ids_keep_normal_rendering() {
+    let mock = MockWorker::start(vec![]).await;
+    let ctx = build_ctx(mock.url.clone());
+    let status = send(
+        ctx,
+        json!({
+            "model": MODEL,
+            "messages": [{"role": "user", "content": "hello there friend"}],
+            "input_ids": null,
+        }),
+    )
+    .await;
+    assert_eq!(status, StatusCode::OK);
+    let body = captured(&mock);
+    assert!(
+        body["input_ids"].as_array().is_some_and(|a| !a.is_empty()),
+        "null input_ids must not suppress rendering; got {body}"
+    );
+}

experimental/sgl-router/tests/scripts/generate_array_content_fixture.py

@@ -0,0 +1,50 @@
+"""Regenerate the array-only template regression using SGLang's content processor.
+
+Run: PYTHONPATH=../../python python tests/scripts/generate_array_content_fixture.py
+"""
+
+import json
+from pathlib import Path
+
+from transformers import PreTrainedTokenizerFast
+
+from sglang.srt.parser.jinja_template_utils import (
+    detect_jinja_template_content_format,
+    process_content_for_template_format,
+)
+
+ROOT = Path(__file__).resolve().parents[1] / "fixtures"
+TEMPLATE = (
+    "{% for message in messages %}{{ message.role }}:"
+    "{% for part in message.content %}"
+    "{% if part.type == 'text' %}{{ part.text }}"
+    "{% elif part.type == 'image' %}<image>{% endif %}"
+    "{% endfor %};{% endfor %}"
+    "{% if add_generation_prompt %}assistant:{% endif %}"
+)
+
+
+def main():
+    tokenizer = PreTrainedTokenizerFast(
+        tokenizer_file=str(ROOT / "tiny_tokenizer.json")
+    )
+    tokenizer.chat_template = TEMPLATE
+    content_format = detect_jinja_template_content_format(TEMPLATE)
+    assert content_format == "openai"
+    cases = []
+    for content in ["hello", [{"type": "text", "text": "hello"}]]:
+        message = process_content_for_template_format(
+            {"role": "user", "content": content}, content_format, [], [], [], []
+        )
+        ids = tokenizer.apply_chat_template(
+            [message], return_dict=False, add_generation_prompt=True
+        )
+        cases.append(json.dumps({"content": content, "engine_token_ids": ids}))
+    cases_json = ",\n".join(cases)
+    (ROOT / "array_content_rendering.json").write_text(
+        f'{{"chat_template": {json.dumps(TEMPLATE)}, "cases": [\n{cases_json}\n]}}\n'
+    )
+
+
+if __name__ == "__main__":
+    main()

experimental/sgl-router/tests/scripts/generate_chat_render_parity.py

@@ -0,0 +1,158 @@
+"""Generate reference prompt IDs with SGLang helpers and cached model tokenizers.
+
+Run: python tests/scripts/generate_chat_render_parity.py
+"""
+
+import copy
+import json
+import pathlib
+import sys
+
+from transformers.utils.hub import cached_file
+
+from sglang.srt.entrypoints.openai import encoding_dsv4
+from sglang.srt.entrypoints.openai.chat_encoding import (
+    resolve_dsv4_reasoning_effort_profile,
+)
+from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
+from sglang.srt.entrypoints.openai.serving_chat import (
+    ThinkingMode,
+    normalize_assistant_tool_call_arguments,
+    normalize_tool_content,
+)
+from sglang.srt.parser.jinja_template_utils import (
+    detect_jinja_template_content_format,
+    process_content_for_template_format,
+)
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
+
+ROOT = pathlib.Path(__file__).resolve().parents[1] / "fixtures" / "chat_render_parity"
+
+MODELS = {
+    "qwen3-8b": "Qwen/Qwen3-8B",
+    "qwen3.5-27b": "Qwen/Qwen3.5-27B",
+    "qwen3.8-27b": "Qwen/Qwen3.8-27B",
+    "gpt-oss-20b": "openai/gpt-oss-20b",
+    "glm-5.2": "zai-org/GLM-5.2-FP8",
+    "minimax-m3": "MiniMaxAI/MiniMax-M3",
+    "deepseek-v4-flash": "deepseek-ai/DeepSeek-V4-Flash",
+}
+
+LONG = "Résumé of the plan: " + "第一步，收集数据。Then we iterate. " * 8
+
+SHAPES = {
+    "user_only": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]},
+    "system_user": {
+        "messages": [
+            {"role": "system", "content": "You are terse."},
+            {"role": "user", "content": "What is 2+2?"},
+        ]
+    },
+    "multi_turn": {
+        "messages": [
+            {"role": "system", "content": "You are terse."},
+            {"role": "user", "content": "Hi"},
+            {"role": "assistant", "content": "Hello! How can I help?"},
+            {"role": "user", "content": LONG},
+        ]
+    },
+    "thinking_off": {
+        "messages": [{"role": "user", "content": "Think about it."}],
+        "chat_template_kwargs": {"enable_thinking": False, "thinking": False},
+    },
+    "thinking_on": {
+        "messages": [{"role": "user", "content": "Think about it."}],
+        "chat_template_kwargs": {"enable_thinking": True, "thinking": True},
+    },
+}
+
+
+def snapshot_dir(model_id):
+    return pathlib.Path(
+        cached_file(model_id, "config.json", local_files_only=True)
+    ).parent
+
+
+def engine_messages(request, content_format):
+    messages = [m.model_dump() for m in request.messages]
+    for message in messages:
+        normalize_assistant_tool_call_arguments(message)
+    out = []
+    for msg in copy.deepcopy(messages):
+        if msg.get("content") is None:
+            msg["content"] = ""
+        processed = process_content_for_template_format(
+            msg, content_format, [], [], [], []
+        )
+        processed["content"] = normalize_tool_content(
+            processed["role"], processed.get("content")
+        )
+        out.append(processed)
+    return out
+
+
+def engine_prompt_ids(model_id, tok, request):
+    """Mirror `_apply_jinja_template` for a text-only request without tools."""
+    snapshot = snapshot_dir(model_id)
+    model_type = json.load(open(snapshot / "config.json")).get("model_type")
+    if model_type == "deepseek_v4":
+        messages = engine_messages(request, "string")
+        if messages[0]["role"] != "system":
+            messages.insert(0, {"role": "system", "content": ""})
+        thinking = (request.chat_template_kwargs or {}).get("thinking", False)
+        text = encoding_dsv4.encode_messages(
+            messages,
+            thinking_mode=ThinkingMode.THINKING if thinking else ThinkingMode.CHAT,
+            reasoning_effort=None,
+            reasoning_effort_profile=resolve_dsv4_reasoning_effort_profile(
+                model_path=str(snapshot)
+            ),
+        )
+        return tok.encode(text)
+
+    template = tok.chat_template
+    if not isinstance(template, str):
+        raise RuntimeError(f"{model_id}: named template dict is not supported here")
+    messages = engine_messages(request, detect_jinja_template_content_format(template))
+    extra = {}
+    if request.reasoning_effort is not None:
+        extra["reasoning_effort"] = request.reasoning_effort
+    if request.chat_template_kwargs:
+        extra.update(request.chat_template_kwargs)
+    rendered = tok.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+        tools=None,
+        return_dict=False,
+        **extra,
+    )
+    encode_kwargs = {"add_special_tokens": False} if len(tok.encode("")) > 0 else {}
+    return tok.encode(rendered, **encode_kwargs)
+
+
+def main():
+    ROOT.mkdir(parents=True, exist_ok=True)
+    for slug, model_id in MODELS.items():
+        try:
+            snapshot = snapshot_dir(model_id)
+        except Exception as e:
+            print(f"skip {model_id}: {e}", file=sys.stderr)
+            continue
+        tok = get_tokenizer(str(snapshot))
+        cases = []
+        for shape, body in SHAPES.items():
+            request = ChatCompletionRequest(model=model_id, **copy.deepcopy(body))
+            ids = engine_prompt_ids(model_id, tok, request)
+            cases.append({"shape": shape, "request": body, "expected_token_ids": ids})
+        out = ROOT / f"{slug}.json"
+        lines = [json.dumps(case, ensure_ascii=False) for case in cases]
+        out.write_text(
+            '{"model_id": %s, "cases": [\n%s\n]}\n'
+            % (json.dumps(model_id), ",\n".join(lines))
+        )
+        print(f"wrote {out} ({len(cases)} cases)")
+
+
+if __name__ == "__main__":
+    main()