From b4584ab83884ff40a4e5a77f0ea4ef98c038d3b9 Mon Sep 17 00:00:00 2001 From: Kan Wu Date: Sat, 26 Sep 2026 20:01:17 -0700 Subject: [PATCH] [sgl-router] Take SGLang's render defaults: --default-chat-template-kwargs and thinking/effort envs (#41221) Co-authored-by: Kan Wu Co-authored-by: Claude Opus 5.5 --- experimental/sgl-router/README.md | 23 +- experimental/sgl-router/src/config/cli.rs | 62 ++++- experimental/sgl-router/src/config/mod.rs | 1 + experimental/sgl-router/src/config/types.rs | 2 + .../sgl-router/src/policies/factory.rs | 1 + .../sgl-router/src/server/app_context.rs | 1 + .../sgl-router/src/server/routes/models.rs | 1 + .../sgl-router/src/server/routes/tokenize.rs | 1 + .../src/tokenizer/chat_formatter.rs | 224 +++++++++++++++++- .../sgl-router/src/tokenizer/deepseek.rs | 63 +++-- experimental/sgl-router/src/tokenizer/mod.rs | 8 +- .../sgl-router/src/workers/manager.rs | 1 + .../tests/component/discovery/static_urls.rs | 1 + .../component/tokenizer/render_parity.rs | 1 + .../tests/fixtures/deepseek/v41.json | 14 +- .../tests/fixtures/kimi_k3/prompts.json | 2 +- .../sgl-router/tests/proxy/bucket_routing.rs | 1 + .../sgl-router/tests/proxy/chat_routing.rs | 1 + .../tests/proxy/common/cache_aware_fixture.rs | 1 + .../sgl-router/tests/proxy/failover.rs | 1 + .../tests/proxy/graceful_shutdown.rs | 1 + .../tests/proxy/header_forwarding.rs | 1 + .../tests/proxy/pd_bootstrap_injection.rs | 1 + .../tests/proxy/pd_pool_isolation.rs | 1 + .../tests/proxy/pd_protocol_binding.rs | 1 + .../tests/proxy/roundrobin_input_ids.rs | 1 + .../tests/proxy/shared_prefill_admission.rs | 1 + .../tests/proxy/sticky_input_ids.rs | 1 + .../sgl-router/tests/proxy/sticky_routing.rs | 1 + .../sgl-router/tests/proxy/timeout.rs | 1 + .../tests/scripts/generate_kimi_parity.py | 6 + 31 files changed, 377 insertions(+), 49 deletions(-) diff --git a/experimental/sgl-router/README.md b/experimental/sgl-router/README.md index e67b394435..5281032887 100644 --- a/experimental/sgl-router/README.md +++ b/experimental/sgl-router/README.md @@ -205,8 +205,13 @@ rendered tokens to the engine as `input_ids`, retaining the original messages, so the engine skips re-tokenizing. Every other request shape is rendered for routing only: the router renders with dynamo-render and does not replicate SGLang's request normalization, so forwarding is enabled shape by shape as -parity is verified. Use matching model files on the router and workers; worker -template overrides and default kwargs are not observable from the request. +parity is verified. Use matching model files on the router and workers, and set +the same `--default-chat-template-kwargs`, `SGLANG_DEFAULT_THINKING`, +`SGLANG_DSV4_REASONING_EFFORT`, and `SGLANG_DSV41_REASONING_EFFORT` on both. +The router reads these render defaults from its own configuration and environment; +it does not discover the workers' settings. Point `--tokenizer-path` at the workers' +model snapshot so the V4 effort profile is read from the same +`encoding/encoding_dsv4.py`. Set `--disable-input-ids-forwarding` for this router's model when worker-side rendering has not been verified to match. This disables router-generated IDs @@ -214,13 +219,13 @@ for every routing policy; cache-aware routing still renders and tokenizes locally, and the original messages reach the workers for engine processing. Caller-supplied `input_ids` remain caller-owned and pass through unchanged. -Forwarding logs its assumptions at startup. In particular, disable it for -`SGLANG_DEFAULT_THINKING=true`, a non-default `SGLANG_DSV4_REASONING_EFFORT`, -worker parser overrides such as `--tool-call-parser deepseekv32` that select a -native encoder over a shipped template, or conversation templates with stop -strings (the engine's `input_ids` path skips those template stops). These worker -settings are not inferred from the router's environment. Disabling forwarding preserves -engine behavior but does not establish parity for local routing hashes. +Forwarding logs its assumptions at startup. In particular, disable it when the +router's render defaults differ from the workers', for worker template overrides +not reflected in the router's model files, for worker parser overrides such as +`--tool-call-parser deepseekv32` that select a native encoder over a shipped +template, or conversation templates with stop strings (the engine's `input_ids` +path skips those template stops). Disabling forwarding preserves engine behavior +but does not establish parity for local routing hashes. Also set `--disable-input-ids-forwarding` for array-only templates: Dynamo may wrap string content into arrays differently from the worker. The pinned Dynamo renderer does not expose diff --git a/experimental/sgl-router/src/config/cli.rs b/experimental/sgl-router/src/config/cli.rs index 8fc9aa73ea..a8981ea41a 100644 --- a/experimental/sgl-router/src/config/cli.rs +++ b/experimental/sgl-router/src/config/cli.rs @@ -3,7 +3,7 @@ //! Grouped CLI options and conversion into a validated [`Config`]. -use anyhow::{anyhow, ensure, Result}; +use anyhow::{anyhow, ensure, Context, Result}; use clap::Parser; use std::num::NonZeroU32; @@ -66,10 +66,14 @@ pub struct ModelArgs { pub tokenizer_path: Option, /// Disable generated input_ids; workers tokenize messages, while routing still renders locally. - /// Use for worker-only thinking defaults, parser/template overrides, or template stop strings. + /// Use for worker parser/template overrides or template stop strings. #[arg(long)] pub disable_input_ids_forwarding: bool, + /// Same as SGLang's --default-chat-template-kwargs; must match the workers. + #[arg(long, value_name = "JSON")] + pub default_chat_template_kwargs: Option, + /// Fleet sampling defaults as JSON, e.g. {"temperature": 1, "top_p": 0.95}. /// Accepts temperature, top_p, top_k, min_p, repetition_penalty, /// frequency_penalty, presence_penalty, and n. Numeric values fill absent @@ -383,6 +387,14 @@ impl Cli { }) .transpose()? .unwrap_or_default(); + let default_chat_template_kwargs = self + .model + .default_chat_template_kwargs + .as_deref() + .map(serde_json::from_str) + .transpose() + .context("--default-chat-template-kwargs must be a JSON object")? + .unwrap_or_default(); let config = Config { server: ServerConfig { @@ -412,6 +424,7 @@ impl Cli { fused, eligibility, sampling_overrides, + default_chat_template_kwargs, }, discovery, proxy: ProxyConfig { @@ -2396,6 +2409,51 @@ mod tests { .to_string(); assert!(err.contains("unknown parameter"), "got: {err}"); } + + #[test] + fn default_chat_template_kwargs_reaches_the_model_config() { + let kwargs = serde_json::json!({"thinking": true, "reasoning_effort": "high"}); + let config = into_config_owned(with_model(&[ + "--worker-urls", + "http://x:30000", + "--default-chat-template-kwargs", + &kwargs.to_string(), + ])) + .unwrap(); + assert_eq!( + serde_json::to_value(config.model.default_chat_template_kwargs).unwrap(), + kwargs + ); + + let defaults = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap(); + assert!(defaults.model.default_chat_template_kwargs.is_empty()); + } + + #[test] + fn malformed_default_chat_template_kwargs_fails_the_launch() { + for kwargs in [ + r#"{"thinking": true"#, + "[]", + "null", + "true", + "42", + r#""high""#, + ] { + let err = into_config_owned(with_model(&[ + "--worker-urls", + "http://x:30000", + "--default-chat-template-kwargs", + kwargs, + ])) + .unwrap_err() + .to_string(); + assert!( + err.contains("--default-chat-template-kwargs must be a JSON object"), + "kwargs={kwargs}, got: {err}" + ); + } + } + #[test] fn input_ids_forwarding_can_be_disabled_for_the_model() { let defaults = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap(); diff --git a/experimental/sgl-router/src/config/mod.rs b/experimental/sgl-router/src/config/mod.rs index 67def2ec27..bebf3c88c6 100644 --- a/experimental/sgl-router/src/config/mod.rs +++ b/experimental/sgl-router/src/config/mod.rs @@ -236,6 +236,7 @@ mod tests { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: urls.iter().map(|s| s.to_string()).collect(), diff --git a/experimental/sgl-router/src/config/types.rs b/experimental/sgl-router/src/config/types.rs index 317dfb7b59..bdda3164ab 100644 --- a/experimental/sgl-router/src/config/types.rs +++ b/experimental/sgl-router/src/config/types.rs @@ -330,6 +330,8 @@ pub struct ModelConfig { pub eligibility: Option, /// Fleet sampling defaults and conflict behavior. See [`SamplingOverrides`]. pub sampling_overrides: SamplingOverrides, + /// Worker `--default-chat-template-kwargs`, applied when rendering. + pub default_chat_template_kwargs: crate::tokenizer::chat_formatter::ChatTemplateKwargs, } /// External KV Indexer client settings. diff --git a/experimental/sgl-router/src/policies/factory.rs b/experimental/sgl-router/src/policies/factory.rs index 82a69604e0..37f02ffcfc 100644 --- a/experimental/sgl-router/src/policies/factory.rs +++ b/experimental/sgl-router/src/policies/factory.rs @@ -348,6 +348,7 @@ mod tests { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/src/server/app_context.rs b/experimental/sgl-router/src/server/app_context.rs index 2aa4e4a2f6..3538a6e9e4 100644 --- a/experimental/sgl-router/src/server/app_context.rs +++ b/experimental/sgl-router/src/server/app_context.rs @@ -196,6 +196,7 @@ impl AppContext { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: crate::config::DiscoveryBackend::StaticUrls( crate::config::StaticUrlsDiscoveryConfig { diff --git a/experimental/sgl-router/src/server/routes/models.rs b/experimental/sgl-router/src/server/routes/models.rs index 8b1a2ecf77..bdf290ec18 100644 --- a/experimental/sgl-router/src/server/routes/models.rs +++ b/experimental/sgl-router/src/server/routes/models.rs @@ -61,6 +61,7 @@ mod tests { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }; let app = crate::server::app::build_router(std::sync::Arc::new(ctx)); let res = app diff --git a/experimental/sgl-router/src/server/routes/tokenize.rs b/experimental/sgl-router/src/server/routes/tokenize.rs index 9dd03714af..7b364d98d6 100644 --- a/experimental/sgl-router/src/server/routes/tokenize.rs +++ b/experimental/sgl-router/src/server/routes/tokenize.rs @@ -129,6 +129,7 @@ mod tests { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: crate::config::DiscoveryBackend::StaticUrls( crate::config::StaticUrlsDiscoveryConfig { diff --git a/experimental/sgl-router/src/tokenizer/chat_formatter.rs b/experimental/sgl-router/src/tokenizer/chat_formatter.rs index acf11dc38b..bd630affe3 100644 --- a/experimental/sgl-router/src/tokenizer/chat_formatter.rs +++ b/experimental/sgl-router/src/tokenizer/chat_formatter.rs @@ -42,6 +42,8 @@ pub struct ChatFormatter { bos_token: Option, deepseek: Option, is_kimi_k3: bool, + /// Worker defaults, merged before model-specific effort conversion. + worker_defaults: ChatTemplateKwargs, } impl ChatFormatter { @@ -163,6 +165,7 @@ impl ChatFormatter { bos_token, deepseek: None, is_kimi_k3: false, + worker_defaults: ChatTemplateKwargs::new(), })) } @@ -178,6 +181,7 @@ impl ChatFormatter { bos_token: Some("[BOS]".into()), deepseek: None, is_kimi_k3: true, + worker_defaults: ChatTemplateKwargs::new(), }) } @@ -205,10 +209,12 @@ impl ChatFormatter { .map_or(name.contains("v4.1") || name.contains("v41"), |t| { t == "deepseek_v41" }); - // Engine defaults: chat mode (`SGLANG_DEFAULT_THINKING=false`) and no - // reasoning-effort preamble; dynamo-render defaults to thinking at high effort. + // Engine defaults: `SGLANG_DEFAULT_THINKING` and no reasoning-effort + // preamble; dynamo-render defaults to thinking at high effort. + let thinking = std::env::var("SGLANG_DEFAULT_THINKING") + .is_ok_and(|v| ["true", "1", "yes", "y"].contains(&v.to_lowercase().as_str())); let defaults = HashMap::from([ - ("thinking".into(), false.into()), + ("thinking".into(), thinking.into()), ("reasoning_effort".into(), "low".into()), ]); Some(Self { @@ -221,6 +227,23 @@ impl ChatFormatter { is_deepseek_v4.then(|| super::deepseek::Encoder::V4(Default::default())) }, is_kimi_k3: false, + worker_defaults: ChatTemplateKwargs::new(), + }) + } + + /// Apply the workers' `--default-chat-template-kwargs`; they fill keys the + /// request leaves unset, and a default `reasoning_effort` acts as the request's. + pub fn with_defaults(mut self, defaults: &ChatTemplateKwargs) -> Self { + self.worker_defaults = defaults.clone(); + self + } + + fn effective_effort(&self, request: &JsonValue) -> Option { + super::deepseek::request_effort(request).or_else(|| { + self.worker_defaults + .get("reasoning_effort") + .filter(|v| !v.is_null()) + .cloned() }) } @@ -270,7 +293,14 @@ impl ChatFormatter { .entry("enable_thinking".into()) .or_insert(thinking.into()); } - if let Some(effort) = effort { + // serving_chat first pops the request kwarg into request.reasoning_effort, + // then merges worker defaults. Jinja receives those merged kwargs over + // the effective effort; Kimi derives thinking_effort only if still absent. + kwargs.remove("reasoning_effort"); + for (key, value) in &self.worker_defaults { + kwargs.entry(key.clone()).or_insert_with(|| value.clone()); + } + if let Some(effort) = self.effective_effort(request) { if self.is_kimi_k3 { if matches!(effort.as_str(), Some("low" | "high" | "max")) { kwargs.entry("thinking_effort".into()).or_insert(effort); @@ -332,8 +362,9 @@ impl ChatFormatter { } } if let Some(profile) = self.deepseek { + let effort = self.effective_effort(request); return Ok(( - RenderedPrompt::text(profile.render(request, messages, &kwargs)?), + RenderedPrompt::text(profile.render(request, messages, &kwargs, effort)?), prefix, )); } @@ -785,6 +816,189 @@ mod tests { assert_eq!(enc.render(&req).unwrap(), ":True:True"); } + #[test] + fn worker_defaults_preserve_jinja_effort_precedence() { + // The engine pops request kwargs.reasoning_effort before merging defaults, + // then overlays merged kwargs on the effective request effort for Jinja. + for (defaults, controls, expected) in [ + (json!({"reasoning_effort":"low"}), json!({}), json!("low")), + ( + json!({"reasoning_effort":"low"}), + json!({"reasoning_effort":"high"}), + json!("low"), + ), + ( + json!({"reasoning_effort":"low"}), + json!({"chat_template_kwargs":{"reasoning_effort":"high"}}), + json!("low"), + ), + ( + json!({"reasoning_effort":null}), + json!({"reasoning_effort":"high"}), + JsonValue::Null, + ), + (json!({}), json!({"reasoning_effort":"high"}), json!("high")), + ( + json!({}), + json!({"reasoning_effort":"low", "chat_template_kwargs":{"reasoning_effort":"high"}}), + json!("high"), + ), + ] { + let enc = jinja(json!({"chat_template":"{{ reasoning_effort | tojson }}"})) + .with_defaults(&serde_json::from_value(defaults.clone()).unwrap()); + let mut req = request(json!([{"role":"user","content":"hi"}])); + req.as_object_mut() + .unwrap() + .extend(controls.as_object().unwrap().clone()); + assert_eq!( + enc.render(&req).unwrap(), + expected.to_string(), + "defaults={defaults}, request={req}" + ); + } + } + + #[test] + fn kimi_worker_defaults_reach_thinking_effort() { + for (defaults, controls, expected) in [ + (json!({"reasoning_effort":"low"}), json!({}), "low"), + ( + json!({"reasoning_effort":"low"}), + json!({"reasoning_effort":"high"}), + "high", + ), + ( + json!({"reasoning_effort":"low"}), + json!({"chat_template_kwargs":{"reasoning_effort":"high"}}), + "high", + ), + ( + json!({"thinking_effort":"high"}), + json!({"reasoning_effort":"low"}), + "high", + ), + ( + json!({"reasoning_effort":"low", "thinking_effort":"high"}), + json!({"chat_template_kwargs":{"thinking_effort":"max"}}), + "max", + ), + ] { + let enc = ChatFormatter::kimi_native(Some("kimi_k3"), "kimi-k3") + .unwrap() + .with_defaults(&serde_json::from_value(defaults.clone()).unwrap()); + let mut req = request(json!([{"role":"user","content":"hi"}])); + req.as_object_mut() + .unwrap() + .extend(controls.as_object().unwrap().clone()); + let prompt = enc.render(&req).unwrap(); + assert!( + prompt.contains(&format!("thinking_effort={expected}")), + "defaults={defaults}, request={req}, prompt={prompt}" + ); + } + } + + #[test] + fn worker_effort_does_not_derive_thinking_flags() { + let enc = jinja(json!({"chat_template":"{{ thinking | default('unset') }}:{{ enable_thinking | default('unset') }}"})) + .with_defaults(&serde_json::from_value(json!({"reasoning_effort":"high"})).unwrap()); + let req = request(json!([{"role":"user","content":"hi"}])); + assert_eq!(enc.render(&req).unwrap(), "unset:unset"); + } + + #[test] + fn null_worker_effort_preserves_deepseek_env_fallback() { + // Set process-wide envs only in a child so parallel formatter tests retain + // their normal defaults. + const CHILD: &str = "SGL_ROUTER_TEST_NULL_WORKER_EFFORT_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "tokenizer::chat_formatter::tests::null_worker_effort_preserves_deepseek_env_fallback", "--nocapture"]) + .env(CHILD, "1") + .env("SGLANG_DSV4_REASONING_EFFORT", "max") + .env("SGLANG_DSV41_REASONING_EFFORT", "low") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + for (model_type, effort) in [("deepseek_v4", "max"), ("deepseek_v41", "low")] { + let enc = ChatFormatter::deepseek_native(Some(model_type), model_type) + .unwrap() + .with_defaults( + &serde_json::from_value(json!({"thinking":true, "reasoning_effort":null})) + .unwrap(), + ); + let mut req = request(json!([{"role":"user","content":"hi"}])); + let actual = enc.render(&req).unwrap(); + req["reasoning_effort"] = json!(effort); + assert_eq!(actual, enc.render(&req).unwrap(), "{model_type}"); + } + } + + #[test] + fn v41_thinking_env_uses_engine_budgets() { + // Isolate the environment so other formatter tests keep their defaults. + const CHILD: &str = "SGL_ROUTER_TEST_V41_BUDGET_CHILD"; + let Ok(default_budget) = std::env::var(CHILD) else { + for (effort, budget) in [ + (None, 75), + (Some("low"), 50), + (Some("high"), 75), + (Some("xhigh"), 75), + (Some("max"), 100), + (Some("42"), 42), + ] { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()); + child + .args([ + "--exact", + "tokenizer::chat_formatter::tests::v41_thinking_env_uses_engine_budgets", + "--nocapture", + ]) + .env(CHILD, budget.to_string()) + .env("SGLANG_DEFAULT_THINKING", "1") + .env_remove("SGLANG_DSV41_REASONING_EFFORT"); + if let Some(effort) = effort { + child.env("SGLANG_DSV41_REASONING_EFFORT", effort); + } + let output = child.output().unwrap(); + assert!( + output.status.success(), + "effort={effort:?}: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + return; + }; + let enc = ChatFormatter::deepseek_native(Some("deepseek_v41"), "alias").unwrap(); + for (effort, budget) in [ + (None, default_budget.as_str()), + (Some("unsupported"), default_budget.as_str()), + (Some("low"), "50"), + (Some("high"), "75"), + (Some("xhigh"), "75"), + (Some("max"), "100"), + ] { + let mut req = request(json!([{"role":"user","content":"hi"}])); + if let Some(effort) = effort { + req["reasoning_effort"] = json!(effort); + } + let prompt = enc.render(&req).unwrap(); + assert!( + prompt.contains(&format!("Reasoning Effort: {budget} (range 1-100,")), + "request={req}, prompt={prompt}" + ); + assert!(prompt.ends_with(""), "{prompt}"); + } + } + /// Messages reach the template shaped like the engine's request schema. #[test] fn messages_match_engine_schema() { diff --git a/experimental/sgl-router/src/tokenizer/deepseek.rs b/experimental/sgl-router/src/tokenizer/deepseek.rs index f9477dae7f..fd489dc7ea 100644 --- a/experimental/sgl-router/src/tokenizer/deepseek.rs +++ b/experimental/sgl-router/src/tokenizer/deepseek.rs @@ -28,6 +28,7 @@ impl Encoder { request: &Value, mut messages: Vec, kwargs: &ChatTemplateKwargs, + effort: Option, ) -> Result { // SGLang drops a later user's task when merging it into an existing // user/tool-result turn. Dynamo otherwise preserves that field. @@ -39,8 +40,8 @@ impl Encoder { } } match self { - Self::V4(profile) => profile.render(request, messages, kwargs), - Self::V41 => render_v41(request, messages, kwargs), + Self::V4(profile) => profile.render(request, messages, kwargs, effort), + Self::V41 => render_v41(request, messages, kwargs, effort), } } } @@ -100,6 +101,7 @@ impl V4Profile { request: &Value, mut messages: Vec, kwargs: &ChatTemplateKwargs, + effort: Option, ) -> Result { ensure!( !messages.is_empty(), @@ -113,10 +115,14 @@ impl V4Profile { if let Some(tools) = request["tools"].as_array().filter(|t| !t.is_empty()) { messages[0]["tools"] = normalize_tools(tools)?.into(); } - let effort = match ( - self, - request_effort(request).as_ref().and_then(Value::as_str), - ) { + // Without a request effort, SGLang uses SGLANG_DSV4_REASONING_EFFORT. + let effort = effort.or_else(|| { + std::env::var("SGLANG_DSV4_REASONING_EFFORT") + .ok() + .filter(|v| !v.is_empty()) + .map(Value::from) + }); + let effort = match (self, effort.as_ref().and_then(Value::as_str)) { (Self::Official, Some("high")) | (Self::Preview, Some("max")) => { Some(ReasoningEffort::High) } @@ -248,6 +254,7 @@ fn render_v41( request: &Value, mut messages: Vec, kwargs: &ChatTemplateKwargs, + effort: Option, ) -> Result { ensure!( !messages.is_empty(), @@ -294,12 +301,35 @@ fn render_v41( }) .collect(); } - // chat_encoding.parse_dsv41_reasoning_effort; unsupported values use the - // engine default (`high`), matching SGLANG_DSV41_REASONING_EFFORT unset. - let budget = match request_effort(request) { + // Unsupported values use SGLANG_DSV41_REASONING_EFFORT, else `high`. + let env_effort = std::env::var("SGLANG_DSV41_REASONING_EFFORT") + .ok() + .map(|v| { + v.trim() + .parse::() + .map_or_else(|_| Value::from(v.trim()), Value::from) + }); + let budget = dsv41_budget(effort) + .or_else(|| dsv41_budget(env_effort)) + .unwrap_or(75); + v41::encode_messages( + &messages, + if thinking { + ThinkingMode::Thinking + } else { + ThinkingMode::Chat + }, + true, + budget, + ) +} + +/// `chat_encoding.parse_dsv41_reasoning_effort`, as a token budget. +fn dsv41_budget(effort: Option) -> Option { + match effort { Some(Value::String(s)) => match s.as_str() { - "low" => Some(25), - "high" => Some(50), + "low" => Some(50), + "high" => Some(75), "xhigh" => Some(75), "max" => Some(100), _ => None, @@ -313,17 +343,6 @@ fn render_v41( }, _ => None, } - .unwrap_or(50); - v41::encode_messages( - &messages, - if thinking { - ThinkingMode::Thinking - } else { - ThinkingMode::Chat - }, - true, - budget, - ) } #[cfg(test)] diff --git a/experimental/sgl-router/src/tokenizer/mod.rs b/experimental/sgl-router/src/tokenizer/mod.rs index 2e6866068e..1f4336cb9b 100644 --- a/experimental/sgl-router/src/tokenizer/mod.rs +++ b/experimental/sgl-router/src/tokenizer/mod.rs @@ -68,6 +68,7 @@ impl TokenizerRegistry { me.inner.insert(m.id.clone(), t); match ChatFormatter::load(&m.id, &m.tokenizer_path) { Ok(Some(formatter)) => { + let formatter = formatter.with_defaults(&m.default_chat_template_kwargs); me.formatters .insert(m.id.clone(), Arc::new(ChatFormatterEntry::new(formatter))); tracing::info!(model = %m.id, "dynamo-render chat rendering enabled"); @@ -83,9 +84,9 @@ impl TokenizerRegistry { routing tokenization remains available"); } else if me.has_chat_formatter(&m.id) { tracing::warn!(model = %m.id, - "router-generated input_ids forwarding enabled: requires matching worker model \ - files and template defaults; native DeepSeek assumes SGLANG_DEFAULT_THINKING=false \ - and default SGLANG_DSV4_REASONING_EFFORT / SGLANG_DSV41_REASONING_EFFORT; worker parser overrides \ + "router-generated input_ids forwarding enabled: requires the workers' model files, \ + --default-chat-template-kwargs, SGLANG_DEFAULT_THINKING, and \ + SGLANG_DSV4_REASONING_EFFORT / SGLANG_DSV41_REASONING_EFFORT; worker parser overrides \ (including --tool-call-parser deepseekv32), content-format detection, and \ conversation-template stop strings are not replicated. Use \ --disable-input-ids-forwarding for array-only templates or when these assumptions do not hold"); @@ -181,6 +182,7 @@ mod tests { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: crate::config::DiscoveryBackend::StaticUrls( crate::config::StaticUrlsDiscoveryConfig { diff --git a/experimental/sgl-router/src/workers/manager.rs b/experimental/sgl-router/src/workers/manager.rs index 4c753e41ef..4e49fc93f5 100644 --- a/experimental/sgl-router/src/workers/manager.rs +++ b/experimental/sgl-router/src/workers/manager.rs @@ -605,6 +605,7 @@ mod tests { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://test:30000".into()], diff --git a/experimental/sgl-router/tests/component/discovery/static_urls.rs b/experimental/sgl-router/tests/component/discovery/static_urls.rs index 852e020b11..fcc3ee1c8e 100644 --- a/experimental/sgl-router/tests/component/discovery/static_urls.rs +++ b/experimental/sgl-router/tests/component/discovery/static_urls.rs @@ -142,6 +142,7 @@ async fn static_urls_pd_role_resolved_end_to_end() { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec![url.clone()], diff --git a/experimental/sgl-router/tests/component/tokenizer/render_parity.rs b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs index fcc6e7dc91..62e975a5b4 100644 --- a/experimental/sgl-router/tests/component/tokenizer/render_parity.rs +++ b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs @@ -123,6 +123,7 @@ fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/fixtures/deepseek/v41.json b/experimental/sgl-router/tests/fixtures/deepseek/v41.json index d6ad7217f2..8c772eef90 100644 --- a/experimental/sgl-router/tests/fixtures/deepseek/v41.json +++ b/experimental/sgl-router/tests/fixtures/deepseek/v41.json @@ -10,18 +10,18 @@ {"name":"tools_named","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"tool_choice":{"type":"function","function":{"name":"bar"}}},"prompt":"<|begin▁of▁sentence|><|System|>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|>","token_count":256,"token_sha256":"89eaf9e9642f765c5f244da76ce14d680a6c22bf97f6a859d1946d528649d458"}, {"name":"message_tools","profile":"v41","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}]},{"role":"user","content":"Hi"}]},"prompt":"<|begin▁of▁sentence|><|System|>SYS\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|>","token_count":272,"token_sha256":"d6ebecca3f9e9a498e18f1a5140ff8d2e833951d64329ff970ff8ef6157f0a0c"}, {"name":"empty_message_tools","profile":"v41","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[]},{"role":"user","content":"Hi"}]},"prompt":"<|begin▁of▁sentence|><|System|>SYS<|User|>Hi<|Assistant|>","token_count":8,"token_sha256":"0c718f5b123ecfacb6fc0d0091dfc63b1b7959c9122b2365fe823b270b6c3d35"}, -{"name":"tool_results","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"foo","arguments":"{\"x\":\"雪\", \"y\":[1, true]}"}},{"id":"b","type":"function","function":{"name":"bar","arguments":{"z":2}}}],"reasoning_content":"Call both"},{"role":"tool","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}],"tool_call_id":"b"},{"role":"tool","content":null,"tool_call_id":"a"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|>Call both\n\n<|DSML| calls>\n<|DSML| invoke name=\"foo\">\n<|DSML| parameter name=\"x\" string=\"true\">雪\n<|DSML| parameter name=\"y\" string=\"false\">[1, true]\n\n<|DSML| invoke name=\"bar\">\n<|DSML| parameter name=\"z\" string=\"false\">2\n\n<|end▁of▁sentence|><|User|>\n\nHello\n\nworld<|Assistant|>","token_count":388,"token_sha256":"f9fb80a884084408d01041c452a9ece38a9c3a959a5fd3ed0289db3f6611c203"}, +{"name":"tool_results","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"foo","arguments":"{\"x\":\"雪\", \"y\":[1, true]}"}},{"id":"b","type":"function","function":{"name":"bar","arguments":{"z":2}}}],"reasoning_content":"Call both"},{"role":"tool","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}],"tool_call_id":"b"},{"role":"tool","content":null,"tool_call_id":"a"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|>Call both\n\n<|DSML| calls>\n<|DSML| invoke name=\"foo\">\n<|DSML| parameter name=\"x\" string=\"true\">雪\n<|DSML| parameter name=\"y\" string=\"false\">[1, true]\n\n<|DSML| invoke name=\"bar\">\n<|DSML| parameter name=\"z\" string=\"false\">2\n\n<|end▁of▁sentence|><|User|>\n\nHello\n\nworld<|Assistant|>","token_count":388,"token_sha256":"a02f2167e3b33c915e6b53619b61258ae96cbb4aa7bb0ee66b4287cde20ba4e4"}, {"name":"kwargs_effort","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":"max"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"a29f18c6e65c09656d098c43e548224b67ced6a674f65186b7e132fb6d997d75"}, -{"name":"effort_conflict","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 25 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"80aa41f1cbf4d847db08af7c5f4dde508ea75aed6e2cb6b94f3a0a806e275d46"}, -{"name":"drop_thinking_ignored","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}],"chat_template_kwargs":{"thinking":true,"drop_thinking":false}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>Answer<|end▁of▁sentence|><|User|>Again<|Assistant|>","token_count":37,"token_sha256":"f6cb0ab0ad4a220297622920684237a208098dd2b2cd368599bc49ebba7994f9"}, +{"name":"effort_conflict","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"}, +{"name":"drop_thinking_ignored","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}],"chat_template_kwargs":{"thinking":true,"drop_thinking":false}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>Answer<|end▁of▁sentence|><|User|>Again<|Assistant|>","token_count":37,"token_sha256":"e1c17a3d6c711c55d0f3e2deb5c56950144267d0337d88ebe5a7ab8e5dd62136"}, {"name":"thinking_false","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"thinking":false}},"prompt":"<|begin▁of▁sentence|><|User|>Hi<|Assistant|>","token_count":5,"token_sha256":"95aeed952d0173ec8849b13faec66eb61ec4412cd9c41dae0c405257559365a6"}, -{"name":"thinking_none","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"none","chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"}, +{"name":"thinking_none","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"none","chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"}, {"name":"consecutive_task","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"user","content":"Again"}],"task":"domain"},"prompt":"<|begin▁of▁sentence|><|User|>Hi\n\nAgain<|Assistant|>","token_count":7,"token_sha256":"d05e248d8a06f2c06df20b7a71af0d4dfae1cd91ce9f614719b5842674042d6b"}, -{"name":"effort_high","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"}, +{"name":"effort_high","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"}, {"name":"effort_max","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"max"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"a29f18c6e65c09656d098c43e548224b67ced6a674f65186b7e132fb6d997d75"}, {"name":"task_action","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"task":"action"},"prompt":"<|begin▁of▁sentence|><|User|>Hi<|Assistant|><|action|>","token_count":6,"token_sha256":"925bcaae248cc41efabd3e5d8f660f17513e43174f1d34aba51a1399aba76e8e"}, -{"name":"effort_low","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"low"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 25 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"80aa41f1cbf4d847db08af7c5f4dde508ea75aed6e2cb6b94f3a0a806e275d46"}, +{"name":"effort_low","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"low"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"}, {"name":"effort_xhigh","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"xhigh"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"}, {"name":"effort_0.75","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":0.75},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"}, -{"name":"kwargs_budget","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":80}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 80 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"32a4462bbf34bce35dd5c163574188ebd71896ac46d58966e0a6c2481b1b4081"} +{"name":"kwargs_budget","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":80}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 80 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|>","token_count":31,"token_sha256":"32a4462bbf34bce35dd5c163574188ebd71896ac46d58966e0a6c2481b1b4081"} ]} diff --git a/experimental/sgl-router/tests/fixtures/kimi_k3/prompts.json b/experimental/sgl-router/tests/fixtures/kimi_k3/prompts.json index 9f724e561c..5faa4bf8af 100644 --- a/experimental/sgl-router/tests/fixtures/kimi_k3/prompts.json +++ b/experimental/sgl-router/tests/fixtures/kimi_k3/prompts.json @@ -1,7 +1,7 @@ [ {"name":"default","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}]},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"}, {"name":"effort_none","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"none"},"token_count":58,"sha256":"d9764be8b6a9a04114759d162f5a1c4e67109a7af2d62e72c5faf8f150c05d59"}, -{"name":"effort_precedence","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"token_count":276,"sha256":"54e51363cdb9f32dd2a11d805c3b5797b969526c338c7a93d0ae7740457073dc"}, +{"name":"effort_precedence","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"token_count":275,"sha256":"8da270ec9559dfa53515591fe37b480e68f6b70275265d5086a2ff43a747d32a"}, {"name":"reasoning_alias","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning":{"effort":"low"}},"token_count":275,"sha256":"8da270ec9559dfa53515591fe37b480e68f6b70275265d5086a2ff43a747d32a"}, {"name":"enable_thinking_only","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"chat_template_kwargs":{"enable_thinking":false}},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"}, {"name":"unsupported_effort","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"medium"},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"}, diff --git a/experimental/sgl-router/tests/proxy/bucket_routing.rs b/experimental/sgl-router/tests/proxy/bucket_routing.rs index 554e35d22e..214498eb84 100644 --- a/experimental/sgl-router/tests/proxy/bucket_routing.rs +++ b/experimental/sgl-router/tests/proxy/bucket_routing.rs @@ -72,6 +72,7 @@ fn build_app_context( fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/chat_routing.rs b/experimental/sgl-router/tests/proxy/chat_routing.rs index 0e251603b2..92cbc80a6f 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing.rs @@ -51,6 +51,7 @@ fn config_for(_worker_url: &str) -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs index a2883e5cba..33cdc2a2ab 100644 --- a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs +++ b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs @@ -37,6 +37,7 @@ pub fn config() -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/failover.rs b/experimental/sgl-router/tests/proxy/failover.rs index d2d4d6e923..3707b0c18a 100644 --- a/experimental/sgl-router/tests/proxy/failover.rs +++ b/experimental/sgl-router/tests/proxy/failover.rs @@ -49,6 +49,7 @@ async fn failover_when_one_worker_dies() { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec![w1.url.clone(), w2.url.clone(), w3.url.clone()], diff --git a/experimental/sgl-router/tests/proxy/graceful_shutdown.rs b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs index 01eb768058..55e6610667 100644 --- a/experimental/sgl-router/tests/proxy/graceful_shutdown.rs +++ b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs @@ -64,6 +64,7 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/header_forwarding.rs b/experimental/sgl-router/tests/proxy/header_forwarding.rs index e29bd57e46..42d0017ef5 100644 --- a/experimental/sgl-router/tests/proxy/header_forwarding.rs +++ b/experimental/sgl-router/tests/proxy/header_forwarding.rs @@ -42,6 +42,7 @@ async fn forwards_whitelisted_headers_strips_others() { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs index 82eb3ab777..fc791082b0 100644 --- a/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs +++ b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs @@ -57,6 +57,7 @@ fn config() -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs index 3fbe944b25..68fd919c6b 100644 --- a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs +++ b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs @@ -56,6 +56,7 @@ fn config() -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs b/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs index 60bec8f382..f06276384b 100644 --- a/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs +++ b/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs @@ -144,6 +144,7 @@ fn config() -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs b/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs index 1d71ac87e4..497d7565c7 100644 --- a/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs +++ b/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs @@ -54,6 +54,7 @@ fn config() -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs b/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs index a275e2b8a1..f0fc241f83 100644 --- a/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs +++ b/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs @@ -160,6 +160,7 @@ fn config(policy: PolicyKind) -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/sticky_input_ids.rs b/experimental/sgl-router/tests/proxy/sticky_input_ids.rs index e6de411172..37187c2d0f 100644 --- a/experimental/sgl-router/tests/proxy/sticky_input_ids.rs +++ b/experimental/sgl-router/tests/proxy/sticky_input_ids.rs @@ -72,6 +72,7 @@ fn config() -> Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/proxy/sticky_routing.rs b/experimental/sgl-router/tests/proxy/sticky_routing.rs index db3a066f63..6f6c27f62a 100644 --- a/experimental/sgl-router/tests/proxy/sticky_routing.rs +++ b/experimental/sgl-router/tests/proxy/sticky_routing.rs @@ -58,6 +58,7 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc Config { fused: None, eligibility: None, sampling_overrides: Default::default(), + default_chat_template_kwargs: Default::default(), }, discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { urls: vec!["http://placeholder:0".into()], diff --git a/experimental/sgl-router/tests/scripts/generate_kimi_parity.py b/experimental/sgl-router/tests/scripts/generate_kimi_parity.py index 9f4c298d4f..9830ccc6b4 100644 --- a/experimental/sgl-router/tests/scripts/generate_kimi_parity.py +++ b/experimental/sgl-router/tests/scripts/generate_kimi_parity.py @@ -51,6 +51,12 @@ for case in cases: if "repeat" in case: data["messages"][0]["content"] *= case["repeat"] request = ChatCompletionRequest(**data) + # _convert_to_internal_request runs before _encode_messages and promotes + # the request kwarg over the top-level effort. + if request.chat_template_kwargs: + effort = request.chat_template_kwargs.pop("reasoning_effort", None) + if effort is not None: + request.reasoning_effort = effort messages = [message.model_dump() for message in request.messages] for message in messages: normalize_assistant_tool_call_arguments(message, strict=False)