[sgl-router] Take SGLang's render defaults: --default-chat-template-kwargs and thinking/effort envs (#41221)
Co-authored-by: Kan Wu <kan.wu@radixark.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Kan Wu
Claude Opus 5.5
parent
effb752188
commit
b4584ab838
@@ -205,8 +205,13 @@ rendered tokens to the engine as `input_ids`, retaining the original messages,
|
||||
so the engine skips re-tokenizing. Every other request shape is rendered for
|
||||
routing only: the router renders with dynamo-render and does not replicate
|
||||
SGLang's request normalization, so forwarding is enabled shape by shape as
|
||||
parity is verified. Use matching model files on the router and workers; worker
|
||||
template overrides and default kwargs are not observable from the request.
|
||||
parity is verified. Use matching model files on the router and workers, and set
|
||||
the same `--default-chat-template-kwargs`, `SGLANG_DEFAULT_THINKING`,
|
||||
`SGLANG_DSV4_REASONING_EFFORT`, and `SGLANG_DSV41_REASONING_EFFORT` on both.
|
||||
The router reads these render defaults from its own configuration and environment;
|
||||
it does not discover the workers' settings. Point `--tokenizer-path` at the workers'
|
||||
model snapshot so the V4 effort profile is read from the same
|
||||
`encoding/encoding_dsv4.py`.
|
||||
|
||||
Set `--disable-input-ids-forwarding` for this router's model when worker-side
|
||||
rendering has not been verified to match. This disables router-generated IDs
|
||||
@@ -214,13 +219,13 @@ for every routing policy; cache-aware routing still renders and tokenizes
|
||||
locally, and the original messages reach the workers for engine processing.
|
||||
Caller-supplied `input_ids` remain caller-owned and pass through unchanged.
|
||||
|
||||
Forwarding logs its assumptions at startup. In particular, disable it for
|
||||
`SGLANG_DEFAULT_THINKING=true`, a non-default `SGLANG_DSV4_REASONING_EFFORT`,
|
||||
worker parser overrides such as `--tool-call-parser deepseekv32` that select a
|
||||
native encoder over a shipped template, or conversation templates with stop
|
||||
strings (the engine's `input_ids` path skips those template stops). These worker
|
||||
settings are not inferred from the router's environment. Disabling forwarding preserves
|
||||
engine behavior but does not establish parity for local routing hashes.
|
||||
Forwarding logs its assumptions at startup. In particular, disable it when the
|
||||
router's render defaults differ from the workers', for worker template overrides
|
||||
not reflected in the router's model files, for worker parser overrides such as
|
||||
`--tool-call-parser deepseekv32` that select a native encoder over a shipped
|
||||
template, or conversation templates with stop strings (the engine's `input_ids`
|
||||
path skips those template stops). Disabling forwarding preserves engine behavior
|
||||
but does not establish parity for local routing hashes.
|
||||
|
||||
Also set `--disable-input-ids-forwarding` for array-only templates: Dynamo may wrap
|
||||
string content into arrays differently from the worker. The pinned Dynamo renderer does not expose
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
//! Grouped CLI options and conversion into a validated [`Config`].
|
||||
|
||||
use anyhow::{anyhow, ensure, Result};
|
||||
use anyhow::{anyhow, ensure, Context, Result};
|
||||
use clap::Parser;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
@@ -66,10 +66,14 @@ pub struct ModelArgs {
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
/// Disable generated input_ids; workers tokenize messages, while routing still renders locally.
|
||||
/// Use for worker-only thinking defaults, parser/template overrides, or template stop strings.
|
||||
/// Use for worker parser/template overrides or template stop strings.
|
||||
#[arg(long)]
|
||||
pub disable_input_ids_forwarding: bool,
|
||||
|
||||
/// Same as SGLang's --default-chat-template-kwargs; must match the workers.
|
||||
#[arg(long, value_name = "JSON")]
|
||||
pub default_chat_template_kwargs: Option<String>,
|
||||
|
||||
/// Fleet sampling defaults as JSON, e.g. {"temperature": 1, "top_p": 0.95}.
|
||||
/// Accepts temperature, top_p, top_k, min_p, repetition_penalty,
|
||||
/// frequency_penalty, presence_penalty, and n. Numeric values fill absent
|
||||
@@ -383,6 +387,14 @@ impl Cli {
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
let default_chat_template_kwargs = self
|
||||
.model
|
||||
.default_chat_template_kwargs
|
||||
.as_deref()
|
||||
.map(serde_json::from_str)
|
||||
.transpose()
|
||||
.context("--default-chat-template-kwargs must be a JSON object")?
|
||||
.unwrap_or_default();
|
||||
|
||||
let config = Config {
|
||||
server: ServerConfig {
|
||||
@@ -412,6 +424,7 @@ impl Cli {
|
||||
fused,
|
||||
eligibility,
|
||||
sampling_overrides,
|
||||
default_chat_template_kwargs,
|
||||
},
|
||||
discovery,
|
||||
proxy: ProxyConfig {
|
||||
@@ -2396,6 +2409,51 @@ mod tests {
|
||||
.to_string();
|
||||
assert!(err.contains("unknown parameter"), "got: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_chat_template_kwargs_reaches_the_model_config() {
|
||||
let kwargs = serde_json::json!({"thinking": true, "reasoning_effort": "high"});
|
||||
let config = into_config_owned(with_model(&[
|
||||
"--worker-urls",
|
||||
"http://x:30000",
|
||||
"--default-chat-template-kwargs",
|
||||
&kwargs.to_string(),
|
||||
]))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(config.model.default_chat_template_kwargs).unwrap(),
|
||||
kwargs
|
||||
);
|
||||
|
||||
let defaults = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap();
|
||||
assert!(defaults.model.default_chat_template_kwargs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_default_chat_template_kwargs_fails_the_launch() {
|
||||
for kwargs in [
|
||||
r#"{"thinking": true"#,
|
||||
"[]",
|
||||
"null",
|
||||
"true",
|
||||
"42",
|
||||
r#""high""#,
|
||||
] {
|
||||
let err = into_config_owned(with_model(&[
|
||||
"--worker-urls",
|
||||
"http://x:30000",
|
||||
"--default-chat-template-kwargs",
|
||||
kwargs,
|
||||
]))
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
err.contains("--default-chat-template-kwargs must be a JSON object"),
|
||||
"kwargs={kwargs}, got: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_ids_forwarding_can_be_disabled_for_the_model() {
|
||||
let defaults = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap();
|
||||
|
||||
@@ -236,6 +236,7 @@ mod tests {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: urls.iter().map(|s| s.to_string()).collect(),
|
||||
|
||||
@@ -330,6 +330,8 @@ pub struct ModelConfig {
|
||||
pub eligibility: Option<EligibilityConfig>,
|
||||
/// Fleet sampling defaults and conflict behavior. See [`SamplingOverrides`].
|
||||
pub sampling_overrides: SamplingOverrides,
|
||||
/// Worker `--default-chat-template-kwargs`, applied when rendering.
|
||||
pub default_chat_template_kwargs: crate::tokenizer::chat_formatter::ChatTemplateKwargs,
|
||||
}
|
||||
|
||||
/// External KV Indexer client settings.
|
||||
|
||||
@@ -348,6 +348,7 @@ mod tests {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -196,6 +196,7 @@ impl AppContext {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: crate::config::DiscoveryBackend::StaticUrls(
|
||||
crate::config::StaticUrlsDiscoveryConfig {
|
||||
|
||||
@@ -61,6 +61,7 @@ mod tests {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
};
|
||||
let app = crate::server::app::build_router(std::sync::Arc::new(ctx));
|
||||
let res = app
|
||||
|
||||
@@ -129,6 +129,7 @@ mod tests {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: crate::config::DiscoveryBackend::StaticUrls(
|
||||
crate::config::StaticUrlsDiscoveryConfig {
|
||||
|
||||
@@ -42,6 +42,8 @@ pub struct ChatFormatter {
|
||||
bos_token: Option<String>,
|
||||
deepseek: Option<super::deepseek::Encoder>,
|
||||
is_kimi_k3: bool,
|
||||
/// Worker defaults, merged before model-specific effort conversion.
|
||||
worker_defaults: ChatTemplateKwargs,
|
||||
}
|
||||
|
||||
impl ChatFormatter {
|
||||
@@ -163,6 +165,7 @@ impl ChatFormatter {
|
||||
bos_token,
|
||||
deepseek: None,
|
||||
is_kimi_k3: false,
|
||||
worker_defaults: ChatTemplateKwargs::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -178,6 +181,7 @@ impl ChatFormatter {
|
||||
bos_token: Some("[BOS]".into()),
|
||||
deepseek: None,
|
||||
is_kimi_k3: true,
|
||||
worker_defaults: ChatTemplateKwargs::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -205,10 +209,12 @@ impl ChatFormatter {
|
||||
.map_or(name.contains("v4.1") || name.contains("v41"), |t| {
|
||||
t == "deepseek_v41"
|
||||
});
|
||||
// Engine defaults: chat mode (`SGLANG_DEFAULT_THINKING=false`) and no
|
||||
// reasoning-effort preamble; dynamo-render defaults to thinking at high effort.
|
||||
// Engine defaults: `SGLANG_DEFAULT_THINKING` and no reasoning-effort
|
||||
// preamble; dynamo-render defaults to thinking at high effort.
|
||||
let thinking = std::env::var("SGLANG_DEFAULT_THINKING")
|
||||
.is_ok_and(|v| ["true", "1", "yes", "y"].contains(&v.to_lowercase().as_str()));
|
||||
let defaults = HashMap::from([
|
||||
("thinking".into(), false.into()),
|
||||
("thinking".into(), thinking.into()),
|
||||
("reasoning_effort".into(), "low".into()),
|
||||
]);
|
||||
Some(Self {
|
||||
@@ -221,6 +227,23 @@ impl ChatFormatter {
|
||||
is_deepseek_v4.then(|| super::deepseek::Encoder::V4(Default::default()))
|
||||
},
|
||||
is_kimi_k3: false,
|
||||
worker_defaults: ChatTemplateKwargs::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Apply the workers' `--default-chat-template-kwargs`; they fill keys the
|
||||
/// request leaves unset, and a default `reasoning_effort` acts as the request's.
|
||||
pub fn with_defaults(mut self, defaults: &ChatTemplateKwargs) -> Self {
|
||||
self.worker_defaults = defaults.clone();
|
||||
self
|
||||
}
|
||||
|
||||
fn effective_effort(&self, request: &JsonValue) -> Option<JsonValue> {
|
||||
super::deepseek::request_effort(request).or_else(|| {
|
||||
self.worker_defaults
|
||||
.get("reasoning_effort")
|
||||
.filter(|v| !v.is_null())
|
||||
.cloned()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -270,7 +293,14 @@ impl ChatFormatter {
|
||||
.entry("enable_thinking".into())
|
||||
.or_insert(thinking.into());
|
||||
}
|
||||
if let Some(effort) = effort {
|
||||
// serving_chat first pops the request kwarg into request.reasoning_effort,
|
||||
// then merges worker defaults. Jinja receives those merged kwargs over
|
||||
// the effective effort; Kimi derives thinking_effort only if still absent.
|
||||
kwargs.remove("reasoning_effort");
|
||||
for (key, value) in &self.worker_defaults {
|
||||
kwargs.entry(key.clone()).or_insert_with(|| value.clone());
|
||||
}
|
||||
if let Some(effort) = self.effective_effort(request) {
|
||||
if self.is_kimi_k3 {
|
||||
if matches!(effort.as_str(), Some("low" | "high" | "max")) {
|
||||
kwargs.entry("thinking_effort".into()).or_insert(effort);
|
||||
@@ -332,8 +362,9 @@ impl ChatFormatter {
|
||||
}
|
||||
}
|
||||
if let Some(profile) = self.deepseek {
|
||||
let effort = self.effective_effort(request);
|
||||
return Ok((
|
||||
RenderedPrompt::text(profile.render(request, messages, &kwargs)?),
|
||||
RenderedPrompt::text(profile.render(request, messages, &kwargs, effort)?),
|
||||
prefix,
|
||||
));
|
||||
}
|
||||
@@ -785,6 +816,189 @@ mod tests {
|
||||
assert_eq!(enc.render(&req).unwrap(), ":True:True");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_defaults_preserve_jinja_effort_precedence() {
|
||||
// The engine pops request kwargs.reasoning_effort before merging defaults,
|
||||
// then overlays merged kwargs on the effective request effort for Jinja.
|
||||
for (defaults, controls, expected) in [
|
||||
(json!({"reasoning_effort":"low"}), json!({}), json!("low")),
|
||||
(
|
||||
json!({"reasoning_effort":"low"}),
|
||||
json!({"reasoning_effort":"high"}),
|
||||
json!("low"),
|
||||
),
|
||||
(
|
||||
json!({"reasoning_effort":"low"}),
|
||||
json!({"chat_template_kwargs":{"reasoning_effort":"high"}}),
|
||||
json!("low"),
|
||||
),
|
||||
(
|
||||
json!({"reasoning_effort":null}),
|
||||
json!({"reasoning_effort":"high"}),
|
||||
JsonValue::Null,
|
||||
),
|
||||
(json!({}), json!({"reasoning_effort":"high"}), json!("high")),
|
||||
(
|
||||
json!({}),
|
||||
json!({"reasoning_effort":"low", "chat_template_kwargs":{"reasoning_effort":"high"}}),
|
||||
json!("high"),
|
||||
),
|
||||
] {
|
||||
let enc = jinja(json!({"chat_template":"{{ reasoning_effort | tojson }}"}))
|
||||
.with_defaults(&serde_json::from_value(defaults.clone()).unwrap());
|
||||
let mut req = request(json!([{"role":"user","content":"hi"}]));
|
||||
req.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(controls.as_object().unwrap().clone());
|
||||
assert_eq!(
|
||||
enc.render(&req).unwrap(),
|
||||
expected.to_string(),
|
||||
"defaults={defaults}, request={req}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kimi_worker_defaults_reach_thinking_effort() {
|
||||
for (defaults, controls, expected) in [
|
||||
(json!({"reasoning_effort":"low"}), json!({}), "low"),
|
||||
(
|
||||
json!({"reasoning_effort":"low"}),
|
||||
json!({"reasoning_effort":"high"}),
|
||||
"high",
|
||||
),
|
||||
(
|
||||
json!({"reasoning_effort":"low"}),
|
||||
json!({"chat_template_kwargs":{"reasoning_effort":"high"}}),
|
||||
"high",
|
||||
),
|
||||
(
|
||||
json!({"thinking_effort":"high"}),
|
||||
json!({"reasoning_effort":"low"}),
|
||||
"high",
|
||||
),
|
||||
(
|
||||
json!({"reasoning_effort":"low", "thinking_effort":"high"}),
|
||||
json!({"chat_template_kwargs":{"thinking_effort":"max"}}),
|
||||
"max",
|
||||
),
|
||||
] {
|
||||
let enc = ChatFormatter::kimi_native(Some("kimi_k3"), "kimi-k3")
|
||||
.unwrap()
|
||||
.with_defaults(&serde_json::from_value(defaults.clone()).unwrap());
|
||||
let mut req = request(json!([{"role":"user","content":"hi"}]));
|
||||
req.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(controls.as_object().unwrap().clone());
|
||||
let prompt = enc.render(&req).unwrap();
|
||||
assert!(
|
||||
prompt.contains(&format!("thinking_effort={expected}")),
|
||||
"defaults={defaults}, request={req}, prompt={prompt}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_effort_does_not_derive_thinking_flags() {
|
||||
let enc = jinja(json!({"chat_template":"{{ thinking | default('unset') }}:{{ enable_thinking | default('unset') }}"}))
|
||||
.with_defaults(&serde_json::from_value(json!({"reasoning_effort":"high"})).unwrap());
|
||||
let req = request(json!([{"role":"user","content":"hi"}]));
|
||||
assert_eq!(enc.render(&req).unwrap(), "unset:unset");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_worker_effort_preserves_deepseek_env_fallback() {
|
||||
// Set process-wide envs only in a child so parallel formatter tests retain
|
||||
// their normal defaults.
|
||||
const CHILD: &str = "SGL_ROUTER_TEST_NULL_WORKER_EFFORT_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
let output = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "tokenizer::chat_formatter::tests::null_worker_effort_preserves_deepseek_env_fallback", "--nocapture"])
|
||||
.env(CHILD, "1")
|
||||
.env("SGLANG_DSV4_REASONING_EFFORT", "max")
|
||||
.env("SGLANG_DSV41_REASONING_EFFORT", "low")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
return;
|
||||
}
|
||||
for (model_type, effort) in [("deepseek_v4", "max"), ("deepseek_v41", "low")] {
|
||||
let enc = ChatFormatter::deepseek_native(Some(model_type), model_type)
|
||||
.unwrap()
|
||||
.with_defaults(
|
||||
&serde_json::from_value(json!({"thinking":true, "reasoning_effort":null}))
|
||||
.unwrap(),
|
||||
);
|
||||
let mut req = request(json!([{"role":"user","content":"hi"}]));
|
||||
let actual = enc.render(&req).unwrap();
|
||||
req["reasoning_effort"] = json!(effort);
|
||||
assert_eq!(actual, enc.render(&req).unwrap(), "{model_type}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v41_thinking_env_uses_engine_budgets() {
|
||||
// Isolate the environment so other formatter tests keep their defaults.
|
||||
const CHILD: &str = "SGL_ROUTER_TEST_V41_BUDGET_CHILD";
|
||||
let Ok(default_budget) = std::env::var(CHILD) else {
|
||||
for (effort, budget) in [
|
||||
(None, 75),
|
||||
(Some("low"), 50),
|
||||
(Some("high"), 75),
|
||||
(Some("xhigh"), 75),
|
||||
(Some("max"), 100),
|
||||
(Some("42"), 42),
|
||||
] {
|
||||
let mut child = std::process::Command::new(std::env::current_exe().unwrap());
|
||||
child
|
||||
.args([
|
||||
"--exact",
|
||||
"tokenizer::chat_formatter::tests::v41_thinking_env_uses_engine_budgets",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, budget.to_string())
|
||||
.env("SGLANG_DEFAULT_THINKING", "1")
|
||||
.env_remove("SGLANG_DSV41_REASONING_EFFORT");
|
||||
if let Some(effort) = effort {
|
||||
child.env("SGLANG_DSV41_REASONING_EFFORT", effort);
|
||||
}
|
||||
let output = child.output().unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"effort={effort:?}: {}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
return;
|
||||
};
|
||||
let enc = ChatFormatter::deepseek_native(Some("deepseek_v41"), "alias").unwrap();
|
||||
for (effort, budget) in [
|
||||
(None, default_budget.as_str()),
|
||||
(Some("unsupported"), default_budget.as_str()),
|
||||
(Some("low"), "50"),
|
||||
(Some("high"), "75"),
|
||||
(Some("xhigh"), "75"),
|
||||
(Some("max"), "100"),
|
||||
] {
|
||||
let mut req = request(json!([{"role":"user","content":"hi"}]));
|
||||
if let Some(effort) = effort {
|
||||
req["reasoning_effort"] = json!(effort);
|
||||
}
|
||||
let prompt = enc.render(&req).unwrap();
|
||||
assert!(
|
||||
prompt.contains(&format!("Reasoning Effort: {budget} (range 1-100,")),
|
||||
"request={req}, prompt={prompt}"
|
||||
);
|
||||
assert!(prompt.ends_with("<think>"), "{prompt}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Messages reach the template shaped like the engine's request schema.
|
||||
#[test]
|
||||
fn messages_match_engine_schema() {
|
||||
|
||||
@@ -28,6 +28,7 @@ impl Encoder {
|
||||
request: &Value,
|
||||
mut messages: Vec<Value>,
|
||||
kwargs: &ChatTemplateKwargs,
|
||||
effort: Option<Value>,
|
||||
) -> Result<String> {
|
||||
// SGLang drops a later user's task when merging it into an existing
|
||||
// user/tool-result turn. Dynamo otherwise preserves that field.
|
||||
@@ -39,8 +40,8 @@ impl Encoder {
|
||||
}
|
||||
}
|
||||
match self {
|
||||
Self::V4(profile) => profile.render(request, messages, kwargs),
|
||||
Self::V41 => render_v41(request, messages, kwargs),
|
||||
Self::V4(profile) => profile.render(request, messages, kwargs, effort),
|
||||
Self::V41 => render_v41(request, messages, kwargs, effort),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -100,6 +101,7 @@ impl V4Profile {
|
||||
request: &Value,
|
||||
mut messages: Vec<Value>,
|
||||
kwargs: &ChatTemplateKwargs,
|
||||
effort: Option<Value>,
|
||||
) -> Result<String> {
|
||||
ensure!(
|
||||
!messages.is_empty(),
|
||||
@@ -113,10 +115,14 @@ impl V4Profile {
|
||||
if let Some(tools) = request["tools"].as_array().filter(|t| !t.is_empty()) {
|
||||
messages[0]["tools"] = normalize_tools(tools)?.into();
|
||||
}
|
||||
let effort = match (
|
||||
self,
|
||||
request_effort(request).as_ref().and_then(Value::as_str),
|
||||
) {
|
||||
// Without a request effort, SGLang uses SGLANG_DSV4_REASONING_EFFORT.
|
||||
let effort = effort.or_else(|| {
|
||||
std::env::var("SGLANG_DSV4_REASONING_EFFORT")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
.map(Value::from)
|
||||
});
|
||||
let effort = match (self, effort.as_ref().and_then(Value::as_str)) {
|
||||
(Self::Official, Some("high")) | (Self::Preview, Some("max")) => {
|
||||
Some(ReasoningEffort::High)
|
||||
}
|
||||
@@ -248,6 +254,7 @@ fn render_v41(
|
||||
request: &Value,
|
||||
mut messages: Vec<Value>,
|
||||
kwargs: &ChatTemplateKwargs,
|
||||
effort: Option<Value>,
|
||||
) -> Result<String> {
|
||||
ensure!(
|
||||
!messages.is_empty(),
|
||||
@@ -294,12 +301,35 @@ fn render_v41(
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
// chat_encoding.parse_dsv41_reasoning_effort; unsupported values use the
|
||||
// engine default (`high`), matching SGLANG_DSV41_REASONING_EFFORT unset.
|
||||
let budget = match request_effort(request) {
|
||||
// Unsupported values use SGLANG_DSV41_REASONING_EFFORT, else `high`.
|
||||
let env_effort = std::env::var("SGLANG_DSV41_REASONING_EFFORT")
|
||||
.ok()
|
||||
.map(|v| {
|
||||
v.trim()
|
||||
.parse::<i64>()
|
||||
.map_or_else(|_| Value::from(v.trim()), Value::from)
|
||||
});
|
||||
let budget = dsv41_budget(effort)
|
||||
.or_else(|| dsv41_budget(env_effort))
|
||||
.unwrap_or(75);
|
||||
v41::encode_messages(
|
||||
&messages,
|
||||
if thinking {
|
||||
ThinkingMode::Thinking
|
||||
} else {
|
||||
ThinkingMode::Chat
|
||||
},
|
||||
true,
|
||||
budget,
|
||||
)
|
||||
}
|
||||
|
||||
/// `chat_encoding.parse_dsv41_reasoning_effort`, as a token budget.
|
||||
fn dsv41_budget(effort: Option<Value>) -> Option<u8> {
|
||||
match effort {
|
||||
Some(Value::String(s)) => match s.as_str() {
|
||||
"low" => Some(25),
|
||||
"high" => Some(50),
|
||||
"low" => Some(50),
|
||||
"high" => Some(75),
|
||||
"xhigh" => Some(75),
|
||||
"max" => Some(100),
|
||||
_ => None,
|
||||
@@ -313,17 +343,6 @@ fn render_v41(
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
.unwrap_or(50);
|
||||
v41::encode_messages(
|
||||
&messages,
|
||||
if thinking {
|
||||
ThinkingMode::Thinking
|
||||
} else {
|
||||
ThinkingMode::Chat
|
||||
},
|
||||
true,
|
||||
budget,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -68,6 +68,7 @@ impl TokenizerRegistry {
|
||||
me.inner.insert(m.id.clone(), t);
|
||||
match ChatFormatter::load(&m.id, &m.tokenizer_path) {
|
||||
Ok(Some(formatter)) => {
|
||||
let formatter = formatter.with_defaults(&m.default_chat_template_kwargs);
|
||||
me.formatters
|
||||
.insert(m.id.clone(), Arc::new(ChatFormatterEntry::new(formatter)));
|
||||
tracing::info!(model = %m.id, "dynamo-render chat rendering enabled");
|
||||
@@ -83,9 +84,9 @@ impl TokenizerRegistry {
|
||||
routing tokenization remains available");
|
||||
} else if me.has_chat_formatter(&m.id) {
|
||||
tracing::warn!(model = %m.id,
|
||||
"router-generated input_ids forwarding enabled: requires matching worker model \
|
||||
files and template defaults; native DeepSeek assumes SGLANG_DEFAULT_THINKING=false \
|
||||
and default SGLANG_DSV4_REASONING_EFFORT / SGLANG_DSV41_REASONING_EFFORT; worker parser overrides \
|
||||
"router-generated input_ids forwarding enabled: requires the workers' model files, \
|
||||
--default-chat-template-kwargs, SGLANG_DEFAULT_THINKING, and \
|
||||
SGLANG_DSV4_REASONING_EFFORT / SGLANG_DSV41_REASONING_EFFORT; worker parser overrides \
|
||||
(including --tool-call-parser deepseekv32), content-format detection, and \
|
||||
conversation-template stop strings are not replicated. Use \
|
||||
--disable-input-ids-forwarding for array-only templates or when these assumptions do not hold");
|
||||
@@ -181,6 +182,7 @@ mod tests {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: crate::config::DiscoveryBackend::StaticUrls(
|
||||
crate::config::StaticUrlsDiscoveryConfig {
|
||||
|
||||
@@ -605,6 +605,7 @@ mod tests {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://test:30000".into()],
|
||||
|
||||
@@ -142,6 +142,7 @@ async fn static_urls_pd_role_resolved_end_to_end() {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec![url.clone()],
|
||||
|
||||
@@ -123,6 +123,7 @@ fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -10,18 +10,18 @@
|
||||
{"name":"tools_named","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"tool_choice":{"type":"function","function":{"name":"bar"}}},"prompt":"<|begin▁of▁sentence|><|System|>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|></think>","token_count":256,"token_sha256":"89eaf9e9642f765c5f244da76ce14d680a6c22bf97f6a859d1946d528649d458"},
|
||||
{"name":"message_tools","profile":"v41","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}]},{"role":"user","content":"Hi"}]},"prompt":"<|begin▁of▁sentence|><|System|>SYS\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|></think>","token_count":272,"token_sha256":"d6ebecca3f9e9a498e18f1a5140ff8d2e833951d64329ff970ff8ef6157f0a0c"},
|
||||
{"name":"empty_message_tools","profile":"v41","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[]},{"role":"user","content":"Hi"}]},"prompt":"<|begin▁of▁sentence|><|System|>SYS<|User|>Hi<|Assistant|></think>","token_count":8,"token_sha256":"0c718f5b123ecfacb6fc0d0091dfc63b1b7959c9122b2365fe823b270b6c3d35"},
|
||||
{"name":"tool_results","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"foo","arguments":"{\"x\":\"雪\", \"y\":[1, true]}"}},{"id":"b","type":"function","function":{"name":"bar","arguments":{"z":2}}}],"reasoning_content":"Call both"},{"role":"tool","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}],"tool_call_id":"b"},{"role":"tool","content":null,"tool_call_id":"a"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|><think>Call both</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"foo\">\n<|DSML| parameter name=\"x\" string=\"true\">雪</|DSML| parameter>\n<|DSML| parameter name=\"y\" string=\"false\">[1, true]</|DSML| parameter>\n</|DSML| invoke>\n<|DSML| invoke name=\"bar\">\n<|DSML| parameter name=\"z\" string=\"false\">2</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|><|User|><tool_result></tool_result>\n\n<tool_result>Hello\n\nworld</tool_result><|Assistant|><think>","token_count":388,"token_sha256":"f9fb80a884084408d01041c452a9ece38a9c3a959a5fd3ed0289db3f6611c203"},
|
||||
{"name":"tool_results","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"foo","arguments":"{\"x\":\"雪\", \"y\":[1, true]}"}},{"id":"b","type":"function","function":{"name":"bar","arguments":{"z":2}}}],"reasoning_content":"Call both"},{"role":"tool","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}],"tool_call_id":"b"},{"role":"tool","content":null,"tool_call_id":"a"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|><think>Call both</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"foo\">\n<|DSML| parameter name=\"x\" string=\"true\">雪</|DSML| parameter>\n<|DSML| parameter name=\"y\" string=\"false\">[1, true]</|DSML| parameter>\n</|DSML| invoke>\n<|DSML| invoke name=\"bar\">\n<|DSML| parameter name=\"z\" string=\"false\">2</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|><|User|><tool_result></tool_result>\n\n<tool_result>Hello\n\nworld</tool_result><|Assistant|><think>","token_count":388,"token_sha256":"a02f2167e3b33c915e6b53619b61258ae96cbb4aa7bb0ee66b4287cde20ba4e4"},
|
||||
{"name":"kwargs_effort","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":"max"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"a29f18c6e65c09656d098c43e548224b67ced6a674f65186b7e132fb6d997d75"},
|
||||
{"name":"effort_conflict","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 25 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"80aa41f1cbf4d847db08af7c5f4dde508ea75aed6e2cb6b94f3a0a806e275d46"},
|
||||
{"name":"drop_thinking_ignored","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}],"chat_template_kwargs":{"thinking":true,"drop_thinking":false}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|></think>Answer<|end▁of▁sentence|><|User|>Again<|Assistant|><think>","token_count":37,"token_sha256":"f6cb0ab0ad4a220297622920684237a208098dd2b2cd368599bc49ebba7994f9"},
|
||||
{"name":"effort_conflict","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
|
||||
{"name":"drop_thinking_ignored","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}],"chat_template_kwargs":{"thinking":true,"drop_thinking":false}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|></think>Answer<|end▁of▁sentence|><|User|>Again<|Assistant|><think>","token_count":37,"token_sha256":"e1c17a3d6c711c55d0f3e2deb5c56950144267d0337d88ebe5a7ab8e5dd62136"},
|
||||
{"name":"thinking_false","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"thinking":false}},"prompt":"<|begin▁of▁sentence|><|User|>Hi<|Assistant|></think>","token_count":5,"token_sha256":"95aeed952d0173ec8849b13faec66eb61ec4412cd9c41dae0c405257559365a6"},
|
||||
{"name":"thinking_none","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"none","chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
|
||||
{"name":"thinking_none","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"none","chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
|
||||
{"name":"consecutive_task","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"user","content":"Again"}],"task":"domain"},"prompt":"<|begin▁of▁sentence|><|User|>Hi\n\nAgain<|Assistant|></think>","token_count":7,"token_sha256":"d05e248d8a06f2c06df20b7a71af0d4dfae1cd91ce9f614719b5842674042d6b"},
|
||||
{"name":"effort_high","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
|
||||
{"name":"effort_high","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
|
||||
{"name":"effort_max","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"max"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"a29f18c6e65c09656d098c43e548224b67ced6a674f65186b7e132fb6d997d75"},
|
||||
{"name":"task_action","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"task":"action"},"prompt":"<|begin▁of▁sentence|><|User|>Hi<|Assistant|></think><|action|>","token_count":6,"token_sha256":"925bcaae248cc41efabd3e5d8f660f17513e43174f1d34aba51a1399aba76e8e"},
|
||||
{"name":"effort_low","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"low"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 25 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"80aa41f1cbf4d847db08af7c5f4dde508ea75aed6e2cb6b94f3a0a806e275d46"},
|
||||
{"name":"effort_low","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"low"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
|
||||
{"name":"effort_xhigh","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"xhigh"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
|
||||
{"name":"effort_0.75","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":0.75},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
|
||||
{"name":"kwargs_budget","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":80}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 80 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"32a4462bbf34bce35dd5c163574188ebd71896ac46d58966e0a6c2481b1b4081"}
|
||||
{"name":"kwargs_budget","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":80}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 80 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"32a4462bbf34bce35dd5c163574188ebd71896ac46d58966e0a6c2481b1b4081"}
|
||||
]}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[
|
||||
{"name":"default","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}]},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"},
|
||||
{"name":"effort_none","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"none"},"token_count":58,"sha256":"d9764be8b6a9a04114759d162f5a1c4e67109a7af2d62e72c5faf8f150c05d59"},
|
||||
{"name":"effort_precedence","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"token_count":276,"sha256":"54e51363cdb9f32dd2a11d805c3b5797b969526c338c7a93d0ae7740457073dc"},
|
||||
{"name":"effort_precedence","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"token_count":275,"sha256":"8da270ec9559dfa53515591fe37b480e68f6b70275265d5086a2ff43a747d32a"},
|
||||
{"name":"reasoning_alias","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning":{"effort":"low"}},"token_count":275,"sha256":"8da270ec9559dfa53515591fe37b480e68f6b70275265d5086a2ff43a747d32a"},
|
||||
{"name":"enable_thinking_only","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"chat_template_kwargs":{"enable_thinking":false}},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"},
|
||||
{"name":"unsupported_effort","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"medium"},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"},
|
||||
|
||||
@@ -72,6 +72,7 @@ fn build_app_context(
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -51,6 +51,7 @@ fn config_for(_worker_url: &str) -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -37,6 +37,7 @@ pub fn config() -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -49,6 +49,7 @@ async fn failover_when_one_worker_dies() {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec![w1.url.clone(), w2.url.clone(), w3.url.clone()],
|
||||
|
||||
@@ -64,6 +64,7 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc<AppContext> {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -42,6 +42,7 @@ async fn forwards_whitelisted_headers_strips_others() {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -57,6 +57,7 @@ fn config() -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -56,6 +56,7 @@ fn config() -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -144,6 +144,7 @@ fn config() -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -54,6 +54,7 @@ fn config() -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -160,6 +160,7 @@ fn config(policy: PolicyKind) -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -72,6 +72,7 @@ fn config() -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -58,6 +58,7 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc<AppContext
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -50,6 +50,7 @@ fn config(_worker_url: &str) -> Config {
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
default_chat_template_kwargs: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
|
||||
@@ -51,6 +51,12 @@ for case in cases:
|
||||
if "repeat" in case:
|
||||
data["messages"][0]["content"] *= case["repeat"]
|
||||
request = ChatCompletionRequest(**data)
|
||||
# _convert_to_internal_request runs before _encode_messages and promotes
|
||||
# the request kwarg over the top-level effort.
|
||||
if request.chat_template_kwargs:
|
||||
effort = request.chat_template_kwargs.pop("reasoning_effort", None)
|
||||
if effort is not None:
|
||||
request.reasoning_effort = effort
|
||||
messages = [message.model_dump() for message in request.messages]
|
||||
for message in messages:
|
||||
normalize_assistant_tool_call_arguments(message, strict=False)
|
||||
|
||||
Reference in New Issue
Block a user