[sgl-router] Take SGLang's render defaults: --default-chat-template-kwargs and thinking/effort envs (#41221)

Co-authored-by: Kan Wu <kan.wu@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-09-27 11:01:17 +08:00
committed by GitHub
co-authored by Kan Wu Claude Opus 5.5
parent effb752188
commit b4584ab838
31 changed files with 377 additions and 49 deletions
+14 -9
View File
@@ -205,8 +205,13 @@ rendered tokens to the engine as `input_ids`, retaining the original messages,
so the engine skips re-tokenizing. Every other request shape is rendered for
routing only: the router renders with dynamo-render and does not replicate
SGLang's request normalization, so forwarding is enabled shape by shape as
parity is verified. Use matching model files on the router and workers; worker
template overrides and default kwargs are not observable from the request.
parity is verified. Use matching model files on the router and workers, and set
the same `--default-chat-template-kwargs`, `SGLANG_DEFAULT_THINKING`,
`SGLANG_DSV4_REASONING_EFFORT`, and `SGLANG_DSV41_REASONING_EFFORT` on both.
The router reads these render defaults from its own configuration and environment;
it does not discover the workers' settings. Point `--tokenizer-path` at the workers'
model snapshot so the V4 effort profile is read from the same
`encoding/encoding_dsv4.py`.
Set `--disable-input-ids-forwarding` for this router's model when worker-side
rendering has not been verified to match. This disables router-generated IDs
@@ -214,13 +219,13 @@ for every routing policy; cache-aware routing still renders and tokenizes
locally, and the original messages reach the workers for engine processing.
Caller-supplied `input_ids` remain caller-owned and pass through unchanged.
Forwarding logs its assumptions at startup. In particular, disable it for
`SGLANG_DEFAULT_THINKING=true`, a non-default `SGLANG_DSV4_REASONING_EFFORT`,
worker parser overrides such as `--tool-call-parser deepseekv32` that select a
native encoder over a shipped template, or conversation templates with stop
strings (the engine's `input_ids` path skips those template stops). These worker
settings are not inferred from the router's environment. Disabling forwarding preserves
engine behavior but does not establish parity for local routing hashes.
Forwarding logs its assumptions at startup. In particular, disable it when the
router's render defaults differ from the workers', for worker template overrides
not reflected in the router's model files, for worker parser overrides such as
`--tool-call-parser deepseekv32` that select a native encoder over a shipped
template, or conversation templates with stop strings (the engine's `input_ids`
path skips those template stops). Disabling forwarding preserves engine behavior
but does not establish parity for local routing hashes.
Also set `--disable-input-ids-forwarding` for array-only templates: Dynamo may wrap
string content into arrays differently from the worker. The pinned Dynamo renderer does not expose
+60 -2
View File
@@ -3,7 +3,7 @@
//! Grouped CLI options and conversion into a validated [`Config`].
use anyhow::{anyhow, ensure, Result};
use anyhow::{anyhow, ensure, Context, Result};
use clap::Parser;
use std::num::NonZeroU32;
@@ -66,10 +66,14 @@ pub struct ModelArgs {
pub tokenizer_path: Option<String>,
/// Disable generated input_ids; workers tokenize messages, while routing still renders locally.
/// Use for worker-only thinking defaults, parser/template overrides, or template stop strings.
/// Use for worker parser/template overrides or template stop strings.
#[arg(long)]
pub disable_input_ids_forwarding: bool,
/// Same as SGLang's --default-chat-template-kwargs; must match the workers.
#[arg(long, value_name = "JSON")]
pub default_chat_template_kwargs: Option<String>,
/// Fleet sampling defaults as JSON, e.g. {"temperature": 1, "top_p": 0.95}.
/// Accepts temperature, top_p, top_k, min_p, repetition_penalty,
/// frequency_penalty, presence_penalty, and n. Numeric values fill absent
@@ -383,6 +387,14 @@ impl Cli {
})
.transpose()?
.unwrap_or_default();
let default_chat_template_kwargs = self
.model
.default_chat_template_kwargs
.as_deref()
.map(serde_json::from_str)
.transpose()
.context("--default-chat-template-kwargs must be a JSON object")?
.unwrap_or_default();
let config = Config {
server: ServerConfig {
@@ -412,6 +424,7 @@ impl Cli {
fused,
eligibility,
sampling_overrides,
default_chat_template_kwargs,
},
discovery,
proxy: ProxyConfig {
@@ -2396,6 +2409,51 @@ mod tests {
.to_string();
assert!(err.contains("unknown parameter"), "got: {err}");
}
#[test]
fn default_chat_template_kwargs_reaches_the_model_config() {
let kwargs = serde_json::json!({"thinking": true, "reasoning_effort": "high"});
let config = into_config_owned(with_model(&[
"--worker-urls",
"http://x:30000",
"--default-chat-template-kwargs",
&kwargs.to_string(),
]))
.unwrap();
assert_eq!(
serde_json::to_value(config.model.default_chat_template_kwargs).unwrap(),
kwargs
);
let defaults = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap();
assert!(defaults.model.default_chat_template_kwargs.is_empty());
}
#[test]
fn malformed_default_chat_template_kwargs_fails_the_launch() {
for kwargs in [
r#"{"thinking": true"#,
"[]",
"null",
"true",
"42",
r#""high""#,
] {
let err = into_config_owned(with_model(&[
"--worker-urls",
"http://x:30000",
"--default-chat-template-kwargs",
kwargs,
]))
.unwrap_err()
.to_string();
assert!(
err.contains("--default-chat-template-kwargs must be a JSON object"),
"kwargs={kwargs}, got: {err}"
);
}
}
#[test]
fn input_ids_forwarding_can_be_disabled_for_the_model() {
let defaults = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap();
@@ -236,6 +236,7 @@ mod tests {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: urls.iter().map(|s| s.to_string()).collect(),
@@ -330,6 +330,8 @@ pub struct ModelConfig {
pub eligibility: Option<EligibilityConfig>,
/// Fleet sampling defaults and conflict behavior. See [`SamplingOverrides`].
pub sampling_overrides: SamplingOverrides,
/// Worker `--default-chat-template-kwargs`, applied when rendering.
pub default_chat_template_kwargs: crate::tokenizer::chat_formatter::ChatTemplateKwargs,
}
/// External KV Indexer client settings.
@@ -348,6 +348,7 @@ mod tests {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -196,6 +196,7 @@ impl AppContext {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: crate::config::DiscoveryBackend::StaticUrls(
crate::config::StaticUrlsDiscoveryConfig {
@@ -61,6 +61,7 @@ mod tests {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
};
let app = crate::server::app::build_router(std::sync::Arc::new(ctx));
let res = app
@@ -129,6 +129,7 @@ mod tests {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: crate::config::DiscoveryBackend::StaticUrls(
crate::config::StaticUrlsDiscoveryConfig {
@@ -42,6 +42,8 @@ pub struct ChatFormatter {
bos_token: Option<String>,
deepseek: Option<super::deepseek::Encoder>,
is_kimi_k3: bool,
/// Worker defaults, merged before model-specific effort conversion.
worker_defaults: ChatTemplateKwargs,
}
impl ChatFormatter {
@@ -163,6 +165,7 @@ impl ChatFormatter {
bos_token,
deepseek: None,
is_kimi_k3: false,
worker_defaults: ChatTemplateKwargs::new(),
}))
}
@@ -178,6 +181,7 @@ impl ChatFormatter {
bos_token: Some("[BOS]".into()),
deepseek: None,
is_kimi_k3: true,
worker_defaults: ChatTemplateKwargs::new(),
})
}
@@ -205,10 +209,12 @@ impl ChatFormatter {
.map_or(name.contains("v4.1") || name.contains("v41"), |t| {
t == "deepseek_v41"
});
// Engine defaults: chat mode (`SGLANG_DEFAULT_THINKING=false`) and no
// reasoning-effort preamble; dynamo-render defaults to thinking at high effort.
// Engine defaults: `SGLANG_DEFAULT_THINKING` and no reasoning-effort
// preamble; dynamo-render defaults to thinking at high effort.
let thinking = std::env::var("SGLANG_DEFAULT_THINKING")
.is_ok_and(|v| ["true", "1", "yes", "y"].contains(&v.to_lowercase().as_str()));
let defaults = HashMap::from([
("thinking".into(), false.into()),
("thinking".into(), thinking.into()),
("reasoning_effort".into(), "low".into()),
]);
Some(Self {
@@ -221,6 +227,23 @@ impl ChatFormatter {
is_deepseek_v4.then(|| super::deepseek::Encoder::V4(Default::default()))
},
is_kimi_k3: false,
worker_defaults: ChatTemplateKwargs::new(),
})
}
/// Apply the workers' `--default-chat-template-kwargs`; they fill keys the
/// request leaves unset, and a default `reasoning_effort` acts as the request's.
pub fn with_defaults(mut self, defaults: &ChatTemplateKwargs) -> Self {
self.worker_defaults = defaults.clone();
self
}
fn effective_effort(&self, request: &JsonValue) -> Option<JsonValue> {
super::deepseek::request_effort(request).or_else(|| {
self.worker_defaults
.get("reasoning_effort")
.filter(|v| !v.is_null())
.cloned()
})
}
@@ -270,7 +293,14 @@ impl ChatFormatter {
.entry("enable_thinking".into())
.or_insert(thinking.into());
}
if let Some(effort) = effort {
// serving_chat first pops the request kwarg into request.reasoning_effort,
// then merges worker defaults. Jinja receives those merged kwargs over
// the effective effort; Kimi derives thinking_effort only if still absent.
kwargs.remove("reasoning_effort");
for (key, value) in &self.worker_defaults {
kwargs.entry(key.clone()).or_insert_with(|| value.clone());
}
if let Some(effort) = self.effective_effort(request) {
if self.is_kimi_k3 {
if matches!(effort.as_str(), Some("low" | "high" | "max")) {
kwargs.entry("thinking_effort".into()).or_insert(effort);
@@ -332,8 +362,9 @@ impl ChatFormatter {
}
}
if let Some(profile) = self.deepseek {
let effort = self.effective_effort(request);
return Ok((
RenderedPrompt::text(profile.render(request, messages, &kwargs)?),
RenderedPrompt::text(profile.render(request, messages, &kwargs, effort)?),
prefix,
));
}
@@ -785,6 +816,189 @@ mod tests {
assert_eq!(enc.render(&req).unwrap(), ":True:True");
}
#[test]
fn worker_defaults_preserve_jinja_effort_precedence() {
// The engine pops request kwargs.reasoning_effort before merging defaults,
// then overlays merged kwargs on the effective request effort for Jinja.
for (defaults, controls, expected) in [
(json!({"reasoning_effort":"low"}), json!({}), json!("low")),
(
json!({"reasoning_effort":"low"}),
json!({"reasoning_effort":"high"}),
json!("low"),
),
(
json!({"reasoning_effort":"low"}),
json!({"chat_template_kwargs":{"reasoning_effort":"high"}}),
json!("low"),
),
(
json!({"reasoning_effort":null}),
json!({"reasoning_effort":"high"}),
JsonValue::Null,
),
(json!({}), json!({"reasoning_effort":"high"}), json!("high")),
(
json!({}),
json!({"reasoning_effort":"low", "chat_template_kwargs":{"reasoning_effort":"high"}}),
json!("high"),
),
] {
let enc = jinja(json!({"chat_template":"{{ reasoning_effort | tojson }}"}))
.with_defaults(&serde_json::from_value(defaults.clone()).unwrap());
let mut req = request(json!([{"role":"user","content":"hi"}]));
req.as_object_mut()
.unwrap()
.extend(controls.as_object().unwrap().clone());
assert_eq!(
enc.render(&req).unwrap(),
expected.to_string(),
"defaults={defaults}, request={req}"
);
}
}
#[test]
fn kimi_worker_defaults_reach_thinking_effort() {
for (defaults, controls, expected) in [
(json!({"reasoning_effort":"low"}), json!({}), "low"),
(
json!({"reasoning_effort":"low"}),
json!({"reasoning_effort":"high"}),
"high",
),
(
json!({"reasoning_effort":"low"}),
json!({"chat_template_kwargs":{"reasoning_effort":"high"}}),
"high",
),
(
json!({"thinking_effort":"high"}),
json!({"reasoning_effort":"low"}),
"high",
),
(
json!({"reasoning_effort":"low", "thinking_effort":"high"}),
json!({"chat_template_kwargs":{"thinking_effort":"max"}}),
"max",
),
] {
let enc = ChatFormatter::kimi_native(Some("kimi_k3"), "kimi-k3")
.unwrap()
.with_defaults(&serde_json::from_value(defaults.clone()).unwrap());
let mut req = request(json!([{"role":"user","content":"hi"}]));
req.as_object_mut()
.unwrap()
.extend(controls.as_object().unwrap().clone());
let prompt = enc.render(&req).unwrap();
assert!(
prompt.contains(&format!("thinking_effort={expected}")),
"defaults={defaults}, request={req}, prompt={prompt}"
);
}
}
#[test]
fn worker_effort_does_not_derive_thinking_flags() {
let enc = jinja(json!({"chat_template":"{{ thinking | default('unset') }}:{{ enable_thinking | default('unset') }}"}))
.with_defaults(&serde_json::from_value(json!({"reasoning_effort":"high"})).unwrap());
let req = request(json!([{"role":"user","content":"hi"}]));
assert_eq!(enc.render(&req).unwrap(), "unset:unset");
}
#[test]
fn null_worker_effort_preserves_deepseek_env_fallback() {
// Set process-wide envs only in a child so parallel formatter tests retain
// their normal defaults.
const CHILD: &str = "SGL_ROUTER_TEST_NULL_WORKER_EFFORT_CHILD";
if std::env::var_os(CHILD).is_none() {
let output = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", "tokenizer::chat_formatter::tests::null_worker_effort_preserves_deepseek_env_fallback", "--nocapture"])
.env(CHILD, "1")
.env("SGLANG_DSV4_REASONING_EFFORT", "max")
.env("SGLANG_DSV41_REASONING_EFFORT", "low")
.output()
.unwrap();
assert!(
output.status.success(),
"{}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
return;
}
for (model_type, effort) in [("deepseek_v4", "max"), ("deepseek_v41", "low")] {
let enc = ChatFormatter::deepseek_native(Some(model_type), model_type)
.unwrap()
.with_defaults(
&serde_json::from_value(json!({"thinking":true, "reasoning_effort":null}))
.unwrap(),
);
let mut req = request(json!([{"role":"user","content":"hi"}]));
let actual = enc.render(&req).unwrap();
req["reasoning_effort"] = json!(effort);
assert_eq!(actual, enc.render(&req).unwrap(), "{model_type}");
}
}
#[test]
fn v41_thinking_env_uses_engine_budgets() {
// Isolate the environment so other formatter tests keep their defaults.
const CHILD: &str = "SGL_ROUTER_TEST_V41_BUDGET_CHILD";
let Ok(default_budget) = std::env::var(CHILD) else {
for (effort, budget) in [
(None, 75),
(Some("low"), 50),
(Some("high"), 75),
(Some("xhigh"), 75),
(Some("max"), 100),
(Some("42"), 42),
] {
let mut child = std::process::Command::new(std::env::current_exe().unwrap());
child
.args([
"--exact",
"tokenizer::chat_formatter::tests::v41_thinking_env_uses_engine_budgets",
"--nocapture",
])
.env(CHILD, budget.to_string())
.env("SGLANG_DEFAULT_THINKING", "1")
.env_remove("SGLANG_DSV41_REASONING_EFFORT");
if let Some(effort) = effort {
child.env("SGLANG_DSV41_REASONING_EFFORT", effort);
}
let output = child.output().unwrap();
assert!(
output.status.success(),
"effort={effort:?}: {}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
return;
};
let enc = ChatFormatter::deepseek_native(Some("deepseek_v41"), "alias").unwrap();
for (effort, budget) in [
(None, default_budget.as_str()),
(Some("unsupported"), default_budget.as_str()),
(Some("low"), "50"),
(Some("high"), "75"),
(Some("xhigh"), "75"),
(Some("max"), "100"),
] {
let mut req = request(json!([{"role":"user","content":"hi"}]));
if let Some(effort) = effort {
req["reasoning_effort"] = json!(effort);
}
let prompt = enc.render(&req).unwrap();
assert!(
prompt.contains(&format!("Reasoning Effort: {budget} (range 1-100,")),
"request={req}, prompt={prompt}"
);
assert!(prompt.ends_with("<think>"), "{prompt}");
}
}
/// Messages reach the template shaped like the engine's request schema.
#[test]
fn messages_match_engine_schema() {
@@ -28,6 +28,7 @@ impl Encoder {
request: &Value,
mut messages: Vec<Value>,
kwargs: &ChatTemplateKwargs,
effort: Option<Value>,
) -> Result<String> {
// SGLang drops a later user's task when merging it into an existing
// user/tool-result turn. Dynamo otherwise preserves that field.
@@ -39,8 +40,8 @@ impl Encoder {
}
}
match self {
Self::V4(profile) => profile.render(request, messages, kwargs),
Self::V41 => render_v41(request, messages, kwargs),
Self::V4(profile) => profile.render(request, messages, kwargs, effort),
Self::V41 => render_v41(request, messages, kwargs, effort),
}
}
}
@@ -100,6 +101,7 @@ impl V4Profile {
request: &Value,
mut messages: Vec<Value>,
kwargs: &ChatTemplateKwargs,
effort: Option<Value>,
) -> Result<String> {
ensure!(
!messages.is_empty(),
@@ -113,10 +115,14 @@ impl V4Profile {
if let Some(tools) = request["tools"].as_array().filter(|t| !t.is_empty()) {
messages[0]["tools"] = normalize_tools(tools)?.into();
}
let effort = match (
self,
request_effort(request).as_ref().and_then(Value::as_str),
) {
// Without a request effort, SGLang uses SGLANG_DSV4_REASONING_EFFORT.
let effort = effort.or_else(|| {
std::env::var("SGLANG_DSV4_REASONING_EFFORT")
.ok()
.filter(|v| !v.is_empty())
.map(Value::from)
});
let effort = match (self, effort.as_ref().and_then(Value::as_str)) {
(Self::Official, Some("high")) | (Self::Preview, Some("max")) => {
Some(ReasoningEffort::High)
}
@@ -248,6 +254,7 @@ fn render_v41(
request: &Value,
mut messages: Vec<Value>,
kwargs: &ChatTemplateKwargs,
effort: Option<Value>,
) -> Result<String> {
ensure!(
!messages.is_empty(),
@@ -294,12 +301,35 @@ fn render_v41(
})
.collect();
}
// chat_encoding.parse_dsv41_reasoning_effort; unsupported values use the
// engine default (`high`), matching SGLANG_DSV41_REASONING_EFFORT unset.
let budget = match request_effort(request) {
// Unsupported values use SGLANG_DSV41_REASONING_EFFORT, else `high`.
let env_effort = std::env::var("SGLANG_DSV41_REASONING_EFFORT")
.ok()
.map(|v| {
v.trim()
.parse::<i64>()
.map_or_else(|_| Value::from(v.trim()), Value::from)
});
let budget = dsv41_budget(effort)
.or_else(|| dsv41_budget(env_effort))
.unwrap_or(75);
v41::encode_messages(
&messages,
if thinking {
ThinkingMode::Thinking
} else {
ThinkingMode::Chat
},
true,
budget,
)
}
/// `chat_encoding.parse_dsv41_reasoning_effort`, as a token budget.
fn dsv41_budget(effort: Option<Value>) -> Option<u8> {
match effort {
Some(Value::String(s)) => match s.as_str() {
"low" => Some(25),
"high" => Some(50),
"low" => Some(50),
"high" => Some(75),
"xhigh" => Some(75),
"max" => Some(100),
_ => None,
@@ -313,17 +343,6 @@ fn render_v41(
},
_ => None,
}
.unwrap_or(50);
v41::encode_messages(
&messages,
if thinking {
ThinkingMode::Thinking
} else {
ThinkingMode::Chat
},
true,
budget,
)
}
#[cfg(test)]
+5 -3
View File
@@ -68,6 +68,7 @@ impl TokenizerRegistry {
me.inner.insert(m.id.clone(), t);
match ChatFormatter::load(&m.id, &m.tokenizer_path) {
Ok(Some(formatter)) => {
let formatter = formatter.with_defaults(&m.default_chat_template_kwargs);
me.formatters
.insert(m.id.clone(), Arc::new(ChatFormatterEntry::new(formatter)));
tracing::info!(model = %m.id, "dynamo-render chat rendering enabled");
@@ -83,9 +84,9 @@ impl TokenizerRegistry {
routing tokenization remains available");
} else if me.has_chat_formatter(&m.id) {
tracing::warn!(model = %m.id,
"router-generated input_ids forwarding enabled: requires matching worker model \
files and template defaults; native DeepSeek assumes SGLANG_DEFAULT_THINKING=false \
and default SGLANG_DSV4_REASONING_EFFORT / SGLANG_DSV41_REASONING_EFFORT; worker parser overrides \
"router-generated input_ids forwarding enabled: requires the workers' model files, \
--default-chat-template-kwargs, SGLANG_DEFAULT_THINKING, and \
SGLANG_DSV4_REASONING_EFFORT / SGLANG_DSV41_REASONING_EFFORT; worker parser overrides \
(including --tool-call-parser deepseekv32), content-format detection, and \
conversation-template stop strings are not replicated. Use \
--disable-input-ids-forwarding for array-only templates or when these assumptions do not hold");
@@ -181,6 +182,7 @@ mod tests {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: crate::config::DiscoveryBackend::StaticUrls(
crate::config::StaticUrlsDiscoveryConfig {
@@ -605,6 +605,7 @@ mod tests {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://test:30000".into()],
@@ -142,6 +142,7 @@ async fn static_urls_pd_role_resolved_end_to_end() {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec![url.clone()],
@@ -123,6 +123,7 @@ fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
+7 -7
View File
@@ -10,18 +10,18 @@
{"name":"tools_named","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"tool_choice":{"type":"function","function":{"name":"bar"}}},"prompt":"<|begin▁of▁sentence|><|System|>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|></think>","token_count":256,"token_sha256":"89eaf9e9642f765c5f244da76ce14d680a6c22bf97f6a859d1946d528649d458"},
{"name":"message_tools","profile":"v41","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}]},{"role":"user","content":"Hi"}]},"prompt":"<|begin▁of▁sentence|><|System|>SYS\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|></think>","token_count":272,"token_sha256":"d6ebecca3f9e9a498e18f1a5140ff8d2e833951d64329ff970ff8ef6157f0a0c"},
{"name":"empty_message_tools","profile":"v41","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[]},{"role":"user","content":"Hi"}]},"prompt":"<|begin▁of▁sentence|><|System|>SYS<|User|>Hi<|Assistant|></think>","token_count":8,"token_sha256":"0c718f5b123ecfacb6fc0d0091dfc63b1b7959c9122b2365fe823b270b6c3d35"},
{"name":"tool_results","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"foo","arguments":"{\"x\":\"雪\", \"y\":[1, true]}"}},{"id":"b","type":"function","function":{"name":"bar","arguments":{"z":2}}}],"reasoning_content":"Call both"},{"role":"tool","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}],"tool_call_id":"b"},{"role":"tool","content":null,"tool_call_id":"a"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|><think>Call both</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"foo\">\n<|DSML| parameter name=\"x\" string=\"true\">雪</|DSML| parameter>\n<|DSML| parameter name=\"y\" string=\"false\">[1, true]</|DSML| parameter>\n</|DSML| invoke>\n<|DSML| invoke name=\"bar\">\n<|DSML| parameter name=\"z\" string=\"false\">2</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|><|User|><tool_result></tool_result>\n\n<tool_result>Hello\n\nworld</tool_result><|Assistant|><think>","token_count":388,"token_sha256":"f9fb80a884084408d01041c452a9ece38a9c3a959a5fd3ed0289db3f6611c203"},
{"name":"tool_results","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"foo","arguments":"{\"x\":\"雪\", \"y\":[1, true]}"}},{"id":"b","type":"function","function":{"name":"bar","arguments":{"z":2}}}],"reasoning_content":"Call both"},{"role":"tool","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}],"tool_call_id":"b"},{"role":"tool","content":null,"tool_call_id":"a"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n<|DSML| calls>\n<|DSML| invoke name=\"$TOOL_NAME\">\n<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</|DSML| parameter>\n...\n</|DSML| invoke>\n<|DSML| invoke name=\"$TOOL_NAME2\">\n...\n</|DSML| invoke>\n</|DSML| calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}}\n{\"name\": \"bar\", \"description\": \"第二个\", \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<|User|>Hi<|Assistant|><think>Call both</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"foo\">\n<|DSML| parameter name=\"x\" string=\"true\">雪</|DSML| parameter>\n<|DSML| parameter name=\"y\" string=\"false\">[1, true]</|DSML| parameter>\n</|DSML| invoke>\n<|DSML| invoke name=\"bar\">\n<|DSML| parameter name=\"z\" string=\"false\">2</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls><|end▁of▁sentence|><|User|><tool_result></tool_result>\n\n<tool_result>Hello\n\nworld</tool_result><|Assistant|><think>","token_count":388,"token_sha256":"a02f2167e3b33c915e6b53619b61258ae96cbb4aa7bb0ee66b4287cde20ba4e4"},
{"name":"kwargs_effort","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":"max"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"a29f18c6e65c09656d098c43e548224b67ced6a674f65186b7e132fb6d997d75"},
{"name":"effort_conflict","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 25 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"80aa41f1cbf4d847db08af7c5f4dde508ea75aed6e2cb6b94f3a0a806e275d46"},
{"name":"drop_thinking_ignored","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}],"chat_template_kwargs":{"thinking":true,"drop_thinking":false}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|></think>Answer<|end▁of▁sentence|><|User|>Again<|Assistant|><think>","token_count":37,"token_sha256":"f6cb0ab0ad4a220297622920684237a208098dd2b2cd368599bc49ebba7994f9"},
{"name":"effort_conflict","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
{"name":"drop_thinking_ignored","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}],"chat_template_kwargs":{"thinking":true,"drop_thinking":false}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|></think>Answer<|end▁of▁sentence|><|User|>Again<|Assistant|><think>","token_count":37,"token_sha256":"e1c17a3d6c711c55d0f3e2deb5c56950144267d0337d88ebe5a7ab8e5dd62136"},
{"name":"thinking_false","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"thinking":false}},"prompt":"<|begin▁of▁sentence|><|User|>Hi<|Assistant|></think>","token_count":5,"token_sha256":"95aeed952d0173ec8849b13faec66eb61ec4412cd9c41dae0c405257559365a6"},
{"name":"thinking_none","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"none","chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
{"name":"thinking_none","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"none","chat_template_kwargs":{"thinking":true}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
{"name":"consecutive_task","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"user","content":"Again"}],"task":"domain"},"prompt":"<|begin▁of▁sentence|><|User|>Hi\n\nAgain<|Assistant|></think>","token_count":7,"token_sha256":"d05e248d8a06f2c06df20b7a71af0d4dfae1cd91ce9f614719b5842674042d6b"},
{"name":"effort_high","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
{"name":"effort_high","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
{"name":"effort_max","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"max"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"a29f18c6e65c09656d098c43e548224b67ced6a674f65186b7e132fb6d997d75"},
{"name":"task_action","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"task":"action"},"prompt":"<|begin▁of▁sentence|><|User|>Hi<|Assistant|></think><|action|>","token_count":6,"token_sha256":"925bcaae248cc41efabd3e5d8f660f17513e43174f1d34aba51a1399aba76e8e"},
{"name":"effort_low","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"low"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 25 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"80aa41f1cbf4d847db08af7c5f4dde508ea75aed6e2cb6b94f3a0a806e275d46"},
{"name":"effort_low","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"low"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"4add8297343cd581f7ded6ecba6c06c5f70f402802db9a021d469efdc2138a15"},
{"name":"effort_xhigh","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"xhigh"},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
{"name":"effort_0.75","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":0.75},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"54a0e744ad23d9a75775f4a71851980f51a778a1011d73de2f370b1d5499a5b7"},
{"name":"kwargs_budget","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":80}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 80 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"32a4462bbf34bce35dd5c163574188ebd71896ac46d58966e0a6c2481b1b4081"}
{"name":"kwargs_budget","profile":"v41","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":80}},"prompt":"<|begin▁of▁sentence|><|System|>Reasoning Effort: 80 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>Hi<|Assistant|><think>","token_count":31,"token_sha256":"32a4462bbf34bce35dd5c163574188ebd71896ac46d58966e0a6c2481b1b4081"}
]}
@@ -1,7 +1,7 @@
[
{"name":"default","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}]},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"},
{"name":"effort_none","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"none"},"token_count":58,"sha256":"d9764be8b6a9a04114759d162f5a1c4e67109a7af2d62e72c5faf8f150c05d59"},
{"name":"effort_precedence","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"token_count":276,"sha256":"54e51363cdb9f32dd2a11d805c3b5797b969526c338c7a93d0ae7740457073dc"},
{"name":"effort_precedence","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"token_count":275,"sha256":"8da270ec9559dfa53515591fe37b480e68f6b70275265d5086a2ff43a747d32a"},
{"name":"reasoning_alias","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning":{"effort":"low"}},"token_count":275,"sha256":"8da270ec9559dfa53515591fe37b480e68f6b70275265d5086a2ff43a747d32a"},
{"name":"enable_thinking_only","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"chat_template_kwargs":{"enable_thinking":false}},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"},
{"name":"unsupported_effort","request":{"messages":[{"role":"user","content":"hello <|open|> <|kimi_image_placeholder|>"}],"reasoning_effort":"medium"},"token_count":275,"sha256":"43f237fef9c404e15ba9ac9a4fc948b6301b8a0bf5336576dd5e90c46d5c1cb8"},
@@ -72,6 +72,7 @@ fn build_app_context(
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -51,6 +51,7 @@ fn config_for(_worker_url: &str) -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -37,6 +37,7 @@ pub fn config() -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -49,6 +49,7 @@ async fn failover_when_one_worker_dies() {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec![w1.url.clone(), w2.url.clone(), w3.url.clone()],
@@ -64,6 +64,7 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc<AppContext> {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -42,6 +42,7 @@ async fn forwards_whitelisted_headers_strips_others() {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -57,6 +57,7 @@ fn config() -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -56,6 +56,7 @@ fn config() -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -144,6 +144,7 @@ fn config() -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -54,6 +54,7 @@ fn config() -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -160,6 +160,7 @@ fn config(policy: PolicyKind) -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -72,6 +72,7 @@ fn config() -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -58,6 +58,7 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc<AppContext
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -50,6 +50,7 @@ fn config(_worker_url: &str) -> Config {
fused: None,
eligibility: None,
sampling_overrides: Default::default(),
default_chat_template_kwargs: Default::default(),
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
@@ -51,6 +51,12 @@ for case in cases:
if "repeat" in case:
data["messages"][0]["content"] *= case["repeat"]
request = ChatCompletionRequest(**data)
# _convert_to_internal_request runs before _encode_messages and promotes
# the request kwarg over the top-level effort.
if request.chat_template_kwargs:
effort = request.chat_template_kwargs.pop("reasoning_effort", None)
if effort is not None:
request.reasoning_effort = effort
messages = [message.model_dump() for message in request.messages]
for message in messages:
normalize_assistant_tool_call_arguments(message, strict=False)