[router] Improve SGLang chat render parity (#39133)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Kan Wu
co-authored by
Claude Fable 5.1
Shangming Cai
Claude Fable 5.1
Shangming Cai
parent
7ccbf5fd04
commit
e4cbb28ea1
@@ -2,3 +2,4 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
mod parity;
|
||||
mod render_parity;
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Compare Dynamo prompt IDs with SGLang-generated fixtures using cached tokenizers.
|
||||
//! CI skips this matrix unless model snapshots are available.
|
||||
|
||||
use serde::Deserialize;
|
||||
use sgl_router::config::{
|
||||
ActiveLoadConfig, Config, DiscoveryBackend, ModelConfig, ObservabilityConfig, PolicyKind,
|
||||
ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig,
|
||||
};
|
||||
use sgl_router::discovery::ModelId;
|
||||
use sgl_router::policies::request_tokens_for;
|
||||
use sgl_router::tokenizer::{adapter, chat_formatter::ChatFormatter, TokenizerRegistry};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Fixture {
|
||||
model_id: String,
|
||||
cases: Vec<Case>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Case {
|
||||
shape: String,
|
||||
request: serde_json::Value,
|
||||
expected_token_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
/// String-to-array conversion is a known parity gap, so these templates must
|
||||
/// opt out of forwarding. This fixture needs no cached model files.
|
||||
#[test]
|
||||
fn array_only_template_content_parity() {
|
||||
let fixture: serde_json::Value =
|
||||
serde_json::from_str(include_str!("../../fixtures/array_content_rendering.json")).unwrap();
|
||||
let formatter = ChatFormatter::from_tokenizer_config(
|
||||
serde_json::json!({"chat_template": fixture["chat_template"]}),
|
||||
None,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let tokenizer = adapter::load("tests/fixtures/tiny_tokenizer.json").unwrap();
|
||||
for case in fixture["cases"].as_array().unwrap() {
|
||||
let request =
|
||||
serde_json::json!({"messages": [{"role": "user", "content": case["content"]}]});
|
||||
let ids = formatter.encode(&tokenizer, &request).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::json!(ids) == case["engine_token_ids"],
|
||||
case["content"].is_array(),
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace every `YYYY-MM-DD` with a placeholder.
|
||||
///
|
||||
/// Templates that call `strftime_now` render the day the prompt is built, so a
|
||||
/// fixture captured earlier differs from today's render in the date alone. That
|
||||
/// is not drift: the engine consumes forwarded IDs verbatim and never re-renders.
|
||||
/// Masking keeps the rest of the prompt under exact comparison, and dates that
|
||||
/// come from the request render the same on both sides, so masking them is a
|
||||
/// no-op.
|
||||
fn mask_dates(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut rest = text;
|
||||
while let Some(c) = rest.chars().next() {
|
||||
if starts_with_iso_date(rest) {
|
||||
out.push_str("<DATE>");
|
||||
rest = &rest[10..];
|
||||
} else {
|
||||
out.push(c);
|
||||
rest = &rest[c.len_utf8()..];
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn starts_with_iso_date(text: &str) -> bool {
|
||||
let b = text.as_bytes();
|
||||
b.len() >= 10
|
||||
&& b[..4].iter().all(u8::is_ascii_digit)
|
||||
&& b[4] == b'-'
|
||||
&& b[5..7].iter().all(u8::is_ascii_digit)
|
||||
&& b[7] == b'-'
|
||||
&& b[8..10].iter().all(u8::is_ascii_digit)
|
||||
}
|
||||
|
||||
fn snapshot_tokenizer(model_id: &str) -> Option<PathBuf> {
|
||||
let hf_home = std::env::var("HF_HOME")
|
||||
.ok()
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| dirs::home_dir().map(|h| h.join(".cache/huggingface")))?;
|
||||
let snapshots = hf_home
|
||||
.join("hub")
|
||||
.join(format!("models--{}", model_id.replace('/', "--")))
|
||||
.join("snapshots");
|
||||
std::fs::read_dir(snapshots)
|
||||
.ok()?
|
||||
.flatten()
|
||||
.map(|e| e.path().join("tokenizer.json"))
|
||||
.find(|p| p.is_file())
|
||||
}
|
||||
|
||||
fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry {
|
||||
let cfg = Config {
|
||||
server: ServerConfig {
|
||||
host: "0".into(),
|
||||
port: 0,
|
||||
..Default::default()
|
||||
},
|
||||
observability: ObservabilityConfig::default(),
|
||||
model: ModelConfig {
|
||||
id: model_id.into(),
|
||||
tokenizer_path: tokenizer_path.to_str().unwrap().into(),
|
||||
disable_input_ids_forwarding: false,
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
circuit_breaker: None,
|
||||
cache_aware: None,
|
||||
affinity: None,
|
||||
sticky: None,
|
||||
fused: None,
|
||||
eligibility: None,
|
||||
sampling_overrides: Default::default(),
|
||||
},
|
||||
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
|
||||
urls: vec!["http://placeholder:0".into()],
|
||||
}),
|
||||
proxy: ProxyConfig::default(),
|
||||
active_load: ActiveLoadConfig::default(),
|
||||
};
|
||||
TokenizerRegistry::load_from_config(&cfg).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_render_parity_matrix() {
|
||||
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/chat_render_parity");
|
||||
let mut checked = 0;
|
||||
for entry in std::fs::read_dir(&root).unwrap().flatten() {
|
||||
let raw = std::fs::read_to_string(entry.path()).unwrap();
|
||||
let fixture: Fixture = serde_json::from_str(&raw).unwrap();
|
||||
let Some(tokenizer_path) = snapshot_tokenizer(&fixture.model_id) else {
|
||||
eprintln!("skip {}: snapshot not cached", fixture.model_id);
|
||||
continue;
|
||||
};
|
||||
let reg = registry(&fixture.model_id, tokenizer_path);
|
||||
assert!(
|
||||
reg.has_chat_formatter(&fixture.model_id),
|
||||
"{}: no chat formatter resolved",
|
||||
fixture.model_id
|
||||
);
|
||||
for case in &fixture.cases {
|
||||
let tokens =
|
||||
request_tokens_for(®, &ModelId(fixture.model_id.clone()), &case.request)
|
||||
.unwrap_or_else(|| panic!("{}/{}: no tokens", fixture.model_id, case.shape));
|
||||
assert!(
|
||||
tokens.rendered_from_chat,
|
||||
"{}/{}: fell back to raw text",
|
||||
fixture.model_id, case.shape
|
||||
);
|
||||
if tokens.ids != case.expected_token_ids {
|
||||
// Decode with special tokens kept, so a difference in them still fails.
|
||||
let tokenizer = reg.get(&fixture.model_id).unwrap();
|
||||
let rendered = adapter::decode_complete(&tokenizer, &tokens.ids, false).unwrap();
|
||||
let expected =
|
||||
adapter::decode_complete(&tokenizer, &case.expected_token_ids, false).unwrap();
|
||||
assert_eq!(
|
||||
mask_dates(&rendered),
|
||||
mask_dates(&expected),
|
||||
"DRIFT on {}/{}",
|
||||
fixture.model_id,
|
||||
case.shape
|
||||
);
|
||||
eprintln!(
|
||||
"{}/{}: date drift only; fixture captured on another day",
|
||||
fixture.model_id, case.shape
|
||||
);
|
||||
}
|
||||
checked += 1;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
checked > 0,
|
||||
"no cached model snapshots; parity was not checked"
|
||||
);
|
||||
eprintln!("chat render parity: {checked} cases checked");
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{"chat_template": "{% for message in messages %}{{ message.role }}:{% for part in message.content %}{% if part.type == 'text' %}{{ part.text }}{% elif part.type == 'image' %}<image>{% endif %}{% endfor %};{% endfor %}{% if add_generation_prompt %}assistant:{% endif %}", "cases": [
|
||||
{"content": "hello", "engine_token_ids": [117, 115, 101, 114, 58, 59, 97, 115, 115, 105, 115, 116, 97, 110, 116, 58]},
|
||||
{"content": [{"type": "text", "text": "hello"}], "engine_token_ids": [117, 115, 101, 114, 58, 104, 101, 108, 108, 111, 59, 97, 115, 115, 105, 115, 116, 97, 110, 116, 58]}
|
||||
]}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{"model_id": "deepseek-ai/DeepSeek-V4-Flash", "cases": [
|
||||
{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [0, 128803, 63006, 19346, 295, 834, 10175, 16, 128804, 128822]},
|
||||
{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [0, 3476, 477, 259, 10935, 16, 128803, 3085, 344, 223, 20, 13, 20, 33, 128804, 128822]},
|
||||
{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [0, 3476, 477, 259, 10935, 16, 128803, 23166, 128804, 128822, 19923, 3, 1730, 588, 342, 1694, 33, 1, 128803, 52, 3319, 90902, 294, 270, 2831, 28, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 49206, 303, 16574, 3374, 320, 12808, 579, 75183, 16, 223, 128804, 128822]},
|
||||
{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [0, 128803, 50012, 943, 436, 16, 128804, 128822]},
|
||||
{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [0, 128803, 50012, 943, 436, 16, 128804, 128821]}
|
||||
]}
|
||||
@@ -0,0 +1,7 @@
|
||||
{"model_id": "zai-org/GLM-5.2-FP8", "cases": [
|
||||
{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154827, 45494, 15576, 304, 825, 11646, 13, 154828, 154841]},
|
||||
{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154826, 2610, 525, 50205, 13, 154827, 3838, 374, 220, 17, 10, 17, 30, 154828, 154841]},
|
||||
{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154826, 2610, 525, 50205, 13, 154827, 13041, 154828, 154841, 154842, 9703, 0, 2585, 646, 358, 1492, 30, 154827, 84140, 1242, 963, 315, 279, 3119, 25, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 98744, 99524, 3837, 102562, 99080, 1773, 12203, 582, 29517, 13, 220, 154828, 154841]},
|
||||
{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [154822, 154824, 154827, 38479, 911, 432, 13, 154828, 154841, 154842]},
|
||||
{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [154822, 154824, 154826, 25062, 287, 29905, 371, 25, 7487, 154827, 38479, 911, 432, 13, 154828, 154841]}
|
||||
]}
|
||||
@@ -0,0 +1,7 @@
|
||||
{"model_id": "openai/gpt-oss-20b", "cases": [
|
||||
{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 62316, 5911, 306, 1001, 21872, 13, 200007, 200006, 173781]},
|
||||
{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 77944, 200008, 2, 68406, 279, 3575, 553, 28410, 364, 200007, 200006, 1428, 200008, 4827, 382, 220, 17, 10, 17, 30, 200007, 200006, 173781]},
|
||||
{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 77944, 200008, 2, 68406, 279, 3575, 553, 28410, 364, 200007, 200006, 1428, 200008, 12194, 200007, 200006, 173781, 200005, 17196, 200008, 13225, 0, 3253, 665, 357, 1652, 30, 200007, 200006, 1428, 200008, 198128, 328, 290, 3496, 25, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 21871, 82066, 979, 18895, 12389, 20009, 788, 19371, 581, 63166, 13, 220, 200007, 200006, 173781]},
|
||||
{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 42421, 1078, 480, 13, 200007, 200006, 173781]},
|
||||
{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [200006, 17360, 200008, 3575, 553, 17554, 162016, 11, 261, 4410, 6439, 2359, 22203, 656, 7788, 17527, 558, 87447, 100594, 25, 220, 1323, 19, 12, 3218, 198, 6576, 3521, 25, 220, 1323, 21, 12, 3114, 12, 994, 279, 30377, 289, 25, 14093, 279, 2, 13888, 18403, 25, 8450, 11, 49159, 11, 1721, 13, 21030, 2804, 413, 7360, 395, 1753, 3176, 13, 200007, 200006, 1428, 200008, 42421, 1078, 480, 13, 200007, 200006, 173781]}
|
||||
]}
|
||||
@@ -0,0 +1,7 @@
|
||||
{"model_id": "MiniMaxAI/MiniMax-M3", "cases": [
|
||||
{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 66938, 13182, 296, 841, 14997, 46, 200020, 10, 200019, 1361, 10]},
|
||||
{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 6000, 101, 46, 200020, 10, 200019, 3995, 10, 3376, 355, 32, 50, 43, 50, 63, 200020, 10, 200019, 1361, 10]},
|
||||
{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 6000, 101, 46, 200020, 10, 200019, 3995, 10, 22700, 200020, 10, 200019, 1361, 10, 200060, 19739, 33, 2329, 566, 343, 1576, 63, 200020, 10, 200019, 3995, 10, 96530, 300, 275, 2748, 58, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 86080, 3564, 321, 31097, 5530, 350, 12283, 563, 76602, 46, 32, 200020, 10, 200019, 1361, 10]},
|
||||
{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 38460, 894, 412, 46, 200020, 10, 200019, 1361, 10]},
|
||||
{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [200034, 200019, 28463, 10, 11393, 2428, 4423, 355, 35353, 12973, 5145, 51, 44, 6415, 531, 35353, 12973, 46, 31058, 70273, 58, 8031, 32, 1421, 54, 46, 106114, 296, 4364, 32, 1421, 50, 44, 35353, 12973, 355, 258, 4746, 14409, 16001, 2428, 3245, 13323, 301, 47613, 275, 148531, 300, 14409, 6853, 19960, 73, 634, 60, 85255, 96871, 10353, 1100, 2985, 581, 258, 6995, 23362, 389, 6369, 390, 301, 3682, 3135, 531, 3135, 1865, 34573, 46, 3437, 6995, 355, 17239, 44, 18220, 641, 29751, 296, 32, 200059, 200060, 20211, 1865, 641, 4108, 46, 3437, 6995, 355, 22147, 44, 4236, 641, 4108, 6467, 1619, 275, 32, 200060, 24255, 46, 3437, 6995, 355, 37760, 44, 10941, 375, 641, 1813, 3784, 301, 1817, 360, 275, 2516, 2906, 320, 14455, 6995, 6972, 58, 37760, 46, 1781, 457, 21280, 301, 1817, 360, 4794, 5663, 23844, 44, 6775, 31428, 29751, 44, 436, 994, 32342, 1695, 164445, 3168, 320, 1579, 85255, 96871, 10353, 62, 200020, 10, 200019, 53556, 10, 2985, 457, 258, 12473, 23413, 46, 200020, 10, 200019, 3995, 10, 38460, 894, 412, 46, 200020, 10, 200019, 1361, 10]}
|
||||
]}
|
||||
@@ -0,0 +1,7 @@
|
||||
{"model_id": "Qwen/Qwen3-8B", "cases": [
|
||||
{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [151644, 872, 198, 45764, 15588, 304, 825, 11652, 13, 151645, 198, 151644, 77091, 198]},
|
||||
{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [151644, 8948, 198, 2610, 525, 50537, 13, 151645, 198, 151644, 872, 198, 3838, 374, 220, 17, 10, 17, 30, 151645, 198, 151644, 77091, 198]},
|
||||
{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [151644, 8948, 198, 2610, 525, 50537, 13, 151645, 198, 151644, 872, 198, 13048, 151645, 198, 151644, 77091, 198, 9707, 0, 2585, 646, 358, 1492, 30, 151645, 198, 151644, 872, 198, 84836, 1242, 963, 315, 279, 3119, 25, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 50331, 99724, 3837, 104412, 20074, 1773, 12209, 582, 29629, 13, 220, 151645, 198, 151644, 77091, 198]},
|
||||
{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [151644, 872, 198, 38687, 911, 432, 13, 151645, 198, 151644, 77091, 198, 151667, 271, 151668, 271]},
|
||||
{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [151644, 872, 198, 38687, 911, 432, 13, 151645, 198, 151644, 77091, 198]}
|
||||
]}
|
||||
@@ -0,0 +1,7 @@
|
||||
{"model_id": "Qwen/Qwen3.5-27B", "cases": [
|
||||
{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [248045, 846, 198, 44240, 15131, 303, 799, 11316, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
|
||||
{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [248045, 8678, 198, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 3710, 369, 220, 17, 10, 17, 30, 248046, 198, 248045, 74455, 198, 248068, 198]},
|
||||
{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [248045, 8678, 198, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 12675, 248046, 198, 248045, 74455, 198, 9419, 0, 2500, 628, 353, 1438, 30, 248046, 198, 248045, 846, 198, 81911, 168699, 314, 279, 3019, 25, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
|
||||
{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271]},
|
||||
{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}
|
||||
]}
|
||||
@@ -0,0 +1,7 @@
|
||||
{"model_id": "Qwen/Qwen3.8-27B", "cases": [
|
||||
{"shape": "user_only", "request": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 44240, 15131, 303, 799, 11316, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
|
||||
{"shape": "system_user", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "What is 2+2?"}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 271, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 3710, 369, 220, 17, 10, 17, 30, 248046, 198, 248045, 74455, 198, 248068, 198]},
|
||||
{"shape": "multi_turn", "request": {"messages": [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello! How can I help?"}, {"role": "user", "content": "Résumé of the plan: 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. 第一步，收集数据。Then we iterate. "}]}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 271, 2523, 513, 48834, 13, 248046, 198, 248045, 846, 198, 12675, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271, 9419, 0, 2500, 628, 353, 1438, 30, 248046, 198, 248045, 846, 198, 81911, 168699, 314, 279, 3019, 25, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 220, 112278, 3709, 100630, 96902, 1710, 11861, 567, 28662, 13, 248046, 198, 248045, 74455, 198, 248068, 198]},
|
||||
{"shape": "thinking_off", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": false, "thinking": false}}, "expected_token_ids": [248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 271, 248069, 271]},
|
||||
{"shape": "thinking_on", "request": {"messages": [{"role": "user", "content": "Think about it."}], "chat_template_kwargs": {"enable_thinking": true, "thinking": true}}, "expected_token_ids": [248045, 8678, 198, 24342, 286, 4879, 369, 716, 310, 830, 11553, 13, 5044, 1683, 15060, 1472, 279, 3274, 11, 9307, 1328, 30800, 11, 2814, 47675, 25605, 11, 321, 60445, 55404, 11, 27224, 11, 321, 30246, 303, 279, 1534, 4087, 13, 248046, 198, 248045, 846, 198, 37405, 883, 424, 13, 248046, 198, 248045, 74455, 198, 248068, 198]}
|
||||
]}
|
||||
@@ -168,3 +168,63 @@ async fn multimodal_request_omits_input_ids() {
|
||||
"multimodal requests must not forward input_ids; got {body}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Caller-supplied `input_ids` are never re-rendered or replaced: a flat u32
|
||||
/// array (empty included) drives routing, anything else yields no routing
|
||||
/// tokens, and the body reaches the engine byte-for-byte for validation.
|
||||
#[tokio::test]
|
||||
async fn caller_input_ids_are_used_for_routing_and_preserved() {
|
||||
let mock = MockWorker::start(vec![]).await;
|
||||
let ctx = build_ctx(mock.url.clone());
|
||||
for (ids, expected) in [
|
||||
(json!([7, 8]), Some(vec![7, 8])),
|
||||
(json!([]), Some(vec![])),
|
||||
(json!([7, -1]), None),
|
||||
(json!("bad"), None),
|
||||
] {
|
||||
let request = json!({
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"input_ids": ids,
|
||||
});
|
||||
let tokens = sgl_router::policies::request_tokens_for(
|
||||
&ctx.tokenizers,
|
||||
&ModelId(MODEL.into()),
|
||||
&request,
|
||||
);
|
||||
assert!(!tokens.as_ref().is_some_and(|t| t.rendered_from_chat));
|
||||
assert_eq!(tokens.map(|t| t.ids), expected, "input_ids: {ids}");
|
||||
assert_eq!(
|
||||
send(Arc::clone(&ctx), request.clone()).await,
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(captured(&mock), request, "body must be forwarded untouched");
|
||||
}
|
||||
// Bypasses are not rendering failures.
|
||||
assert!(!ctx
|
||||
.metrics
|
||||
.render()
|
||||
.contains("sgl_router_ingress_tokenize_errors_total{"));
|
||||
}
|
||||
|
||||
/// `input_ids: null` is the same as absent: the router renders and forwards.
|
||||
#[tokio::test]
|
||||
async fn null_input_ids_keep_normal_rendering() {
|
||||
let mock = MockWorker::start(vec![]).await;
|
||||
let ctx = build_ctx(mock.url.clone());
|
||||
let status = send(
|
||||
ctx,
|
||||
json!({
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "hello there friend"}],
|
||||
"input_ids": null,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
let body = captured(&mock);
|
||||
assert!(
|
||||
body["input_ids"].as_array().is_some_and(|a| !a.is_empty()),
|
||||
"null input_ids must not suppress rendering; got {body}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Regenerate the array-only template regression using SGLang's content processor.
|
||||
|
||||
Run: PYTHONPATH=../../python python tests/scripts/generate_array_content_fixture.py
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
|
||||
from sglang.srt.parser.jinja_template_utils import (
|
||||
detect_jinja_template_content_format,
|
||||
process_content_for_template_format,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] / "fixtures"
|
||||
TEMPLATE = (
|
||||
"{% for message in messages %}{{ message.role }}:"
|
||||
"{% for part in message.content %}"
|
||||
"{% if part.type == 'text' %}{{ part.text }}"
|
||||
"{% elif part.type == 'image' %}<image>{% endif %}"
|
||||
"{% endfor %};{% endfor %}"
|
||||
"{% if add_generation_prompt %}assistant:{% endif %}"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
tokenizer = PreTrainedTokenizerFast(
|
||||
tokenizer_file=str(ROOT / "tiny_tokenizer.json")
|
||||
)
|
||||
tokenizer.chat_template = TEMPLATE
|
||||
content_format = detect_jinja_template_content_format(TEMPLATE)
|
||||
assert content_format == "openai"
|
||||
cases = []
|
||||
for content in ["hello", [{"type": "text", "text": "hello"}]]:
|
||||
message = process_content_for_template_format(
|
||||
{"role": "user", "content": content}, content_format, [], [], [], []
|
||||
)
|
||||
ids = tokenizer.apply_chat_template(
|
||||
[message], return_dict=False, add_generation_prompt=True
|
||||
)
|
||||
cases.append(json.dumps({"content": content, "engine_token_ids": ids}))
|
||||
cases_json = ",\n".join(cases)
|
||||
(ROOT / "array_content_rendering.json").write_text(
|
||||
f'{{"chat_template": {json.dumps(TEMPLATE)}, "cases": [\n{cases_json}\n]}}\n'
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Generate reference prompt IDs with SGLang helpers and cached model tokenizers.
|
||||
|
||||
Run: python tests/scripts/generate_chat_render_parity.py
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from transformers.utils.hub import cached_file
|
||||
|
||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
||||
from sglang.srt.entrypoints.openai.chat_encoding import (
|
||||
resolve_dsv4_reasoning_effort_profile,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||
from sglang.srt.entrypoints.openai.serving_chat import (
|
||||
ThinkingMode,
|
||||
normalize_assistant_tool_call_arguments,
|
||||
normalize_tool_content,
|
||||
)
|
||||
from sglang.srt.parser.jinja_template_utils import (
|
||||
detect_jinja_template_content_format,
|
||||
process_content_for_template_format,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1] / "fixtures" / "chat_render_parity"
|
||||
|
||||
MODELS = {
|
||||
"qwen3-8b": "Qwen/Qwen3-8B",
|
||||
"qwen3.5-27b": "Qwen/Qwen3.5-27B",
|
||||
"qwen3.8-27b": "Qwen/Qwen3.8-27B",
|
||||
"gpt-oss-20b": "openai/gpt-oss-20b",
|
||||
"glm-5.2": "zai-org/GLM-5.2-FP8",
|
||||
"minimax-m3": "MiniMaxAI/MiniMax-M3",
|
||||
"deepseek-v4-flash": "deepseek-ai/DeepSeek-V4-Flash",
|
||||
}
|
||||
|
||||
LONG = "Résumé of the plan: " + "第一步，收集数据。Then we iterate. " * 8
|
||||
|
||||
SHAPES = {
|
||||
"user_only": {"messages": [{"role": "user", "content": "Say hi in one sentence."}]},
|
||||
"system_user": {
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
]
|
||||
},
|
||||
"multi_turn": {
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{"role": "user", "content": "Hi"},
|
||||
{"role": "assistant", "content": "Hello! How can I help?"},
|
||||
{"role": "user", "content": LONG},
|
||||
]
|
||||
},
|
||||
"thinking_off": {
|
||||
"messages": [{"role": "user", "content": "Think about it."}],
|
||||
"chat_template_kwargs": {"enable_thinking": False, "thinking": False},
|
||||
},
|
||||
"thinking_on": {
|
||||
"messages": [{"role": "user", "content": "Think about it."}],
|
||||
"chat_template_kwargs": {"enable_thinking": True, "thinking": True},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def snapshot_dir(model_id):
|
||||
return pathlib.Path(
|
||||
cached_file(model_id, "config.json", local_files_only=True)
|
||||
).parent
|
||||
|
||||
|
||||
def engine_messages(request, content_format):
|
||||
messages = [m.model_dump() for m in request.messages]
|
||||
for message in messages:
|
||||
normalize_assistant_tool_call_arguments(message)
|
||||
out = []
|
||||
for msg in copy.deepcopy(messages):
|
||||
if msg.get("content") is None:
|
||||
msg["content"] = ""
|
||||
processed = process_content_for_template_format(
|
||||
msg, content_format, [], [], [], []
|
||||
)
|
||||
processed["content"] = normalize_tool_content(
|
||||
processed["role"], processed.get("content")
|
||||
)
|
||||
out.append(processed)
|
||||
return out
|
||||
|
||||
|
||||
def engine_prompt_ids(model_id, tok, request):
|
||||
"""Mirror `_apply_jinja_template` for a text-only request without tools."""
|
||||
snapshot = snapshot_dir(model_id)
|
||||
model_type = json.load(open(snapshot / "config.json")).get("model_type")
|
||||
if model_type == "deepseek_v4":
|
||||
messages = engine_messages(request, "string")
|
||||
if messages[0]["role"] != "system":
|
||||
messages.insert(0, {"role": "system", "content": ""})
|
||||
thinking = (request.chat_template_kwargs or {}).get("thinking", False)
|
||||
text = encoding_dsv4.encode_messages(
|
||||
messages,
|
||||
thinking_mode=ThinkingMode.THINKING if thinking else ThinkingMode.CHAT,
|
||||
reasoning_effort=None,
|
||||
reasoning_effort_profile=resolve_dsv4_reasoning_effort_profile(
|
||||
model_path=str(snapshot)
|
||||
),
|
||||
)
|
||||
return tok.encode(text)
|
||||
|
||||
template = tok.chat_template
|
||||
if not isinstance(template, str):
|
||||
raise RuntimeError(f"{model_id}: named template dict is not supported here")
|
||||
messages = engine_messages(request, detect_jinja_template_content_format(template))
|
||||
extra = {}
|
||||
if request.reasoning_effort is not None:
|
||||
extra["reasoning_effort"] = request.reasoning_effort
|
||||
if request.chat_template_kwargs:
|
||||
extra.update(request.chat_template_kwargs)
|
||||
rendered = tok.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
tools=None,
|
||||
return_dict=False,
|
||||
**extra,
|
||||
)
|
||||
encode_kwargs = {"add_special_tokens": False} if len(tok.encode("")) > 0 else {}
|
||||
return tok.encode(rendered, **encode_kwargs)
|
||||
|
||||
|
||||
def main():
|
||||
ROOT.mkdir(parents=True, exist_ok=True)
|
||||
for slug, model_id in MODELS.items():
|
||||
try:
|
||||
snapshot = snapshot_dir(model_id)
|
||||
except Exception as e:
|
||||
print(f"skip {model_id}: {e}", file=sys.stderr)
|
||||
continue
|
||||
tok = get_tokenizer(str(snapshot))
|
||||
cases = []
|
||||
for shape, body in SHAPES.items():
|
||||
request = ChatCompletionRequest(model=model_id, **copy.deepcopy(body))
|
||||
ids = engine_prompt_ids(model_id, tok, request)
|
||||
cases.append({"shape": shape, "request": body, "expected_token_ids": ids})
|
||||
out = ROOT / f"{slug}.json"
|
||||
lines = [json.dumps(case, ensure_ascii=False) for case in cases]
|
||||
out.write_text(
|
||||
'{"model_id": %s, "cases": [\n%s\n]}\n'
|
||||
% (json.dumps(model_id), ",\n".join(lines))
|
||||
)
|
||||
print(f"wrote {out} ({len(cases)} cases)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user