[router] Improve SGLang chat render parity (#39133)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Kan Wu
2026-09-17 23:53:40 +08:00
committed by GitHub
co-authored by Claude Fable 5.1 Shangming Cai
parent 7ccbf5fd04
commit e4cbb28ea1
16 changed files with 841 additions and 18 deletions
@@ -168,3 +168,63 @@ async fn multimodal_request_omits_input_ids() {
"multimodal requests must not forward input_ids; got {body}"
);
}
/// Caller-supplied `input_ids` are never re-rendered or replaced: a flat u32
/// array (empty included) drives routing, anything else yields no routing
/// tokens, and the body reaches the engine byte-for-byte for validation.
#[tokio::test]
async fn caller_input_ids_are_used_for_routing_and_preserved() {
let mock = MockWorker::start(vec![]).await;
let ctx = build_ctx(mock.url.clone());
for (ids, expected) in [
(json!([7, 8]), Some(vec![7, 8])),
(json!([]), Some(vec![])),
(json!([7, -1]), None),
(json!("bad"), None),
] {
let request = json!({
"model": MODEL,
"messages": [{"role": "user", "content": "hi"}],
"input_ids": ids,
});
let tokens = sgl_router::policies::request_tokens_for(
&ctx.tokenizers,
&ModelId(MODEL.into()),
&request,
);
assert!(!tokens.as_ref().is_some_and(|t| t.rendered_from_chat));
assert_eq!(tokens.map(|t| t.ids), expected, "input_ids: {ids}");
assert_eq!(
send(Arc::clone(&ctx), request.clone()).await,
StatusCode::OK
);
assert_eq!(captured(&mock), request, "body must be forwarded untouched");
}
// Bypasses are not rendering failures.
assert!(!ctx
.metrics
.render()
.contains("sgl_router_ingress_tokenize_errors_total{"));
}
/// `input_ids: null` is the same as absent: the router renders and forwards.
#[tokio::test]
async fn null_input_ids_keep_normal_rendering() {
let mock = MockWorker::start(vec![]).await;
let ctx = build_ctx(mock.url.clone());
let status = send(
ctx,
json!({
"model": MODEL,
"messages": [{"role": "user", "content": "hello there friend"}],
"input_ids": null,
}),
)
.await;
assert_eq!(status, StatusCode::OK);
let body = captured(&mock);
assert!(
body["input_ids"].as_array().is_some_and(|a| !a.is_empty()),
"null input_ids must not suppress rendering; got {body}"
);
}