[router] Improve SGLang chat render parity (#39133)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Shangming Cai
parent
7ccbf5fd04
commit
e4cbb28ea1
@@ -168,3 +168,63 @@ async fn multimodal_request_omits_input_ids() {
|
||||
"multimodal requests must not forward input_ids; got {body}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Caller-supplied `input_ids` are never re-rendered or replaced: a flat u32
|
||||
/// array (empty included) drives routing, anything else yields no routing
|
||||
/// tokens, and the body reaches the engine byte-for-byte for validation.
|
||||
#[tokio::test]
|
||||
async fn caller_input_ids_are_used_for_routing_and_preserved() {
|
||||
let mock = MockWorker::start(vec![]).await;
|
||||
let ctx = build_ctx(mock.url.clone());
|
||||
for (ids, expected) in [
|
||||
(json!([7, 8]), Some(vec![7, 8])),
|
||||
(json!([]), Some(vec![])),
|
||||
(json!([7, -1]), None),
|
||||
(json!("bad"), None),
|
||||
] {
|
||||
let request = json!({
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"input_ids": ids,
|
||||
});
|
||||
let tokens = sgl_router::policies::request_tokens_for(
|
||||
&ctx.tokenizers,
|
||||
&ModelId(MODEL.into()),
|
||||
&request,
|
||||
);
|
||||
assert!(!tokens.as_ref().is_some_and(|t| t.rendered_from_chat));
|
||||
assert_eq!(tokens.map(|t| t.ids), expected, "input_ids: {ids}");
|
||||
assert_eq!(
|
||||
send(Arc::clone(&ctx), request.clone()).await,
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(captured(&mock), request, "body must be forwarded untouched");
|
||||
}
|
||||
// Bypasses are not rendering failures.
|
||||
assert!(!ctx
|
||||
.metrics
|
||||
.render()
|
||||
.contains("sgl_router_ingress_tokenize_errors_total{"));
|
||||
}
|
||||
|
||||
/// `input_ids: null` is the same as absent: the router renders and forwards.
|
||||
#[tokio::test]
|
||||
async fn null_input_ids_keep_normal_rendering() {
|
||||
let mock = MockWorker::start(vec![]).await;
|
||||
let ctx = build_ctx(mock.url.clone());
|
||||
let status = send(
|
||||
ctx,
|
||||
json!({
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "hello there friend"}],
|
||||
"input_ids": null,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
let body = captured(&mock);
|
||||
assert!(
|
||||
body["input_ids"].as_array().is_some_and(|a| !a.is_empty()),
|
||||
"null input_ids must not suppress rendering; got {body}"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user