[sgl-router] Add SGLang's /v1/rerank (#42205)

Co-authored-by: Kan Wu <kan.wu@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-10-03 05:33:40 +08:00
committed by GitHub
co-authored by Kan Wu Claude Opus 5.5
parent 8c9030ce3f
commit ebf161cea4
9 changed files with 225 additions and 41 deletions
+13 -5
View File
@@ -4,11 +4,11 @@ Slim, KV-aware, OpenAI-compatible router for SGLang workers.
Serves a single model and routes across its workers. Exposes
`/v1/tokenize`, `/v1/detokenize`, `/v1/models`, [`/v1/embeddings`](#embeddings),
`/v1/chat/completions` and SGLang's native [`/generate`](#native-generate)
(buffered and SSE), plus `/healthz` / `/readyz` and `/metrics`. Worker
pools come from either a static URL list or Kubernetes EndpointSlice
discovery. Both edges speak cleartext HTTP/2 where the peer does — see
[HTTP/2](#http2).
[`/v1/rerank`](#rerank), `/v1/chat/completions` and SGLang's native
[`/generate`](#native-generate) (buffered and SSE), plus `/healthz` / `/readyz`
and `/metrics`. Worker pools come from either a static URL list or Kubernetes
EndpointSlice discovery. Both edges speak cleartext HTTP/2 where the peer does —
see [HTTP/2](#http2).
## Building
@@ -406,6 +406,14 @@ render. A list is a batch for one worker, routed on load. A single prompt with n
`rid` gets a minted one. Under `--dp-aware` the engine picks the rank, since its
embeddings endpoint reads none.
## Rerank
`/v1/rerank` has the engine's interface: the same `V1RerankReqInput` body and
response, sent as `POST` or `PUT` to the model this router serves. The body is
forwarded as sent, since the engine renders and tokenizes each query-document
pair. Routing is on load, with each pair's size estimated from its text. As for
embeddings, a PD fleet answers 400 and `--dp-aware` pins no rank.
## DeepSeek V4
Native V4 rendering follows SGLang's serving path (`serving_chat.py`), not
@@ -218,6 +218,13 @@ pub fn build_router(ctx: Arc<AppContext>) -> Router {
.layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES))
.layer(middleware::from_fn(log_413)),
)
.route(
"/v1/rerank",
post(crate::server::routes::chat::rerank)
.put(crate::server::routes::chat::rerank)
.layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES))
.layer(middleware::from_fn(log_413)),
)
.route(
"/flush_cache",
post(crate::server::routes::cache::flush_cache),
@@ -86,18 +86,38 @@ pub async fn embeddings(
let start = Instant::now();
let (model, value) = parse_embedding_request(&body)?;
let routing = ModelRouting::lookup(&ctx, &model)?;
// Prefill and decode engines serve generation; embeddings need plain workers.
let registered = ctx.registry.workers_for(&model);
if registered.iter().any(|w| w.mode() != WorkerMode::Plain) {
return Err(ApiError::BadRequest(
"embeddings are not served by prefill-decode workers".into(),
));
}
require_plain_workers(&ctx, &model, "/v1/embeddings")?;
let request = PreparedRequest::embeddings(&ctx, model, body, value)?;
let workers = routing.select_workers(&ctx, &request, &headers).await?;
forward_request(&ctx, request, workers, headers, start).await
}
/// SGLang's `/v1/rerank`, forwarded as sent to the model this router serves.
pub async fn rerank(
State(ctx): State<Arc<AppContext>>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response<Body>, ApiError> {
let start = Instant::now();
let model = ModelId(ctx.config.model.id.clone());
let routing = ModelRouting::lookup(&ctx, &model)?;
require_plain_workers(&ctx, &model, "/v1/rerank")?;
let request = PreparedRequest::rerank(model, body)?;
let workers = routing.select_workers(&ctx, &request, &headers).await?;
forward_request(&ctx, request, workers, headers, start).await
}
/// Prefill and decode engines serve generation only.
fn require_plain_workers(ctx: &AppContext, model: &ModelId, path: &str) -> Result<(), ApiError> {
let registered = ctx.registry.workers_for(model);
if registered.iter().any(|w| w.mode() != WorkerMode::Plain) {
return Err(ApiError::BadRequest(format!(
"{path} is not served by prefill-decode workers"
)));
}
Ok(())
}
/// A model's routing state, resolved before the request is prepared.
enum ModelRouting<'a> {
Legacy(Arc<dyn Policy>),
@@ -2,7 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
//! Request validation, optional tokenization, and outgoing body preparation for
//! chat completions, the native `/generate` endpoint, and embeddings.
//! chat completions, the native `/generate` endpoint, embeddings and rerank.
use crate::config::{ConflictPolicy, ParamSpec, SamplingField, SamplingOverrides};
use crate::discovery::ModelId;
@@ -22,6 +22,7 @@ const BYTES_PER_TOKEN_ESTIMATE: usize = 4;
const CHAT_PATH: &str = "/v1/chat/completions";
const GENERATE_PATH: &str = "/generate";
const EMBEDDINGS_PATH: &str = "/v1/embeddings";
const RERANK_PATH: &str = "/v1/rerank";
/// Validated routing inputs and the original body, ready for worker selection.
///
@@ -44,7 +45,7 @@ pub(super) struct PreparedRequest {
pub(super) expected_peak_sequence_tokens: Option<u64>,
caller_set_rid: bool,
pub(super) fans_out: bool,
/// `None` for `/generate` and embeddings, whose tokens replace text during preparation.
/// `None` for `/generate`, embeddings and rerank, which prepare their own body.
forwarding_scope: Option<ForwardingScope>,
parsed_body: Option<Value>,
sampling_defaults: Vec<(SamplingField, Number)>,
@@ -194,17 +195,13 @@ impl PreparedRequest {
.into();
}
let prompts = text_ids.or_else(|| caller_token_rows(&value["input"]));
// Context limits apply to each prompt even when the engine must tokenize text;
// multimodal items count only their text, as image tokens are the engine's to count.
let estimate = |item: &Value| {
estimate_prefill_tokens(item.as_str().or(item["text"].as_str()).map_or(0, str::len))
};
let lengths = match (&prompts, &value["input"]) {
// Context limits apply to each prompt even when the engine must tokenize text.
let estimate = |item: &Value| estimate_prefill_tokens(text_len(item));
let lengths: Vec<_> = match (&prompts, &value["input"]) {
(Some(prompts), _) => prompts.iter().map(|ids| ids.len().max(1)).collect(),
(None, Value::Array(items)) => items.iter().map(estimate).collect(),
(None, input) => vec![estimate(input)],
};
let sequence_tokens = lengths.iter().copied().max().unwrap_or(1);
let tokens = prompts
.filter(|_| !batch)
.and_then(|prompts| prompts.into_iter().next())
@@ -213,7 +210,38 @@ impl PreparedRequest {
rendered_from_chat: false,
});
Ok(Self {
path: EMBEDDINGS_PATH,
tokens,
caller_set_rid: !value["rid"].is_null(),
fans_out: batch,
..Self::no_output(EMBEDDINGS_PATH, model, body, &lengths)
})
}
/// SGLang's `V1RerankReqInput`, forwarded as sent: the engine renders and
/// tokenizes each query-document pair as its own prompt.
pub(super) fn rerank(model: ModelId, body: Bytes) -> Result<Self, ApiError> {
let value = Value::Object(serde_json::from_slice(&body).map_err(|_| invalid_request())?);
// Qwen rerankers include the instruction in every query-document prompt.
let shared = text_len(&value["instruct"]) + text_len(&value["query"]);
let lengths: Vec<_> = value["documents"]
.as_array()
.into_iter()
.flatten()
.map(|document| estimate_prefill_tokens(shared + text_len(document)))
.collect();
Ok(Self {
// Each document is its own engine request, and the engine reads no `rid`.
fans_out: true,
..Self::no_output(RERANK_PATH, model, body, &lengths)
})
}
/// A request of `lengths` prompts that generates nothing. No PD serves it,
/// so it keeps no parsed body for bootstrap fields.
fn no_output(path: &'static str, model: ModelId, body: Bytes, lengths: &[usize]) -> Self {
let sequence_tokens = lengths.iter().copied().max().unwrap_or(1);
Self {
path,
model,
streaming: false,
output_tokens: Some(0),
@@ -221,19 +249,18 @@ impl PreparedRequest {
sequence_token_count: sequence_tokens,
expected_peak_sequence_tokens: Some(sequence_tokens as u64),
body,
tokens,
caller_set_rid: !value["rid"].is_null(),
fans_out: batch,
tokens: None,
caller_set_rid: false,
fans_out: false,
forwarding_scope: None,
// Only PD bootstrap needs the parsed body, and embeddings have no PD.
parsed_body: None,
sampling_defaults: Vec::new(),
})
}
}
/// Whether the engine endpoint honors a routed DP rank; `/v1/embeddings` takes none.
/// Whether the engine endpoint honors a routed DP rank; embeddings and rerank take none.
pub(super) fn accepts_dp_rank(&self) -> bool {
self.path != EMBEDDINGS_PATH
!matches!(self.path, EMBEDDINGS_PATH | RERANK_PATH)
}
pub(super) fn engine_rid(&self) -> Option<String> {
@@ -415,6 +442,18 @@ fn tokenize_input_text(ctx: &AppContext, model: &ModelId, input: &Value) -> Opti
.collect()
}
/// Prompt text bytes of a string, a multimodal item or a list of content parts;
/// image tokens are the engine's to count.
fn text_len(content: &Value) -> usize {
match content {
Value::Array(parts) => parts.iter().map(text_len).sum(),
content => content
.as_str()
.or(content["text"].as_str())
.map_or(0, str::len),
}
}
/// Caller token IDs, one flat list or a row per prompt; `None` if malformed,
/// which the engine reports.
fn caller_token_rows(input: &Value) -> Option<Vec<Vec<u32>>> {
@@ -1188,6 +1227,21 @@ mod tests {
assert_eq!((input, r.tokens.map(|t| t.ids)), (json!("hi"), ids));
}
#[test]
fn rerank_estimates_each_query_document_pair() {
let image = json!({"type": "image_url", "image_url": {"url": "x".repeat(4096)}});
let documents = json!(["abcd", [{"type": "text", "text": "abcdefgh"}, image]]);
for (instruct, total, longest) in [(json!(null), 5, 3), (json!("abcdefgh"), 9, 5)] {
let body = json!({"query": "abcd", "documents": documents, "instruct": instruct});
let r = PreparedRequest::rerank(ModelId("stub-model".into()), body.to_string().into());
let r = r.unwrap();
assert_eq!(
(r.input_token_count, r.sequence_token_count),
(total, longest)
);
}
}
#[test]
fn generate_counts_batches_per_prompt() {
// (fans out, (input total, longest prompt, output total, longest peak))
@@ -991,6 +991,39 @@ async fn full_decode_group_falls_back_to_another_version_group() {
assert_eq!(dispatched, [false, true, false, true]);
}
#[tokio::test]
async fn rerank_instructions_count_toward_bucket_context_limits() {
let short_worker = MockWorker::start(vec![]).await;
let long_worker = MockWorker::start(vec![]).await;
let policy = Arc::new(FirstPolicy::default());
let mut short = Bucket::new("short", BucketGroups::Plain(group("short", policy.clone())));
short.max_context_tokens = Some(1024);
let long = Bucket::new("long", BucketGroups::Plain(group("long", policy)));
let app = build_router(context(
&[
("short", Stage::Plain, &short_worker),
("long", Stage::Plain, &long_worker),
],
vec![long, short],
));
// Qwen rerankers include the instruction in every query-document prompt.
for (instruct, worker) in [
("Rank relevant documents.".to_owned(), &short_worker),
("instruction ".repeat(8192), &long_worker),
] {
let body = serde_json::json!({"query": "hi", "documents": ["yo"], "instruct": instruct});
let req = Request::post("/v1/rerank")
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap();
let response = app.clone().oneshot(req).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
response.into_body().collect().await.unwrap();
assert_eq!(worker.captured_json().await, body);
}
}
#[tokio::test]
async fn embeddings_fallback_batches_use_per_prompt_context_limits() {
let worker = MockWorker::start(vec![]).await;
@@ -75,6 +75,7 @@ impl MockWorker {
.route("/v1/chat/completions", post(chat))
.route("/generate", post(generate))
.route("/v1/embeddings", post(embeddings))
.route("/v1/rerank", post(rerank))
.route("/server_info", get(serve_tiny_server_info))
.route("/abort_request", abort_request_route(abort_log.clone()))
.with_state(state);
@@ -519,3 +520,14 @@ async fn embeddings(
let data = [serde_json::json!({"object": "embedding", "embedding": [0.5], "index": 0})];
Json(serde_json::json!({"object": "list", "data": data, "model": "tiny"})).into_response()
}
/// A rerank response, a list of scored documents.
#[allow(dead_code)] // Used by `MockWorker::start`, only some test files need it.
async fn rerank(
State(s): State<MockWorkerState>,
headers: HeaderMap,
body: Bytes,
) -> Response<Body> {
capture_request(&s, &headers, &body);
Json(serde_json::json!([{"score": 0.5, "index": 0}])).into_response()
}
@@ -242,23 +242,27 @@ async fn pd_fan_out_leaves_the_prefill_rank_to_the_engine() {
}
}
/// The engine's `/v1/embeddings` reads no rank, so the router pins none.
/// The engine's `/v1/embeddings` and `/v1/rerank` read no rank, so the router pins none.
#[tokio::test]
async fn embeddings_leave_the_rank_to_the_engine() {
async fn embeddings_and_rerank_leave_the_rank_to_the_engine() {
let worker = MockWorker::start(vec![]).await;
let app = router(
sticky_config(),
&[(&worker, WorkerMode::Plain, 4)],
Default::default(),
);
let request = Request::post("/v1/embeddings")
.header("content-type", "application/json")
.header(KEY, "conv-a")
.body(Body::from(
json!({"model": MODEL, "input": "hi"}).to_string(),
))
.unwrap();
assert!(app.oneshot(request).await.unwrap().status().is_success());
assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None);
assert!(worker.captured_json().await.get("routed_dp_rank").is_none());
for (path, body) in [
("/v1/embeddings", json!({"model": MODEL, "input": "hi"})),
("/v1/rerank", json!({"query": "hi", "documents": ["yo"]})),
] {
let request = Request::post(path)
.header("content-type", "application/json")
.header(KEY, "conv-a")
.body(Body::from(body.to_string()))
.unwrap();
let response = app.clone().oneshot(request).await.unwrap();
assert!(response.status().is_success());
assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None);
assert!(worker.captured_json().await.get("routed_dp_rank").is_none());
}
}
@@ -27,6 +27,7 @@ mod pd_pool_isolation;
mod pd_protocol_binding;
mod radix_tree_routing;
mod reorg_launch;
mod rerank_routing;
mod roundrobin_input_ids;
mod sampling_overrides;
mod shared_prefill_admission;
@@ -0,0 +1,45 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
//! SGLang's `/v1/rerank`: the engine's request and response, forwarded as sent.
use axum::body::Body;
use axum::http::{Request, StatusCode};
use serde_json::json;
use sgl_router::discovery::WorkerMode;
use sgl_router::state::kv_events::HashTree;
use tower::ServiceExt;
use crate::common::cache_aware_fixture::radix_router;
use crate::common::mock_worker::MockWorker;
use crate::common::streaming::collect_body;
#[tokio::test]
async fn rerank_forwards_put_and_rejects_prefill_decode_workers() {
let (engine, prefill, decode) = (
MockWorker::start(vec![]).await,
MockWorker::start(vec![]).await,
MockWorker::start(vec![]).await,
);
let body = json!({"query": "hi", "documents": ["yo", [{"type": "text", "text": "x"}]]});
let send = |workers: &[(&MockWorker, WorkerMode)]| {
let request = Request::put("/v1/rerank")
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap();
radix_router(workers, HashTree::new()).oneshot(request)
};
let response = send(&[(&engine, WorkerMode::Plain)]).await.unwrap();
let response = collect_body(response.into_body()).await;
assert_eq!(response, json!([{"score": 0.5, "index": 0}]).to_string());
assert_eq!(engine.captured_json().await, body);
let pd = [
(&prefill, WorkerMode::Prefill),
(&decode, WorkerMode::Decode),
];
let response = send(&pd).await.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
assert!(prefill.captured.lock().unwrap().last_body.is_none());
}