[sgl-router] Add SGLang's /v1/rerank (#42205)
Co-authored-by: Kan Wu <kan.wu@radixark.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Kan Wu
Claude Opus 5.5
parent
8c9030ce3f
commit
ebf161cea4
@@ -4,11 +4,11 @@ Slim, KV-aware, OpenAI-compatible router for SGLang workers.
|
||||
|
||||
Serves a single model and routes across its workers. Exposes
|
||||
`/v1/tokenize`, `/v1/detokenize`, `/v1/models`, [`/v1/embeddings`](#embeddings),
|
||||
`/v1/chat/completions` and SGLang's native [`/generate`](#native-generate)
|
||||
(buffered and SSE), plus `/healthz` / `/readyz` and `/metrics`. Worker
|
||||
pools come from either a static URL list or Kubernetes EndpointSlice
|
||||
discovery. Both edges speak cleartext HTTP/2 where the peer does — see
|
||||
[HTTP/2](#http2).
|
||||
[`/v1/rerank`](#rerank), `/v1/chat/completions` and SGLang's native
|
||||
[`/generate`](#native-generate) (buffered and SSE), plus `/healthz` / `/readyz`
|
||||
and `/metrics`. Worker pools come from either a static URL list or Kubernetes
|
||||
EndpointSlice discovery. Both edges speak cleartext HTTP/2 where the peer does —
|
||||
see [HTTP/2](#http2).
|
||||
|
||||
## Building
|
||||
|
||||
@@ -406,6 +406,14 @@ render. A list is a batch for one worker, routed on load. A single prompt with n
|
||||
`rid` gets a minted one. Under `--dp-aware` the engine picks the rank, since its
|
||||
embeddings endpoint reads none.
|
||||
|
||||
## Rerank
|
||||
|
||||
`/v1/rerank` has the engine's interface: the same `V1RerankReqInput` body and
|
||||
response, sent as `POST` or `PUT` to the model this router serves. The body is
|
||||
forwarded as sent, since the engine renders and tokenizes each query-document
|
||||
pair. Routing is on load, with each pair's size estimated from its text. As for
|
||||
embeddings, a PD fleet answers 400 and `--dp-aware` pins no rank.
|
||||
|
||||
## DeepSeek V4
|
||||
|
||||
Native V4 rendering follows SGLang's serving path (`serving_chat.py`), not
|
||||
|
||||
@@ -218,6 +218,13 @@ pub fn build_router(ctx: Arc<AppContext>) -> Router {
|
||||
.layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES))
|
||||
.layer(middleware::from_fn(log_413)),
|
||||
)
|
||||
.route(
|
||||
"/v1/rerank",
|
||||
post(crate::server::routes::chat::rerank)
|
||||
.put(crate::server::routes::chat::rerank)
|
||||
.layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES))
|
||||
.layer(middleware::from_fn(log_413)),
|
||||
)
|
||||
.route(
|
||||
"/flush_cache",
|
||||
post(crate::server::routes::cache::flush_cache),
|
||||
|
||||
@@ -86,18 +86,38 @@ pub async fn embeddings(
|
||||
let start = Instant::now();
|
||||
let (model, value) = parse_embedding_request(&body)?;
|
||||
let routing = ModelRouting::lookup(&ctx, &model)?;
|
||||
// Prefill and decode engines serve generation; embeddings need plain workers.
|
||||
let registered = ctx.registry.workers_for(&model);
|
||||
if registered.iter().any(|w| w.mode() != WorkerMode::Plain) {
|
||||
return Err(ApiError::BadRequest(
|
||||
"embeddings are not served by prefill-decode workers".into(),
|
||||
));
|
||||
}
|
||||
require_plain_workers(&ctx, &model, "/v1/embeddings")?;
|
||||
let request = PreparedRequest::embeddings(&ctx, model, body, value)?;
|
||||
let workers = routing.select_workers(&ctx, &request, &headers).await?;
|
||||
forward_request(&ctx, request, workers, headers, start).await
|
||||
}
|
||||
|
||||
/// SGLang's `/v1/rerank`, forwarded as sent to the model this router serves.
|
||||
pub async fn rerank(
|
||||
State(ctx): State<Arc<AppContext>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Result<Response<Body>, ApiError> {
|
||||
let start = Instant::now();
|
||||
let model = ModelId(ctx.config.model.id.clone());
|
||||
let routing = ModelRouting::lookup(&ctx, &model)?;
|
||||
require_plain_workers(&ctx, &model, "/v1/rerank")?;
|
||||
let request = PreparedRequest::rerank(model, body)?;
|
||||
let workers = routing.select_workers(&ctx, &request, &headers).await?;
|
||||
forward_request(&ctx, request, workers, headers, start).await
|
||||
}
|
||||
|
||||
/// Prefill and decode engines serve generation only.
|
||||
fn require_plain_workers(ctx: &AppContext, model: &ModelId, path: &str) -> Result<(), ApiError> {
|
||||
let registered = ctx.registry.workers_for(model);
|
||||
if registered.iter().any(|w| w.mode() != WorkerMode::Plain) {
|
||||
return Err(ApiError::BadRequest(format!(
|
||||
"{path} is not served by prefill-decode workers"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A model's routing state, resolved before the request is prepared.
|
||||
enum ModelRouting<'a> {
|
||||
Legacy(Arc<dyn Policy>),
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Request validation, optional tokenization, and outgoing body preparation for
|
||||
//! chat completions, the native `/generate` endpoint, and embeddings.
|
||||
//! chat completions, the native `/generate` endpoint, embeddings and rerank.
|
||||
|
||||
use crate::config::{ConflictPolicy, ParamSpec, SamplingField, SamplingOverrides};
|
||||
use crate::discovery::ModelId;
|
||||
@@ -22,6 +22,7 @@ const BYTES_PER_TOKEN_ESTIMATE: usize = 4;
|
||||
const CHAT_PATH: &str = "/v1/chat/completions";
|
||||
const GENERATE_PATH: &str = "/generate";
|
||||
const EMBEDDINGS_PATH: &str = "/v1/embeddings";
|
||||
const RERANK_PATH: &str = "/v1/rerank";
|
||||
|
||||
/// Validated routing inputs and the original body, ready for worker selection.
|
||||
///
|
||||
@@ -44,7 +45,7 @@ pub(super) struct PreparedRequest {
|
||||
pub(super) expected_peak_sequence_tokens: Option<u64>,
|
||||
caller_set_rid: bool,
|
||||
pub(super) fans_out: bool,
|
||||
/// `None` for `/generate` and embeddings, whose tokens replace text during preparation.
|
||||
/// `None` for `/generate`, embeddings and rerank, which prepare their own body.
|
||||
forwarding_scope: Option<ForwardingScope>,
|
||||
parsed_body: Option<Value>,
|
||||
sampling_defaults: Vec<(SamplingField, Number)>,
|
||||
@@ -194,17 +195,13 @@ impl PreparedRequest {
|
||||
.into();
|
||||
}
|
||||
let prompts = text_ids.or_else(|| caller_token_rows(&value["input"]));
|
||||
// Context limits apply to each prompt even when the engine must tokenize text;
|
||||
// multimodal items count only their text, as image tokens are the engine's to count.
|
||||
let estimate = |item: &Value| {
|
||||
estimate_prefill_tokens(item.as_str().or(item["text"].as_str()).map_or(0, str::len))
|
||||
};
|
||||
let lengths = match (&prompts, &value["input"]) {
|
||||
// Context limits apply to each prompt even when the engine must tokenize text.
|
||||
let estimate = |item: &Value| estimate_prefill_tokens(text_len(item));
|
||||
let lengths: Vec<_> = match (&prompts, &value["input"]) {
|
||||
(Some(prompts), _) => prompts.iter().map(|ids| ids.len().max(1)).collect(),
|
||||
(None, Value::Array(items)) => items.iter().map(estimate).collect(),
|
||||
(None, input) => vec![estimate(input)],
|
||||
};
|
||||
let sequence_tokens = lengths.iter().copied().max().unwrap_or(1);
|
||||
let tokens = prompts
|
||||
.filter(|_| !batch)
|
||||
.and_then(|prompts| prompts.into_iter().next())
|
||||
@@ -213,7 +210,38 @@ impl PreparedRequest {
|
||||
rendered_from_chat: false,
|
||||
});
|
||||
Ok(Self {
|
||||
path: EMBEDDINGS_PATH,
|
||||
tokens,
|
||||
caller_set_rid: !value["rid"].is_null(),
|
||||
fans_out: batch,
|
||||
..Self::no_output(EMBEDDINGS_PATH, model, body, &lengths)
|
||||
})
|
||||
}
|
||||
|
||||
/// SGLang's `V1RerankReqInput`, forwarded as sent: the engine renders and
|
||||
/// tokenizes each query-document pair as its own prompt.
|
||||
pub(super) fn rerank(model: ModelId, body: Bytes) -> Result<Self, ApiError> {
|
||||
let value = Value::Object(serde_json::from_slice(&body).map_err(|_| invalid_request())?);
|
||||
// Qwen rerankers include the instruction in every query-document prompt.
|
||||
let shared = text_len(&value["instruct"]) + text_len(&value["query"]);
|
||||
let lengths: Vec<_> = value["documents"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|document| estimate_prefill_tokens(shared + text_len(document)))
|
||||
.collect();
|
||||
Ok(Self {
|
||||
// Each document is its own engine request, and the engine reads no `rid`.
|
||||
fans_out: true,
|
||||
..Self::no_output(RERANK_PATH, model, body, &lengths)
|
||||
})
|
||||
}
|
||||
|
||||
/// A request of `lengths` prompts that generates nothing. No PD serves it,
|
||||
/// so it keeps no parsed body for bootstrap fields.
|
||||
fn no_output(path: &'static str, model: ModelId, body: Bytes, lengths: &[usize]) -> Self {
|
||||
let sequence_tokens = lengths.iter().copied().max().unwrap_or(1);
|
||||
Self {
|
||||
path,
|
||||
model,
|
||||
streaming: false,
|
||||
output_tokens: Some(0),
|
||||
@@ -221,19 +249,18 @@ impl PreparedRequest {
|
||||
sequence_token_count: sequence_tokens,
|
||||
expected_peak_sequence_tokens: Some(sequence_tokens as u64),
|
||||
body,
|
||||
tokens,
|
||||
caller_set_rid: !value["rid"].is_null(),
|
||||
fans_out: batch,
|
||||
tokens: None,
|
||||
caller_set_rid: false,
|
||||
fans_out: false,
|
||||
forwarding_scope: None,
|
||||
// Only PD bootstrap needs the parsed body, and embeddings have no PD.
|
||||
parsed_body: None,
|
||||
sampling_defaults: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the engine endpoint honors a routed DP rank; `/v1/embeddings` takes none.
|
||||
/// Whether the engine endpoint honors a routed DP rank; embeddings and rerank take none.
|
||||
pub(super) fn accepts_dp_rank(&self) -> bool {
|
||||
self.path != EMBEDDINGS_PATH
|
||||
!matches!(self.path, EMBEDDINGS_PATH | RERANK_PATH)
|
||||
}
|
||||
|
||||
pub(super) fn engine_rid(&self) -> Option<String> {
|
||||
@@ -415,6 +442,18 @@ fn tokenize_input_text(ctx: &AppContext, model: &ModelId, input: &Value) -> Opti
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Prompt text bytes of a string, a multimodal item or a list of content parts;
|
||||
/// image tokens are the engine's to count.
|
||||
fn text_len(content: &Value) -> usize {
|
||||
match content {
|
||||
Value::Array(parts) => parts.iter().map(text_len).sum(),
|
||||
content => content
|
||||
.as_str()
|
||||
.or(content["text"].as_str())
|
||||
.map_or(0, str::len),
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller token IDs, one flat list or a row per prompt; `None` if malformed,
|
||||
/// which the engine reports.
|
||||
fn caller_token_rows(input: &Value) -> Option<Vec<Vec<u32>>> {
|
||||
@@ -1188,6 +1227,21 @@ mod tests {
|
||||
assert_eq!((input, r.tokens.map(|t| t.ids)), (json!("hi"), ids));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rerank_estimates_each_query_document_pair() {
|
||||
let image = json!({"type": "image_url", "image_url": {"url": "x".repeat(4096)}});
|
||||
let documents = json!(["abcd", [{"type": "text", "text": "abcdefgh"}, image]]);
|
||||
for (instruct, total, longest) in [(json!(null), 5, 3), (json!("abcdefgh"), 9, 5)] {
|
||||
let body = json!({"query": "abcd", "documents": documents, "instruct": instruct});
|
||||
let r = PreparedRequest::rerank(ModelId("stub-model".into()), body.to_string().into());
|
||||
let r = r.unwrap();
|
||||
assert_eq!(
|
||||
(r.input_token_count, r.sequence_token_count),
|
||||
(total, longest)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_counts_batches_per_prompt() {
|
||||
// (fans out, (input total, longest prompt, output total, longest peak))
|
||||
|
||||
@@ -991,6 +991,39 @@ async fn full_decode_group_falls_back_to_another_version_group() {
|
||||
assert_eq!(dispatched, [false, true, false, true]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rerank_instructions_count_toward_bucket_context_limits() {
|
||||
let short_worker = MockWorker::start(vec![]).await;
|
||||
let long_worker = MockWorker::start(vec![]).await;
|
||||
let policy = Arc::new(FirstPolicy::default());
|
||||
let mut short = Bucket::new("short", BucketGroups::Plain(group("short", policy.clone())));
|
||||
short.max_context_tokens = Some(1024);
|
||||
let long = Bucket::new("long", BucketGroups::Plain(group("long", policy)));
|
||||
let app = build_router(context(
|
||||
&[
|
||||
("short", Stage::Plain, &short_worker),
|
||||
("long", Stage::Plain, &long_worker),
|
||||
],
|
||||
vec![long, short],
|
||||
));
|
||||
|
||||
// Qwen rerankers include the instruction in every query-document prompt.
|
||||
for (instruct, worker) in [
|
||||
("Rank relevant documents.".to_owned(), &short_worker),
|
||||
("instruction ".repeat(8192), &long_worker),
|
||||
] {
|
||||
let body = serde_json::json!({"query": "hi", "documents": ["yo"], "instruct": instruct});
|
||||
let req = Request::post("/v1/rerank")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap();
|
||||
let response = app.clone().oneshot(req).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
response.into_body().collect().await.unwrap();
|
||||
assert_eq!(worker.captured_json().await, body);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn embeddings_fallback_batches_use_per_prompt_context_limits() {
|
||||
let worker = MockWorker::start(vec![]).await;
|
||||
|
||||
@@ -75,6 +75,7 @@ impl MockWorker {
|
||||
.route("/v1/chat/completions", post(chat))
|
||||
.route("/generate", post(generate))
|
||||
.route("/v1/embeddings", post(embeddings))
|
||||
.route("/v1/rerank", post(rerank))
|
||||
.route("/server_info", get(serve_tiny_server_info))
|
||||
.route("/abort_request", abort_request_route(abort_log.clone()))
|
||||
.with_state(state);
|
||||
@@ -519,3 +520,14 @@ async fn embeddings(
|
||||
let data = [serde_json::json!({"object": "embedding", "embedding": [0.5], "index": 0})];
|
||||
Json(serde_json::json!({"object": "list", "data": data, "model": "tiny"})).into_response()
|
||||
}
|
||||
|
||||
/// A rerank response, a list of scored documents.
|
||||
#[allow(dead_code)] // Used by `MockWorker::start`, only some test files need it.
|
||||
async fn rerank(
|
||||
State(s): State<MockWorkerState>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response<Body> {
|
||||
capture_request(&s, &headers, &body);
|
||||
Json(serde_json::json!([{"score": 0.5, "index": 0}])).into_response()
|
||||
}
|
||||
|
||||
@@ -242,23 +242,27 @@ async fn pd_fan_out_leaves_the_prefill_rank_to_the_engine() {
|
||||
}
|
||||
}
|
||||
|
||||
/// The engine's `/v1/embeddings` reads no rank, so the router pins none.
|
||||
/// The engine's `/v1/embeddings` and `/v1/rerank` read no rank, so the router pins none.
|
||||
#[tokio::test]
|
||||
async fn embeddings_leave_the_rank_to_the_engine() {
|
||||
async fn embeddings_and_rerank_leave_the_rank_to_the_engine() {
|
||||
let worker = MockWorker::start(vec![]).await;
|
||||
let app = router(
|
||||
sticky_config(),
|
||||
&[(&worker, WorkerMode::Plain, 4)],
|
||||
Default::default(),
|
||||
);
|
||||
let request = Request::post("/v1/embeddings")
|
||||
.header("content-type", "application/json")
|
||||
.header(KEY, "conv-a")
|
||||
.body(Body::from(
|
||||
json!({"model": MODEL, "input": "hi"}).to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
assert!(app.oneshot(request).await.unwrap().status().is_success());
|
||||
assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None);
|
||||
assert!(worker.captured_json().await.get("routed_dp_rank").is_none());
|
||||
for (path, body) in [
|
||||
("/v1/embeddings", json!({"model": MODEL, "input": "hi"})),
|
||||
("/v1/rerank", json!({"query": "hi", "documents": ["yo"]})),
|
||||
] {
|
||||
let request = Request::post(path)
|
||||
.header("content-type", "application/json")
|
||||
.header(KEY, "conv-a")
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap();
|
||||
let response = app.clone().oneshot(request).await.unwrap();
|
||||
assert!(response.status().is_success());
|
||||
assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None);
|
||||
assert!(worker.captured_json().await.get("routed_dp_rank").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ mod pd_pool_isolation;
|
||||
mod pd_protocol_binding;
|
||||
mod radix_tree_routing;
|
||||
mod reorg_launch;
|
||||
mod rerank_routing;
|
||||
mod roundrobin_input_ids;
|
||||
mod sampling_overrides;
|
||||
mod shared_prefill_admission;
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! SGLang's `/v1/rerank`: the engine's request and response, forwarded as sent.
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use serde_json::json;
|
||||
use sgl_router::discovery::WorkerMode;
|
||||
use sgl_router::state::kv_events::HashTree;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::common::cache_aware_fixture::radix_router;
|
||||
use crate::common::mock_worker::MockWorker;
|
||||
use crate::common::streaming::collect_body;
|
||||
|
||||
#[tokio::test]
|
||||
async fn rerank_forwards_put_and_rejects_prefill_decode_workers() {
|
||||
let (engine, prefill, decode) = (
|
||||
MockWorker::start(vec![]).await,
|
||||
MockWorker::start(vec![]).await,
|
||||
MockWorker::start(vec![]).await,
|
||||
);
|
||||
let body = json!({"query": "hi", "documents": ["yo", [{"type": "text", "text": "x"}]]});
|
||||
let send = |workers: &[(&MockWorker, WorkerMode)]| {
|
||||
let request = Request::put("/v1/rerank")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap();
|
||||
radix_router(workers, HashTree::new()).oneshot(request)
|
||||
};
|
||||
|
||||
let response = send(&[(&engine, WorkerMode::Plain)]).await.unwrap();
|
||||
let response = collect_body(response.into_body()).await;
|
||||
assert_eq!(response, json!([{"score": 0.5, "index": 0}]).to_string());
|
||||
assert_eq!(engine.captured_json().await, body);
|
||||
|
||||
let pd = [
|
||||
(&prefill, WorkerMode::Prefill),
|
||||
(&decode, WorkerMode::Decode),
|
||||
];
|
||||
let response = send(&pd).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert!(prefill.captured.lock().unwrap().last_body.is_none());
|
||||
}
|
||||
Reference in New Issue
Block a user