diff --git a/experimental/sgl-router/README.md b/experimental/sgl-router/README.md index ea4174b989..7e7e8fb38b 100644 --- a/experimental/sgl-router/README.md +++ b/experimental/sgl-router/README.md @@ -4,11 +4,11 @@ Slim, KV-aware, OpenAI-compatible router for SGLang workers. Serves a single model and routes across its workers. Exposes `/v1/tokenize`, `/v1/detokenize`, `/v1/models`, [`/v1/embeddings`](#embeddings), -`/v1/chat/completions` and SGLang's native [`/generate`](#native-generate) -(buffered and SSE), plus `/healthz` / `/readyz` and `/metrics`. Worker -pools come from either a static URL list or Kubernetes EndpointSlice -discovery. Both edges speak cleartext HTTP/2 where the peer does — see -[HTTP/2](#http2). +[`/v1/rerank`](#rerank), `/v1/chat/completions` and SGLang's native +[`/generate`](#native-generate) (buffered and SSE), plus `/healthz` / `/readyz` +and `/metrics`. Worker pools come from either a static URL list or Kubernetes +EndpointSlice discovery. Both edges speak cleartext HTTP/2 where the peer does — +see [HTTP/2](#http2). ## Building @@ -406,6 +406,14 @@ render. A list is a batch for one worker, routed on load. A single prompt with n `rid` gets a minted one. Under `--dp-aware` the engine picks the rank, since its embeddings endpoint reads none. +## Rerank + +`/v1/rerank` has the engine's interface: the same `V1RerankReqInput` body and +response, sent as `POST` or `PUT` to the model this router serves. The body is +forwarded as sent, since the engine renders and tokenizes each query-document +pair. Routing is on load, with each pair's size estimated from its text. As for +embeddings, a PD fleet answers 400 and `--dp-aware` pins no rank. + ## DeepSeek V4 Native V4 rendering follows SGLang's serving path (`serving_chat.py`), not diff --git a/experimental/sgl-router/src/server/app.rs b/experimental/sgl-router/src/server/app.rs index 880a7c5041..0e6adc502b 100644 --- a/experimental/sgl-router/src/server/app.rs +++ b/experimental/sgl-router/src/server/app.rs @@ -218,6 +218,13 @@ pub fn build_router(ctx: Arc) -> Router { .layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES)) .layer(middleware::from_fn(log_413)), ) + .route( + "/v1/rerank", + post(crate::server::routes::chat::rerank) + .put(crate::server::routes::chat::rerank) + .layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES)) + .layer(middleware::from_fn(log_413)), + ) .route( "/flush_cache", post(crate::server::routes::cache::flush_cache), diff --git a/experimental/sgl-router/src/server/routes/chat.rs b/experimental/sgl-router/src/server/routes/chat.rs index 7b0dbab18f..f636a81117 100644 --- a/experimental/sgl-router/src/server/routes/chat.rs +++ b/experimental/sgl-router/src/server/routes/chat.rs @@ -86,18 +86,38 @@ pub async fn embeddings( let start = Instant::now(); let (model, value) = parse_embedding_request(&body)?; let routing = ModelRouting::lookup(&ctx, &model)?; - // Prefill and decode engines serve generation; embeddings need plain workers. - let registered = ctx.registry.workers_for(&model); - if registered.iter().any(|w| w.mode() != WorkerMode::Plain) { - return Err(ApiError::BadRequest( - "embeddings are not served by prefill-decode workers".into(), - )); - } + require_plain_workers(&ctx, &model, "/v1/embeddings")?; let request = PreparedRequest::embeddings(&ctx, model, body, value)?; let workers = routing.select_workers(&ctx, &request, &headers).await?; forward_request(&ctx, request, workers, headers, start).await } +/// SGLang's `/v1/rerank`, forwarded as sent to the model this router serves. +pub async fn rerank( + State(ctx): State>, + headers: HeaderMap, + body: Bytes, +) -> Result, ApiError> { + let start = Instant::now(); + let model = ModelId(ctx.config.model.id.clone()); + let routing = ModelRouting::lookup(&ctx, &model)?; + require_plain_workers(&ctx, &model, "/v1/rerank")?; + let request = PreparedRequest::rerank(model, body)?; + let workers = routing.select_workers(&ctx, &request, &headers).await?; + forward_request(&ctx, request, workers, headers, start).await +} + +/// Prefill and decode engines serve generation only. +fn require_plain_workers(ctx: &AppContext, model: &ModelId, path: &str) -> Result<(), ApiError> { + let registered = ctx.registry.workers_for(model); + if registered.iter().any(|w| w.mode() != WorkerMode::Plain) { + return Err(ApiError::BadRequest(format!( + "{path} is not served by prefill-decode workers" + ))); + } + Ok(()) +} + /// A model's routing state, resolved before the request is prepared. enum ModelRouting<'a> { Legacy(Arc), diff --git a/experimental/sgl-router/src/server/routes/chat/preparation.rs b/experimental/sgl-router/src/server/routes/chat/preparation.rs index 9e6b790958..1a0b0266ab 100644 --- a/experimental/sgl-router/src/server/routes/chat/preparation.rs +++ b/experimental/sgl-router/src/server/routes/chat/preparation.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Request validation, optional tokenization, and outgoing body preparation for -//! chat completions, the native `/generate` endpoint, and embeddings. +//! chat completions, the native `/generate` endpoint, embeddings and rerank. use crate::config::{ConflictPolicy, ParamSpec, SamplingField, SamplingOverrides}; use crate::discovery::ModelId; @@ -22,6 +22,7 @@ const BYTES_PER_TOKEN_ESTIMATE: usize = 4; const CHAT_PATH: &str = "/v1/chat/completions"; const GENERATE_PATH: &str = "/generate"; const EMBEDDINGS_PATH: &str = "/v1/embeddings"; +const RERANK_PATH: &str = "/v1/rerank"; /// Validated routing inputs and the original body, ready for worker selection. /// @@ -44,7 +45,7 @@ pub(super) struct PreparedRequest { pub(super) expected_peak_sequence_tokens: Option, caller_set_rid: bool, pub(super) fans_out: bool, - /// `None` for `/generate` and embeddings, whose tokens replace text during preparation. + /// `None` for `/generate`, embeddings and rerank, which prepare their own body. forwarding_scope: Option, parsed_body: Option, sampling_defaults: Vec<(SamplingField, Number)>, @@ -194,17 +195,13 @@ impl PreparedRequest { .into(); } let prompts = text_ids.or_else(|| caller_token_rows(&value["input"])); - // Context limits apply to each prompt even when the engine must tokenize text; - // multimodal items count only their text, as image tokens are the engine's to count. - let estimate = |item: &Value| { - estimate_prefill_tokens(item.as_str().or(item["text"].as_str()).map_or(0, str::len)) - }; - let lengths = match (&prompts, &value["input"]) { + // Context limits apply to each prompt even when the engine must tokenize text. + let estimate = |item: &Value| estimate_prefill_tokens(text_len(item)); + let lengths: Vec<_> = match (&prompts, &value["input"]) { (Some(prompts), _) => prompts.iter().map(|ids| ids.len().max(1)).collect(), (None, Value::Array(items)) => items.iter().map(estimate).collect(), (None, input) => vec![estimate(input)], }; - let sequence_tokens = lengths.iter().copied().max().unwrap_or(1); let tokens = prompts .filter(|_| !batch) .and_then(|prompts| prompts.into_iter().next()) @@ -213,7 +210,38 @@ impl PreparedRequest { rendered_from_chat: false, }); Ok(Self { - path: EMBEDDINGS_PATH, + tokens, + caller_set_rid: !value["rid"].is_null(), + fans_out: batch, + ..Self::no_output(EMBEDDINGS_PATH, model, body, &lengths) + }) + } + + /// SGLang's `V1RerankReqInput`, forwarded as sent: the engine renders and + /// tokenizes each query-document pair as its own prompt. + pub(super) fn rerank(model: ModelId, body: Bytes) -> Result { + let value = Value::Object(serde_json::from_slice(&body).map_err(|_| invalid_request())?); + // Qwen rerankers include the instruction in every query-document prompt. + let shared = text_len(&value["instruct"]) + text_len(&value["query"]); + let lengths: Vec<_> = value["documents"] + .as_array() + .into_iter() + .flatten() + .map(|document| estimate_prefill_tokens(shared + text_len(document))) + .collect(); + Ok(Self { + // Each document is its own engine request, and the engine reads no `rid`. + fans_out: true, + ..Self::no_output(RERANK_PATH, model, body, &lengths) + }) + } + + /// A request of `lengths` prompts that generates nothing. No PD serves it, + /// so it keeps no parsed body for bootstrap fields. + fn no_output(path: &'static str, model: ModelId, body: Bytes, lengths: &[usize]) -> Self { + let sequence_tokens = lengths.iter().copied().max().unwrap_or(1); + Self { + path, model, streaming: false, output_tokens: Some(0), @@ -221,19 +249,18 @@ impl PreparedRequest { sequence_token_count: sequence_tokens, expected_peak_sequence_tokens: Some(sequence_tokens as u64), body, - tokens, - caller_set_rid: !value["rid"].is_null(), - fans_out: batch, + tokens: None, + caller_set_rid: false, + fans_out: false, forwarding_scope: None, - // Only PD bootstrap needs the parsed body, and embeddings have no PD. parsed_body: None, sampling_defaults: Vec::new(), - }) + } } - /// Whether the engine endpoint honors a routed DP rank; `/v1/embeddings` takes none. + /// Whether the engine endpoint honors a routed DP rank; embeddings and rerank take none. pub(super) fn accepts_dp_rank(&self) -> bool { - self.path != EMBEDDINGS_PATH + !matches!(self.path, EMBEDDINGS_PATH | RERANK_PATH) } pub(super) fn engine_rid(&self) -> Option { @@ -415,6 +442,18 @@ fn tokenize_input_text(ctx: &AppContext, model: &ModelId, input: &Value) -> Opti .collect() } +/// Prompt text bytes of a string, a multimodal item or a list of content parts; +/// image tokens are the engine's to count. +fn text_len(content: &Value) -> usize { + match content { + Value::Array(parts) => parts.iter().map(text_len).sum(), + content => content + .as_str() + .or(content["text"].as_str()) + .map_or(0, str::len), + } +} + /// Caller token IDs, one flat list or a row per prompt; `None` if malformed, /// which the engine reports. fn caller_token_rows(input: &Value) -> Option>> { @@ -1188,6 +1227,21 @@ mod tests { assert_eq!((input, r.tokens.map(|t| t.ids)), (json!("hi"), ids)); } + #[test] + fn rerank_estimates_each_query_document_pair() { + let image = json!({"type": "image_url", "image_url": {"url": "x".repeat(4096)}}); + let documents = json!(["abcd", [{"type": "text", "text": "abcdefgh"}, image]]); + for (instruct, total, longest) in [(json!(null), 5, 3), (json!("abcdefgh"), 9, 5)] { + let body = json!({"query": "abcd", "documents": documents, "instruct": instruct}); + let r = PreparedRequest::rerank(ModelId("stub-model".into()), body.to_string().into()); + let r = r.unwrap(); + assert_eq!( + (r.input_token_count, r.sequence_token_count), + (total, longest) + ); + } + } + #[test] fn generate_counts_batches_per_prompt() { // (fans out, (input total, longest prompt, output total, longest peak)) diff --git a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs index 1e829ba5a4..afa779303c 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs @@ -991,6 +991,39 @@ async fn full_decode_group_falls_back_to_another_version_group() { assert_eq!(dispatched, [false, true, false, true]); } +#[tokio::test] +async fn rerank_instructions_count_toward_bucket_context_limits() { + let short_worker = MockWorker::start(vec![]).await; + let long_worker = MockWorker::start(vec![]).await; + let policy = Arc::new(FirstPolicy::default()); + let mut short = Bucket::new("short", BucketGroups::Plain(group("short", policy.clone()))); + short.max_context_tokens = Some(1024); + let long = Bucket::new("long", BucketGroups::Plain(group("long", policy))); + let app = build_router(context( + &[ + ("short", Stage::Plain, &short_worker), + ("long", Stage::Plain, &long_worker), + ], + vec![long, short], + )); + + // Qwen rerankers include the instruction in every query-document prompt. + for (instruct, worker) in [ + ("Rank relevant documents.".to_owned(), &short_worker), + ("instruction ".repeat(8192), &long_worker), + ] { + let body = serde_json::json!({"query": "hi", "documents": ["yo"], "instruct": instruct}); + let req = Request::post("/v1/rerank") + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(); + let response = app.clone().oneshot(req).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + response.into_body().collect().await.unwrap(); + assert_eq!(worker.captured_json().await, body); + } +} + #[tokio::test] async fn embeddings_fallback_batches_use_per_prompt_context_limits() { let worker = MockWorker::start(vec![]).await; diff --git a/experimental/sgl-router/tests/proxy/common/mock_worker.rs b/experimental/sgl-router/tests/proxy/common/mock_worker.rs index eb3a9d0dce..56bbd77319 100644 --- a/experimental/sgl-router/tests/proxy/common/mock_worker.rs +++ b/experimental/sgl-router/tests/proxy/common/mock_worker.rs @@ -75,6 +75,7 @@ impl MockWorker { .route("/v1/chat/completions", post(chat)) .route("/generate", post(generate)) .route("/v1/embeddings", post(embeddings)) + .route("/v1/rerank", post(rerank)) .route("/server_info", get(serve_tiny_server_info)) .route("/abort_request", abort_request_route(abort_log.clone())) .with_state(state); @@ -519,3 +520,14 @@ async fn embeddings( let data = [serde_json::json!({"object": "embedding", "embedding": [0.5], "index": 0})]; Json(serde_json::json!({"object": "list", "data": data, "model": "tiny"})).into_response() } + +/// A rerank response, a list of scored documents. +#[allow(dead_code)] // Used by `MockWorker::start`, only some test files need it. +async fn rerank( + State(s): State, + headers: HeaderMap, + body: Bytes, +) -> Response { + capture_request(&s, &headers, &body); + Json(serde_json::json!([{"score": 0.5, "index": 0}])).into_response() +} diff --git a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs index 70aa72a5e5..4d979ab0b8 100644 --- a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs +++ b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs @@ -242,23 +242,27 @@ async fn pd_fan_out_leaves_the_prefill_rank_to_the_engine() { } } -/// The engine's `/v1/embeddings` reads no rank, so the router pins none. +/// The engine's `/v1/embeddings` and `/v1/rerank` read no rank, so the router pins none. #[tokio::test] -async fn embeddings_leave_the_rank_to_the_engine() { +async fn embeddings_and_rerank_leave_the_rank_to_the_engine() { let worker = MockWorker::start(vec![]).await; let app = router( sticky_config(), &[(&worker, WorkerMode::Plain, 4)], Default::default(), ); - let request = Request::post("/v1/embeddings") - .header("content-type", "application/json") - .header(KEY, "conv-a") - .body(Body::from( - json!({"model": MODEL, "input": "hi"}).to_string(), - )) - .unwrap(); - assert!(app.oneshot(request).await.unwrap().status().is_success()); - assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None); - assert!(worker.captured_json().await.get("routed_dp_rank").is_none()); + for (path, body) in [ + ("/v1/embeddings", json!({"model": MODEL, "input": "hi"})), + ("/v1/rerank", json!({"query": "hi", "documents": ["yo"]})), + ] { + let request = Request::post(path) + .header("content-type", "application/json") + .header(KEY, "conv-a") + .body(Body::from(body.to_string())) + .unwrap(); + let response = app.clone().oneshot(request).await.unwrap(); + assert!(response.status().is_success()); + assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None); + assert!(worker.captured_json().await.get("routed_dp_rank").is_none()); + } } diff --git a/experimental/sgl-router/tests/proxy/main.rs b/experimental/sgl-router/tests/proxy/main.rs index 42ad6aa735..d3bc6a9cd3 100644 --- a/experimental/sgl-router/tests/proxy/main.rs +++ b/experimental/sgl-router/tests/proxy/main.rs @@ -27,6 +27,7 @@ mod pd_pool_isolation; mod pd_protocol_binding; mod radix_tree_routing; mod reorg_launch; +mod rerank_routing; mod roundrobin_input_ids; mod sampling_overrides; mod shared_prefill_admission; diff --git a/experimental/sgl-router/tests/proxy/rerank_routing.rs b/experimental/sgl-router/tests/proxy/rerank_routing.rs new file mode 100644 index 0000000000..0726f42aa5 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/rerank_routing.rs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! SGLang's `/v1/rerank`: the engine's request and response, forwarded as sent. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::json; +use sgl_router::discovery::WorkerMode; +use sgl_router::state::kv_events::HashTree; +use tower::ServiceExt; + +use crate::common::cache_aware_fixture::radix_router; +use crate::common::mock_worker::MockWorker; +use crate::common::streaming::collect_body; + +#[tokio::test] +async fn rerank_forwards_put_and_rejects_prefill_decode_workers() { + let (engine, prefill, decode) = ( + MockWorker::start(vec![]).await, + MockWorker::start(vec![]).await, + MockWorker::start(vec![]).await, + ); + let body = json!({"query": "hi", "documents": ["yo", [{"type": "text", "text": "x"}]]}); + let send = |workers: &[(&MockWorker, WorkerMode)]| { + let request = Request::put("/v1/rerank") + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(); + radix_router(workers, HashTree::new()).oneshot(request) + }; + + let response = send(&[(&engine, WorkerMode::Plain)]).await.unwrap(); + let response = collect_body(response.into_body()).await; + assert_eq!(response, json!([{"score": 0.5, "index": 0}]).to_string()); + assert_eq!(engine.captured_json().await, body); + + let pd = [ + (&prefill, WorkerMode::Prefill), + (&decode, WorkerMode::Decode), + ]; + let response = send(&pd).await.unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(prefill.captured.lock().unwrap().last_body.is_none()); +}