From edee4308bcd7204d550234189d52a4cc25d93348 Mon Sep 17 00:00:00 2001 From: Kan Wu Date: Fri, 2 Oct 2026 05:18:24 -0700 Subject: [PATCH] [sgl-router] Add OpenAI /v1/embeddings (#42191) Co-authored-by: Kan Wu Co-authored-by: Claude Opus 5.5 --- experimental/sgl-router/README.md | 19 +- experimental/sgl-router/src/server/app.rs | 6 + .../sgl-router/src/server/routes/chat.rs | 23 ++- .../src/server/routes/chat/forward.rs | 2 +- .../src/server/routes/chat/preparation.rs | 195 +++++++++++++++++- .../sgl-router/src/tokenizer/adapter.rs | 127 +++++++++--- experimental/sgl-router/src/tokenizer/mod.rs | 9 +- .../tests/proxy/chat_routing/reorg.rs | 42 ++++ .../tests/proxy/common/mock_worker.rs | 13 ++ .../sgl-router/tests/proxy/dp_rank_routing.rs | 21 ++ .../tests/proxy/embeddings_routing.rs | 97 +++++++++ experimental/sgl-router/tests/proxy/main.rs | 1 + 12 files changed, 510 insertions(+), 45 deletions(-) create mode 100644 experimental/sgl-router/tests/proxy/embeddings_routing.rs diff --git a/experimental/sgl-router/README.md b/experimental/sgl-router/README.md index b15312b514..fec6c94180 100644 --- a/experimental/sgl-router/README.md +++ b/experimental/sgl-router/README.md @@ -3,9 +3,9 @@ Slim, KV-aware, OpenAI-compatible router for SGLang workers. Serves a single model and routes across its workers. Exposes -`/v1/tokenize`, `/v1/detokenize`, `/v1/models`, `/v1/chat/completions` and -SGLang's native [`/generate`](#native-generate) (buffered and SSE), plus -`/healthz` / `/readyz` and `/metrics`. Worker +`/v1/tokenize`, `/v1/detokenize`, `/v1/models`, [`/v1/embeddings`](#embeddings), +`/v1/chat/completions` and SGLang's native [`/generate`](#native-generate) +(buffered and SSE), plus `/healthz` / `/readyz` and `/metrics`. Worker pools come from either a static URL list or Kubernetes EndpointSlice discovery. Both edges speak cleartext HTTP/2 where the peer does — see [HTTP/2](#http2). @@ -379,6 +379,19 @@ a single prompt that has none, and `--override-sampling-params` defaults. Under batch or `n > 1` request leaves the prefill rank to the engine, which gives item `i` the bootstrap room `room + i`. +## Embeddings + +`/v1/embeddings` has the engine's interface: the same OpenAI `EmbeddingRequest` +body and response. As for chat completions, `model` must name the served model. +A PD fleet answers 400, since prefill and decode engines serve no embeddings. + +Text `input` (a string or a list) is tokenized and forwarded as token IDs, as for +[`/generate`](#native-generate), plus the EOS SGLang appends for EmbeddingGemma. +Blank prompts and multimodal items stay as sent, for the engine to reject or +render. A list is a batch for one worker, routed on load. A single prompt with no +`rid` gets a minted one. Under `--dp-aware` the engine picks the rank, since its +embeddings endpoint reads none. + ## DeepSeek V4 Native V4 rendering follows SGLang's serving path (`serving_chat.py`), not diff --git a/experimental/sgl-router/src/server/app.rs b/experimental/sgl-router/src/server/app.rs index b4489afcae..880a7c5041 100644 --- a/experimental/sgl-router/src/server/app.rs +++ b/experimental/sgl-router/src/server/app.rs @@ -212,6 +212,12 @@ pub fn build_router(ctx: Arc) -> Router { .layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES)) .layer(middleware::from_fn(log_413)), ) + .route( + "/v1/embeddings", + post(crate::server::routes::chat::embeddings) + .layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES)) + .layer(middleware::from_fn(log_413)), + ) .route( "/flush_cache", post(crate::server::routes::cache::flush_cache), diff --git a/experimental/sgl-router/src/server/routes/chat.rs b/experimental/sgl-router/src/server/routes/chat.rs index b4e67aaf67..7b0dbab18f 100644 --- a/experimental/sgl-router/src/server/routes/chat.rs +++ b/experimental/sgl-router/src/server/routes/chat.rs @@ -24,7 +24,7 @@ use axum::extract::State; use axum::http::{HeaderMap, HeaderName, Response}; use bytes::Bytes; use forward::{forward_request, SelectedWorkers}; -use preparation::{parse_routing_fields, PreparedRequest}; +use preparation::{parse_embedding_request, parse_routing_fields, PreparedRequest}; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; @@ -77,6 +77,27 @@ pub async fn generate( forward_request(&ctx, request, workers, headers, start).await } +/// OpenAI `/v1/embeddings`, forwarded to the engine's with the same request and response. +pub async fn embeddings( + State(ctx): State>, + headers: HeaderMap, + body: Bytes, +) -> Result, ApiError> { + let start = Instant::now(); + let (model, value) = parse_embedding_request(&body)?; + let routing = ModelRouting::lookup(&ctx, &model)?; + // Prefill and decode engines serve generation; embeddings need plain workers. + let registered = ctx.registry.workers_for(&model); + if registered.iter().any(|w| w.mode() != WorkerMode::Plain) { + return Err(ApiError::BadRequest( + "embeddings are not served by prefill-decode workers".into(), + )); + } + let request = PreparedRequest::embeddings(&ctx, model, body, value)?; + let workers = routing.select_workers(&ctx, &request, &headers).await?; + forward_request(&ctx, request, workers, headers, start).await +} + /// A model's routing state, resolved before the request is prepared. enum ModelRouting<'a> { Legacy(Arc), diff --git a/experimental/sgl-router/src/server/routes/chat/forward.rs b/experimental/sgl-router/src/server/routes/chat/forward.rs index 4c3bf6d349..0f45615c6c 100644 --- a/experimental/sgl-router/src/server/routes/chat/forward.rs +++ b/experimental/sgl-router/src/server/routes/chat/forward.rs @@ -61,7 +61,7 @@ pub(super) async fn forward_request( } // Only the router chooses DP ranks; a client-supplied rank is never forwarded. headers.remove(X_DATA_PARALLEL_RANK); - let dp_aware = ctx.config.model.dp_aware; + let dp_aware = ctx.config.model.dp_aware && request.accepts_dp_rank(); // The engine gives fan-out item i the room `room + i`, so in PD decode looks // for each item on a different prefill rank; one pinned rank would break that. let unpin_prefill = dp_aware && decode.is_some() && request.fans_out; diff --git a/experimental/sgl-router/src/server/routes/chat/preparation.rs b/experimental/sgl-router/src/server/routes/chat/preparation.rs index cb30b629f1..9e6b790958 100644 --- a/experimental/sgl-router/src/server/routes/chat/preparation.rs +++ b/experimental/sgl-router/src/server/routes/chat/preparation.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Request validation, optional tokenization, and outgoing body preparation for -//! chat completions and the native `/generate` endpoint. +//! chat completions, the native `/generate` endpoint, and embeddings. use crate::config::{ConflictPolicy, ParamSpec, SamplingField, SamplingOverrides}; use crate::discovery::ModelId; @@ -21,6 +21,7 @@ const BYTES_PER_TOKEN_ESTIMATE: usize = 4; const CHAT_PATH: &str = "/v1/chat/completions"; const GENERATE_PATH: &str = "/generate"; +const EMBEDDINGS_PATH: &str = "/v1/embeddings"; /// Validated routing inputs and the original body, ready for worker selection. /// @@ -43,7 +44,7 @@ pub(super) struct PreparedRequest { pub(super) expected_peak_sequence_tokens: Option, caller_set_rid: bool, pub(super) fans_out: bool, - /// `None` for `/generate`, whose tokens replace `text` during preparation. + /// `None` for `/generate` and embeddings, whose tokens replace text during preparation. forwarding_scope: Option, parsed_body: Option, sampling_defaults: Vec<(SamplingField, Number)>, @@ -175,6 +176,66 @@ impl PreparedRequest { }) } + /// The engine's OpenAI `EmbeddingRequest`, whose text `input` is forwarded + /// as token IDs, as `/generate` forwards `text`. + pub(super) fn embeddings( + ctx: &AppContext, + model: ModelId, + mut body: Bytes, + mut value: Value, + ) -> Result { + let batch = is_embedding_batch(&value["input"]); + let text_ids = tokenize_input_text(ctx, &model, &value["input"]); + let forward = !ctx.config.model.disable_input_ids_forwarding; + if let Some(ids) = text_ids.as_ref().filter(|_| forward) { + value["input"] = if batch { json!(ids) } else { json!(ids[0]) }; + body = serde_json::to_vec(&value) + .map_err(|error| ApiError::Internal(error.into()))? + .into(); + } + let prompts = text_ids.or_else(|| caller_token_rows(&value["input"])); + // Context limits apply to each prompt even when the engine must tokenize text; + // multimodal items count only their text, as image tokens are the engine's to count. + let estimate = |item: &Value| { + estimate_prefill_tokens(item.as_str().or(item["text"].as_str()).map_or(0, str::len)) + }; + let lengths = match (&prompts, &value["input"]) { + (Some(prompts), _) => prompts.iter().map(|ids| ids.len().max(1)).collect(), + (None, Value::Array(items)) => items.iter().map(estimate).collect(), + (None, input) => vec![estimate(input)], + }; + let sequence_tokens = lengths.iter().copied().max().unwrap_or(1); + let tokens = prompts + .filter(|_| !batch) + .and_then(|prompts| prompts.into_iter().next()) + .map(|ids| RequestTokens { + ids, + rendered_from_chat: false, + }); + Ok(Self { + path: EMBEDDINGS_PATH, + model, + streaming: false, + output_tokens: Some(0), + input_token_count: lengths.iter().sum::().max(1), + sequence_token_count: sequence_tokens, + expected_peak_sequence_tokens: Some(sequence_tokens as u64), + body, + tokens, + caller_set_rid: !value["rid"].is_null(), + fans_out: batch, + forwarding_scope: None, + // Only PD bootstrap needs the parsed body, and embeddings have no PD. + parsed_body: None, + sampling_defaults: Vec::new(), + }) + } + + /// Whether the engine endpoint honors a routed DP rank; `/v1/embeddings` takes none. + pub(super) fn accepts_dp_rank(&self) -> bool { + self.path != EMBEDDINGS_PATH + } + pub(super) fn engine_rid(&self) -> Option { // Caller IDs are unsafe for prefix aborts; fan-out regenerates IDs. // PD shares the minted ID, but only decode arms abort-on-drop. @@ -310,6 +371,60 @@ fn parallel_samples(value: &Value) -> u64 { } } +/// An embeddings body and the model it names; chat completions also require one. +pub(super) fn parse_embedding_request(body: &Bytes) -> Result<(ModelId, Value), ApiError> { + let value = Value::Object(serde_json::from_slice(body).map_err(|_| invalid_request())?); + let model = Option::::deserialize(&value["model"]) + .map_err(|_| invalid_request())? + .ok_or_else(|| ApiError::BadRequest("missing `model` field".into()))?; + Ok((ModelId(model), value)) +} + +/// Whether the engine runs an embedding `input` as a batch: any list but one +/// of token IDs or a single multimodal item (`serving_embedding.py`). +fn is_embedding_batch(input: &Value) -> bool { + match input.as_array().map(Vec::as_slice) { + Some([first, ..]) if first.is_number() => false, + Some([only]) if only.is_object() => false, + Some(items) => !items.is_empty(), + None => false, + } +} + +/// Text `input` as the engine's tokens, one row per prompt; `None` for other +/// input and for text the engine rejects: mixed lists and blank prompts. +fn tokenize_input_text(ctx: &AppContext, model: &ModelId, input: &Value) -> Option>> { + let texts = match input { + Value::String(text) => vec![text.as_str()], + Value::Array(items) if items.first()?.is_string() => { + items.iter().map(Value::as_str).collect::>()? + } + _ => return None, + }; + // Python's `str.strip()` also strips U+001C..U+001F. + let blank = |text: &&str| { + text.trim_matches(|c: char| c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)) + .is_empty() + }; + if texts.iter().any(blank) { + return None; + } + texts + .into_iter() + .map(|text| ctx.tokenizers.encode_prompt(&model.0, text)) + .collect() +} + +/// Caller token IDs, one flat list or a row per prompt; `None` if malformed, +/// which the engine reports. +fn caller_token_rows(input: &Value) -> Option>> { + match input.as_array()?.first()? { + Value::Number(_) => Some(vec![Vec::deserialize(input).ok()?]), + Value::Array(_) => Vec::deserialize(input).ok(), + _ => None, + } +} + /// Routing and sampling fields retained by the lightweight request parser. #[derive(Debug, Default)] pub(super) struct RoutingFields { @@ -985,9 +1100,8 @@ mod tests { } } - /// A `/generate` request prepared with the tiny test tokenizer loaded. - fn prepare_generate(body: Value) -> PreparedRequest { - // The tiny tokenizer, with a post-processor that prepends BOS (id 256). + /// The stub context with the tiny tokenizer, whose post-processor prepends BOS (id 256). + fn tokenizer_ctx() -> AppContext { let tokenizer = std::fs::read_to_string("tests/fixtures/tiny_tokenizer.json").unwrap(); let mut tokenizer: Value = serde_json::from_str(&tokenizer).unwrap(); tokenizer["post_processor"] = json!({"type": "TemplateProcessing", "pair": [], "single": [ @@ -1001,8 +1115,77 @@ mod tests { let mut ctx = AppContext::stub(); ctx.config.model.tokenizer_path = Some(path.to_str().unwrap().into()); ctx.tokenizers = Arc::new(TokenizerRegistry::load_from_config(&ctx.config).unwrap()); + ctx + } + + /// A `/generate` request prepared with the tiny test tokenizer loaded. + fn prepare_generate(body: Value) -> PreparedRequest { let body = Bytes::from(body.to_string()); - PreparedRequest::generate(&ctx, ModelId("stub-model".into()), body).unwrap() + PreparedRequest::generate(&tokenizer_ctx(), ModelId("stub-model".into()), body).unwrap() + } + + fn prepare_embeddings(ctx: &AppContext, input: Value) -> (PreparedRequest, Value) { + let body = Bytes::from(json!({"model": "stub-model", "input": input}).to_string()); + let (model, value) = parse_embedding_request(&body).unwrap(); + let r = PreparedRequest::embeddings(ctx, model, body, value).unwrap(); + let input = serde_json::from_slice::(&r.body).unwrap()["input"].take(); + (r, input) + } + + #[test] + fn embeddings_forward_text_input_as_token_ids() { + let ctx = tokenizer_ctx(); + let hi = ctx.tokenizers.encode_prompt("stub-model", "hi").unwrap(); + let n = hi.len(); + // (input, forwarded input, fans out, (input total, longest prompt)) + for (sent, forwarded, fans_out, counts) in [ + (json!("hi"), json!(hi), false, (n, n)), + // The engine runs any text list as a batch, even of one. + (json!(["hi"]), json!([hi]), true, (n, n)), + (json!([[5, 6], [7]]), json!([[5, 6], [7]]), true, (3, 2)), + ] { + let (r, input) = prepare_embeddings(&ctx, sent); + let got = (r.input_token_count, r.sequence_token_count); + assert_eq!((input, r.fans_out, got), (forwarded, fans_out, counts)); + assert_eq!(r.tokens.is_some(), !fans_out); + } + } + + #[test] + fn embeddings_estimate_fallback_text_per_prompt() { + let ctx = AppContext::stub(); + for (input, total, longest) in [ + (json!("abcdefgh"), 2, 2), + (json!(["abcd", "abcdefgh"]), 3, 2), + // An image's bytes are not prompt text. + (json!([{"text": "abcd", "image": "x".repeat(4096)}]), 1, 1), + ] { + let (r, forwarded) = prepare_embeddings(&ctx, input.clone()); + assert_eq!(forwarded, input); + assert!(r.tokens.is_none()); + assert_eq!(r.input_token_count, total); + assert_eq!(r.sequence_token_count, longest); + assert_eq!(r.expected_peak_sequence_tokens, Some(longest as u64)); + } + } + + #[test] + fn embeddings_leave_other_input_to_the_engine() { + let mut ctx = tokenizer_ctx(); + // The engine rejects blank and mixed lists, and renders multimodal items. + for sent in [ + json!(["hi", " \u{1c}"]), + json!(["hi", 5]), + json!([{"text": "hi"}]), + ] { + let (r, input) = prepare_embeddings(&ctx, sent.clone()); + assert!(input == sent && r.tokens.is_none()); + } + // Without forwarding, routing still sees the engine's tokens. + ctx.config.model.disable_input_ids_forwarding = true; + let (r, input) = prepare_embeddings(&ctx, json!("hi")); + let ids = ctx.tokenizers.encode_prompt("stub-model", "hi"); + assert_eq!((input, r.tokens.map(|t| t.ids)), (json!("hi"), ids)); } #[test] diff --git a/experimental/sgl-router/src/tokenizer/adapter.rs b/experimental/sgl-router/src/tokenizer/adapter.rs index c69a6c7390..639687535a 100644 --- a/experimental/sgl-router/src/tokenizer/adapter.rs +++ b/experimental/sgl-router/src/tokenizer/adapter.rs @@ -73,11 +73,30 @@ fn engine_keeps_normalizer(class: &str, normalizer: &serde_json::Value) -> bool } } -/// Token ids SGLang's `tokenizer(text)` puts around a prompt and [`encode`] -/// leaves out: BOS per `add_bos_token` (default true) for [`BOS_FLAG_CLASSES`], -/// else the tokenizer.json post-processor's. Tiktoken models add none. Errs when -/// the router cannot reproduce the engine's tokens. -pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result<(Vec, Vec)> { +/// Special tokens SGLang's `tokenizer(text)` puts around a prompt and [`encode`] leaves out. +#[derive(Debug, Default, PartialEq)] +pub struct PromptAffixes { + pub prefix: Vec, + pub suffix: Vec, + /// EOS that SGLang appends to an EmbeddingGemma prompt not already ending in it. + pub eos: Option, +} + +impl PromptAffixes { + /// [`encode`]'s `ids` as the engine tokenizes the same text. + pub fn apply(&self, ids: &[u32]) -> Vec { + let mut ids = [self.prefix.as_slice(), ids, &self.suffix].concat(); + if let Some(eos) = self.eos.filter(|&eos| ids.last() != Some(&eos)) { + ids.push(eos); + } + ids + } +} + +/// BOS per `add_bos_token` (default true) for [`BOS_FLAG_CLASSES`], else the +/// tokenizer.json post-processor's, and EmbeddingGemma's EOS. Tiktoken models add +/// none. Errs when the router cannot reproduce the engine's tokens. +pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result { let path = resolve(source)?; if !path.ends_with(".json") { return Ok(Default::default()); @@ -98,19 +117,34 @@ pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result<(Vec, Vec let ids = |t: &dyn traits::Tokenizer, text: &str| -> Result> { Ok(t.encode(text)?.token_ids().to_vec()) }; - if BOS_FLAG_CLASSES.contains(&class) { - let bos = &config["bos_token"]; - let bos = bos.as_str().or(bos["content"].as_str()); - let prefix = if config["add_bos_token"].as_bool().unwrap_or(true) { - let bos = bos.context("add_bos_token is set without a bos_token")?; - match ids(plain.as_ref(), bos)?.as_slice() { - [id] => vec![*id], - _ => anyhow::bail!("bos_token {bos:?} is not a single token"), - } - } else { - Vec::new() + let special = |key: &str| -> Result> { + let Some(token) = config[key].as_str().or(config[key]["content"].as_str()) else { + return Ok(None); }; - return Ok((prefix, Vec::new())); + match ids(plain.as_ref(), token)?.as_slice() { + [id] => Ok(Some(*id)), + _ => anyhow::bail!("{key} {token:?} is not a single token"), + } + }; + // SGLang's `is_embedding_gemma`, whose `_tokenize_texts` ends each prompt with EOS. + files.ensure_downloaded("config.json")?; + let model = files.json("config.json")?.unwrap_or_default(); + let embedding_gemma = + model["model_type"] == "gemma3_text" && model["use_bidirectional_attention"] == true; + let eos = embedding_gemma + .then(|| special("eos_token")?.context("EmbeddingGemma declares no eos_token")) + .transpose()?; + if BOS_FLAG_CLASSES.contains(&class) { + let add_bos = config["add_bos_token"].as_bool().unwrap_or(true); + let bos = add_bos + .then(|| special("bos_token")?.context("add_bos_token is set without a bos_token")) + .transpose()?; + let prefix = bos.into_iter().collect(); + return Ok(PromptAffixes { + prefix, + suffix: Vec::new(), + eos, + }); } let options = TokenizerOptions { add_special_tokens: true, @@ -125,7 +159,12 @@ pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result<(Vec, Vec .windows(bare.len()) .position(|window| window == bare) .context("the post-processor rewrites the prompt")?; - Ok((full[..start].to_vec(), full[start + bare.len()..].to_vec())) + let (prefix, suffix) = (full[..start].to_vec(), full[start + bare.len()..].to_vec()); + Ok(PromptAffixes { + prefix, + suffix, + eos, + }) } fn build(path: &str, cfg: TokenizerConfig) -> Result<(Arc, Arc)> { @@ -503,16 +542,18 @@ mod model_files_tests { #[cfg(test)] mod prompt_affix_tests { - use super::{prompt_affixes, ModelFiles}; + use super::{prompt_affixes, ModelFiles, PromptAffixes}; use anyhow::Result; use serde_json::{json, Value}; - /// `prompt_affixes` of the tiny tokenizer with `normalizer`, next to `config` unless null. + /// `prompt_affixes` of the tiny tokenizer with `normalizer`, next to `config` + /// (unless null) and the `model` config. fn affixes( normalizer: Value, post_processor: Value, config: Value, - ) -> Result<(Vec, Vec)> { + model: Value, + ) -> Result { let mut data: Value = serde_json::from_str(include_str!("../../tests/fixtures/tiny_tokenizer.json")).unwrap(); data["normalizer"] = normalizer; @@ -523,6 +564,7 @@ mod prompt_affix_tests { if !config.is_null() { std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap(); } + std::fs::write(dir.path().join("config.json"), model.to_string()).unwrap(); let path = path.to_str().unwrap(); prompt_affixes(path, &ModelFiles::open(path)) } @@ -536,20 +578,33 @@ mod prompt_affix_tests { let llama = json!({"tokenizer_class": "LlamaTokenizerFast", "bos_token": "<|endoftext|>"}); let mut no_bos = llama.clone(); no_bos["add_bos_token"] = false.into(); - for (post_processor, config, prefix) in [ - (Value::Null, json!({}), vec![]), - (template.clone(), json!({}), vec![256]), + let embedding_gemma = + json!({"model_type": "gemma3_text", "use_bidirectional_attention": true}); + let eos = json!({"eos_token": {"content": "<|endoftext|>"}}); + assert!(affixes(Value::Null, Value::Null, json!({}), embedding_gemma.clone()).is_err()); + for (post_processor, config, model, prefix, eos) in [ + (Value::Null, json!({}), json!({}), vec![], None), + (template.clone(), json!({}), json!({}), vec![256], None), // SGLang adds BOS by `add_bos_token`, ignoring the post-processor. - (Value::Null, llama, vec![256]), - (template, no_bos, vec![]), + (Value::Null, llama, json!({}), vec![256], None), + (template, no_bos, json!({}), vec![], None), + (Value::Null, eos, embedding_gemma, vec![], Some(256)), ] { - let affixes = affixes(Value::Null, post_processor, config).unwrap(); - assert_eq!(affixes, (prefix, vec![])); + let affixes = affixes(Value::Null, post_processor, config, model).unwrap(); + let suffix = vec![]; + assert_eq!( + affixes, + PromptAffixes { + prefix, + suffix, + eos + } + ); } // Without the config or its BOS, the engine's special tokens are unknown. let unknown_bos = json!({"tokenizer_class": "LlamaTokenizerFast"}); for config in [Value::Null, unknown_bos] { - assert!(affixes(Value::Null, Value::Null, config).is_err()); + assert!(affixes(Value::Null, Value::Null, config, json!({})).is_err()); } } @@ -570,10 +625,22 @@ mod prompt_affix_tests { ), ] { let config = json!({"tokenizer_class": class, "add_bos_token": false}); - let affixes = affixes(normalizer, Value::Null, config); + let affixes = affixes(normalizer, Value::Null, config, json!({})); assert_eq!(affixes.is_ok(), kept, "{class}"); } } + + #[test] + fn embedding_gemma_eos_ends_the_prompt_once() { + let affixes = PromptAffixes { + eos: Some(1), + ..Default::default() + }; + assert_eq!( + (affixes.apply(&[7]), affixes.apply(&[7, 1])), + (vec![7, 1], vec![7, 1]) + ); + } } #[cfg(test)] diff --git a/experimental/sgl-router/src/tokenizer/mod.rs b/experimental/sgl-router/src/tokenizer/mod.rs index 9e270c1456..64ca707ff0 100644 --- a/experimental/sgl-router/src/tokenizer/mod.rs +++ b/experimental/sgl-router/src/tokenizer/mod.rs @@ -65,7 +65,7 @@ pub struct TokenizerRegistry { /// Resolved encode backend and L1 cache counters of the served model's tokenizer. stats: Arc, /// Special tokens the engine adds around a raw prompt; `None` if unknown. - prompt_affixes: Option<(Vec, Vec)>, + prompt_affixes: Option, } impl std::fmt::Debug for TokenizerRegistry { @@ -94,7 +94,8 @@ impl TokenizerRegistry { me.prompt_affixes = adapter::prompt_affixes(tokenizer_path, &files) .map_err(|e| { tracing::warn!(model = %m.id, error = %format!("{e:#}"), - "cannot reproduce the engine's tokens; /generate forwards text") + "cannot reproduce the engine's tokens; /generate and /v1/embeddings \ + forward text") }) .ok(); match ChatFormatter::load_from(&m.id, &files) { @@ -147,7 +148,7 @@ impl TokenizerRegistry { /// Encode a raw prompt as the engine's `tokenizer(text)` does, special tokens included. pub fn encode_prompt(&self, model_id: &str, text: &str) -> Option> { - let (prefix, suffix) = self.prompt_affixes.as_ref()?; + let affixes = self.prompt_affixes.as_ref()?; let tokenizer = self.get(model_id)?; // The supported tiktoken models use Kimi's chunked encoding, also used for chat. let ids = if self.stats.backend() == stats::EncodeBackend::Tiktoken { @@ -156,7 +157,7 @@ impl TokenizerRegistry { adapter::encode(&tokenizer, text) } .ok()?; - (!ids.is_empty()).then(|| [prefix.as_slice(), &ids, suffix].concat()) + (!ids.is_empty()).then(|| affixes.apply(&ids)) } /// Whether this model has a chat formatter (and thus the chat-aware diff --git a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs index 9de384bfaf..1e829ba5a4 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs @@ -990,3 +990,45 @@ async fn full_decode_group_falls_back_to_another_version_group() { .collect(); assert_eq!(dispatched, [false, true, false, true]); } + +#[tokio::test] +async fn embeddings_fallback_batches_use_per_prompt_context_limits() { + let worker = MockWorker::start(vec![]).await; + let policy = Arc::new(FirstPolicy::default()); + let mut bucket = Bucket::new("small", BucketGroups::Plain(group("w", policy))); + bucket.max_context_tokens = Some(128); + let mut ctx = Arc::try_unwrap(context(&[("w", Stage::Plain, &worker)], vec![bucket])) + .unwrap_or_else(|_| panic!("context is shared")); + // A loadable tokenizer whose engine tokenization cannot be reproduced. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("tokenizer.json"); + std::fs::copy("tests/fixtures/tiny_tokenizer.json", &path).unwrap(); + std::fs::write( + dir.path().join("tokenizer_config.json"), + r#"{"tokenizer_class":"CodeLlamaTokenizerFast"}"#, + ) + .unwrap(); + ctx.config.model.tokenizer_path = Some(path.to_str().unwrap().into()); + ctx.tokenizers = Arc::new(TokenizerRegistry::load_from_config(&ctx.config).unwrap()); + assert!(ctx.tokenizers.encode_prompt("tiny", "hello").is_none()); + let app = build_router(Arc::new(ctx)); + for (input, expected) in [ + (serde_json::json!("hello"), StatusCode::OK), + (serde_json::json!(vec!["hello"; 128]), StatusCode::OK), + ( + serde_json::json!(["hello", "x".repeat(600)]), + StatusCode::BAD_REQUEST, + ), + ] { + let req = Request::post("/v1/embeddings") + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({"model":"tiny", "input":input}).to_string(), + )) + .unwrap(); + let response = app.clone().oneshot(req).await.unwrap(); + let status = response.status(); + let body = crate::common::streaming::collect_body(response.into_body()).await; + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&body)); + } +} diff --git a/experimental/sgl-router/tests/proxy/common/mock_worker.rs b/experimental/sgl-router/tests/proxy/common/mock_worker.rs index d86a5284fc..eb3a9d0dce 100644 --- a/experimental/sgl-router/tests/proxy/common/mock_worker.rs +++ b/experimental/sgl-router/tests/proxy/common/mock_worker.rs @@ -74,6 +74,7 @@ impl MockWorker { let app = axum::Router::new() .route("/v1/chat/completions", post(chat)) .route("/generate", post(generate)) + .route("/v1/embeddings", post(embeddings)) .route("/server_info", get(serve_tiny_server_info)) .route("/abort_request", abort_request_route(abort_log.clone())) .with_state(state); @@ -506,3 +507,15 @@ async fn generate( })) .into_response() } + +/// An OpenAI embeddings response. +#[allow(dead_code)] // Used by `MockWorker::start`, only some test files need it. +async fn embeddings( + State(s): State, + headers: HeaderMap, + body: Bytes, +) -> Response { + capture_request(&s, &headers, &body); + let data = [serde_json::json!({"object": "embedding", "embedding": [0.5], "index": 0})]; + Json(serde_json::json!({"object": "list", "data": data, "model": "tiny"})).into_response() +} diff --git a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs index ae75e819f7..70aa72a5e5 100644 --- a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs +++ b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs @@ -241,3 +241,24 @@ async fn pd_fan_out_leaves_the_prefill_rank_to_the_engine() { ); } } + +/// The engine's `/v1/embeddings` reads no rank, so the router pins none. +#[tokio::test] +async fn embeddings_leave_the_rank_to_the_engine() { + let worker = MockWorker::start(vec![]).await; + let app = router( + sticky_config(), + &[(&worker, WorkerMode::Plain, 4)], + Default::default(), + ); + let request = Request::post("/v1/embeddings") + .header("content-type", "application/json") + .header(KEY, "conv-a") + .body(Body::from( + json!({"model": MODEL, "input": "hi"}).to_string(), + )) + .unwrap(); + assert!(app.oneshot(request).await.unwrap().status().is_success()); + assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None); + assert!(worker.captured_json().await.get("routed_dp_rank").is_none()); +} diff --git a/experimental/sgl-router/tests/proxy/embeddings_routing.rs b/experimental/sgl-router/tests/proxy/embeddings_routing.rs new file mode 100644 index 0000000000..65f792bfbd --- /dev/null +++ b/experimental/sgl-router/tests/proxy/embeddings_routing.rs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! OpenAI `/v1/embeddings`: the engine's schema, with text input forwarded as router tokens. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use axum::response::Response; +use axum::Router; +use serde_json::{json, Value}; +use sgl_router::discovery::WorkerMode; +use sgl_router::state::kv_events::{compute_block_hashes, HashTree, KvWorkerId}; +use sgl_router::tokenizer::TokenizerRegistry; +use tower::ServiceExt; + +use crate::common::cache_aware_fixture::{config, radix_router, MODEL}; +use crate::common::mock_worker::MockWorker; +use crate::common::streaming::collect_body; + +async fn send(app: &Router, body: Value) -> Response { + let request = Request::post("/v1/embeddings") + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(); + app.clone().oneshot(request).await.unwrap() +} + +fn prompt_ids(text: &str) -> Vec { + let tokenizers = TokenizerRegistry::load_from_config(&config()).unwrap(); + tokenizers.encode_prompt(MODEL, text).unwrap() +} + +#[tokio::test] +async fn embeddings_forward_text_as_input_ids() { + let engine = MockWorker::start(vec![]).await; + let app = radix_router(&[(&engine, WorkerMode::Plain)], HashTree::new()); + + let res = send(&app, json!({"model": MODEL, "input": "hi"})).await; + let response: Value = serde_json::from_slice(&collect_body(res.into_body()).await).unwrap(); + assert_eq!(response["data"][0]["embedding"], json!([0.5])); + // A single prompt also gets an rid for abort-on-disconnect. + let forwarded = engine.captured_json().await; + assert_eq!(forwarded["input"], json!(prompt_ids("hi"))); + assert!(forwarded["rid"] + .as_str() + .is_some_and(crate::common::is_engine_shaped_rid)); + + // A batch needs a list of rids, so none is minted. + send(&app, json!({"model": MODEL, "input": ["hi", "yo"]})).await; + let expected = json!({"model": MODEL, "input": [prompt_ids("hi"), prompt_ids("yo")]}); + assert_eq!(engine.captured_json().await, expected); +} + +#[tokio::test] +async fn embeddings_route_text_to_the_worker_caching_its_prefix() { + let (cached, uncached) = ( + MockWorker::start(vec![]).await, + MockWorker::start(vec![]).await, + ); + let tree = HashTree::new(); + let hashes = compute_block_hashes(&prompt_ids("local radix cache hit"), 1); + tree.insert(&KvWorkerId::new(cached.url.clone(), 0), None, &hashes); + let app = radix_router( + &[(&uncached, WorkerMode::Plain), (&cached, WorkerMode::Plain)], + tree, + ); + + // Without tokens the pick is a load tie broken at random, so one request proves nothing. + for _ in 0..16 { + let res = send( + &app, + json!({"model": MODEL, "input": "local radix cache hit"}), + ) + .await; + assert_eq!(res.status(), StatusCode::OK); + } + assert!(uncached.captured.lock().unwrap().last_body.is_none()); +} + +#[tokio::test] +async fn embeddings_reject_prefill_decode_workers_and_other_models() { + let (prefill, decode) = ( + MockWorker::start(vec![]).await, + MockWorker::start(vec![]).await, + ); + let workers = [ + (&prefill, WorkerMode::Prefill), + (&decode, WorkerMode::Decode), + ]; + let app = radix_router(&workers, HashTree::new()); + + let res = send(&app, json!({"model": MODEL, "input": "hi"})).await; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let res = send(&app, json!({"model": "other", "input": "hi"})).await; + assert_eq!(res.status(), StatusCode::NOT_FOUND); + assert!(prefill.captured.lock().unwrap().last_body.is_none()); +} diff --git a/experimental/sgl-router/tests/proxy/main.rs b/experimental/sgl-router/tests/proxy/main.rs index 34f9592217..42ad6aa735 100644 --- a/experimental/sgl-router/tests/proxy/main.rs +++ b/experimental/sgl-router/tests/proxy/main.rs @@ -14,6 +14,7 @@ mod bucket_routing; mod cache_aware_input_ids; mod chat_routing; mod dp_rank_routing; +mod embeddings_routing; mod external_indexer_routing; mod failover; mod generate_routing;