[sgl-router] Add OpenAI /v1/embeddings (#42191)

Co-authored-by: Kan Wu <kan.wu@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-10-02 20:18:24 +08:00
committed by GitHub
co-authored by Kan Wu Claude Opus 5.5
parent 1330e437e1
commit edee4308bc
12 changed files with 510 additions and 45 deletions
+16 -3
View File
@@ -3,9 +3,9 @@
Slim, KV-aware, OpenAI-compatible router for SGLang workers.
Serves a single model and routes across its workers. Exposes
`/v1/tokenize`, `/v1/detokenize`, `/v1/models`, `/v1/chat/completions` and
SGLang's native [`/generate`](#native-generate) (buffered and SSE), plus
`/healthz` / `/readyz` and `/metrics`. Worker
`/v1/tokenize`, `/v1/detokenize`, `/v1/models`, [`/v1/embeddings`](#embeddings),
`/v1/chat/completions` and SGLang's native [`/generate`](#native-generate)
(buffered and SSE), plus `/healthz` / `/readyz` and `/metrics`. Worker
pools come from either a static URL list or Kubernetes EndpointSlice
discovery. Both edges speak cleartext HTTP/2 where the peer does — see
[HTTP/2](#http2).
@@ -379,6 +379,19 @@ a single prompt that has none, and `--override-sampling-params` defaults. Under
batch or `n > 1` request leaves the prefill rank to the engine, which gives item
`i` the bootstrap room `room + i`.
## Embeddings
`/v1/embeddings` has the engine's interface: the same OpenAI `EmbeddingRequest`
body and response. As for chat completions, `model` must name the served model.
A PD fleet answers 400, since prefill and decode engines serve no embeddings.
Text `input` (a string or a list) is tokenized and forwarded as token IDs, as for
[`/generate`](#native-generate), plus the EOS SGLang appends for EmbeddingGemma.
Blank prompts and multimodal items stay as sent, for the engine to reject or
render. A list is a batch for one worker, routed on load. A single prompt with no
`rid` gets a minted one. Under `--dp-aware` the engine picks the rank, since its
embeddings endpoint reads none.
## DeepSeek V4
Native V4 rendering follows SGLang's serving path (`serving_chat.py`), not
@@ -212,6 +212,12 @@ pub fn build_router(ctx: Arc<AppContext>) -> Router {
.layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES))
.layer(middleware::from_fn(log_413)),
)
.route(
"/v1/embeddings",
post(crate::server::routes::chat::embeddings)
.layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES))
.layer(middleware::from_fn(log_413)),
)
.route(
"/flush_cache",
post(crate::server::routes::cache::flush_cache),
@@ -24,7 +24,7 @@ use axum::extract::State;
use axum::http::{HeaderMap, HeaderName, Response};
use bytes::Bytes;
use forward::{forward_request, SelectedWorkers};
use preparation::{parse_routing_fields, PreparedRequest};
use preparation::{parse_embedding_request, parse_routing_fields, PreparedRequest};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Instant;
@@ -77,6 +77,27 @@ pub async fn generate(
forward_request(&ctx, request, workers, headers, start).await
}
/// OpenAI `/v1/embeddings`, forwarded to the engine's with the same request and response.
pub async fn embeddings(
State(ctx): State<Arc<AppContext>>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response<Body>, ApiError> {
let start = Instant::now();
let (model, value) = parse_embedding_request(&body)?;
let routing = ModelRouting::lookup(&ctx, &model)?;
// Prefill and decode engines serve generation; embeddings need plain workers.
let registered = ctx.registry.workers_for(&model);
if registered.iter().any(|w| w.mode() != WorkerMode::Plain) {
return Err(ApiError::BadRequest(
"embeddings are not served by prefill-decode workers".into(),
));
}
let request = PreparedRequest::embeddings(&ctx, model, body, value)?;
let workers = routing.select_workers(&ctx, &request, &headers).await?;
forward_request(&ctx, request, workers, headers, start).await
}
/// A model's routing state, resolved before the request is prepared.
enum ModelRouting<'a> {
Legacy(Arc<dyn Policy>),
@@ -61,7 +61,7 @@ pub(super) async fn forward_request(
}
// Only the router chooses DP ranks; a client-supplied rank is never forwarded.
headers.remove(X_DATA_PARALLEL_RANK);
let dp_aware = ctx.config.model.dp_aware;
let dp_aware = ctx.config.model.dp_aware && request.accepts_dp_rank();
// The engine gives fan-out item i the room `room + i`, so in PD decode looks
// for each item on a different prefill rank; one pinned rank would break that.
let unpin_prefill = dp_aware && decode.is_some() && request.fans_out;
@@ -2,7 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
//! Request validation, optional tokenization, and outgoing body preparation for
//! chat completions and the native `/generate` endpoint.
//! chat completions, the native `/generate` endpoint, and embeddings.
use crate::config::{ConflictPolicy, ParamSpec, SamplingField, SamplingOverrides};
use crate::discovery::ModelId;
@@ -21,6 +21,7 @@ const BYTES_PER_TOKEN_ESTIMATE: usize = 4;
const CHAT_PATH: &str = "/v1/chat/completions";
const GENERATE_PATH: &str = "/generate";
const EMBEDDINGS_PATH: &str = "/v1/embeddings";
/// Validated routing inputs and the original body, ready for worker selection.
///
@@ -43,7 +44,7 @@ pub(super) struct PreparedRequest {
pub(super) expected_peak_sequence_tokens: Option<u64>,
caller_set_rid: bool,
pub(super) fans_out: bool,
/// `None` for `/generate`, whose tokens replace `text` during preparation.
/// `None` for `/generate` and embeddings, whose tokens replace text during preparation.
forwarding_scope: Option<ForwardingScope>,
parsed_body: Option<Value>,
sampling_defaults: Vec<(SamplingField, Number)>,
@@ -175,6 +176,66 @@ impl PreparedRequest {
})
}
/// The engine's OpenAI `EmbeddingRequest`, whose text `input` is forwarded
/// as token IDs, as `/generate` forwards `text`.
pub(super) fn embeddings(
ctx: &AppContext,
model: ModelId,
mut body: Bytes,
mut value: Value,
) -> Result<Self, ApiError> {
let batch = is_embedding_batch(&value["input"]);
let text_ids = tokenize_input_text(ctx, &model, &value["input"]);
let forward = !ctx.config.model.disable_input_ids_forwarding;
if let Some(ids) = text_ids.as_ref().filter(|_| forward) {
value["input"] = if batch { json!(ids) } else { json!(ids[0]) };
body = serde_json::to_vec(&value)
.map_err(|error| ApiError::Internal(error.into()))?
.into();
}
let prompts = text_ids.or_else(|| caller_token_rows(&value["input"]));
// Context limits apply to each prompt even when the engine must tokenize text;
// multimodal items count only their text, as image tokens are the engine's to count.
let estimate = |item: &Value| {
estimate_prefill_tokens(item.as_str().or(item["text"].as_str()).map_or(0, str::len))
};
let lengths = match (&prompts, &value["input"]) {
(Some(prompts), _) => prompts.iter().map(|ids| ids.len().max(1)).collect(),
(None, Value::Array(items)) => items.iter().map(estimate).collect(),
(None, input) => vec![estimate(input)],
};
let sequence_tokens = lengths.iter().copied().max().unwrap_or(1);
let tokens = prompts
.filter(|_| !batch)
.and_then(|prompts| prompts.into_iter().next())
.map(|ids| RequestTokens {
ids,
rendered_from_chat: false,
});
Ok(Self {
path: EMBEDDINGS_PATH,
model,
streaming: false,
output_tokens: Some(0),
input_token_count: lengths.iter().sum::<usize>().max(1),
sequence_token_count: sequence_tokens,
expected_peak_sequence_tokens: Some(sequence_tokens as u64),
body,
tokens,
caller_set_rid: !value["rid"].is_null(),
fans_out: batch,
forwarding_scope: None,
// Only PD bootstrap needs the parsed body, and embeddings have no PD.
parsed_body: None,
sampling_defaults: Vec::new(),
})
}
/// Whether the engine endpoint honors a routed DP rank; `/v1/embeddings` takes none.
pub(super) fn accepts_dp_rank(&self) -> bool {
self.path != EMBEDDINGS_PATH
}
pub(super) fn engine_rid(&self) -> Option<String> {
// Caller IDs are unsafe for prefix aborts; fan-out regenerates IDs.
// PD shares the minted ID, but only decode arms abort-on-drop.
@@ -310,6 +371,60 @@ fn parallel_samples(value: &Value) -> u64 {
}
}
/// An embeddings body and the model it names; chat completions also require one.
pub(super) fn parse_embedding_request(body: &Bytes) -> Result<(ModelId, Value), ApiError> {
let value = Value::Object(serde_json::from_slice(body).map_err(|_| invalid_request())?);
let model = Option::<String>::deserialize(&value["model"])
.map_err(|_| invalid_request())?
.ok_or_else(|| ApiError::BadRequest("missing `model` field".into()))?;
Ok((ModelId(model), value))
}
/// Whether the engine runs an embedding `input` as a batch: any list but one
/// of token IDs or a single multimodal item (`serving_embedding.py`).
fn is_embedding_batch(input: &Value) -> bool {
match input.as_array().map(Vec::as_slice) {
Some([first, ..]) if first.is_number() => false,
Some([only]) if only.is_object() => false,
Some(items) => !items.is_empty(),
None => false,
}
}
/// Text `input` as the engine's tokens, one row per prompt; `None` for other
/// input and for text the engine rejects: mixed lists and blank prompts.
fn tokenize_input_text(ctx: &AppContext, model: &ModelId, input: &Value) -> Option<Vec<Vec<u32>>> {
let texts = match input {
Value::String(text) => vec![text.as_str()],
Value::Array(items) if items.first()?.is_string() => {
items.iter().map(Value::as_str).collect::<Option<_>>()?
}
_ => return None,
};
// Python's `str.strip()` also strips U+001C..U+001F.
let blank = |text: &&str| {
text.trim_matches(|c: char| c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c))
.is_empty()
};
if texts.iter().any(blank) {
return None;
}
texts
.into_iter()
.map(|text| ctx.tokenizers.encode_prompt(&model.0, text))
.collect()
}
/// Caller token IDs, one flat list or a row per prompt; `None` if malformed,
/// which the engine reports.
fn caller_token_rows(input: &Value) -> Option<Vec<Vec<u32>>> {
match input.as_array()?.first()? {
Value::Number(_) => Some(vec![Vec::deserialize(input).ok()?]),
Value::Array(_) => Vec::deserialize(input).ok(),
_ => None,
}
}
/// Routing and sampling fields retained by the lightweight request parser.
#[derive(Debug, Default)]
pub(super) struct RoutingFields {
@@ -985,9 +1100,8 @@ mod tests {
}
}
/// A `/generate` request prepared with the tiny test tokenizer loaded.
fn prepare_generate(body: Value) -> PreparedRequest {
// The tiny tokenizer, with a post-processor that prepends BOS (id 256).
/// The stub context with the tiny tokenizer, whose post-processor prepends BOS (id 256).
fn tokenizer_ctx() -> AppContext {
let tokenizer = std::fs::read_to_string("tests/fixtures/tiny_tokenizer.json").unwrap();
let mut tokenizer: Value = serde_json::from_str(&tokenizer).unwrap();
tokenizer["post_processor"] = json!({"type": "TemplateProcessing", "pair": [], "single": [
@@ -1001,8 +1115,77 @@ mod tests {
let mut ctx = AppContext::stub();
ctx.config.model.tokenizer_path = Some(path.to_str().unwrap().into());
ctx.tokenizers = Arc::new(TokenizerRegistry::load_from_config(&ctx.config).unwrap());
ctx
}
/// A `/generate` request prepared with the tiny test tokenizer loaded.
fn prepare_generate(body: Value) -> PreparedRequest {
let body = Bytes::from(body.to_string());
PreparedRequest::generate(&ctx, ModelId("stub-model".into()), body).unwrap()
PreparedRequest::generate(&tokenizer_ctx(), ModelId("stub-model".into()), body).unwrap()
}
fn prepare_embeddings(ctx: &AppContext, input: Value) -> (PreparedRequest, Value) {
let body = Bytes::from(json!({"model": "stub-model", "input": input}).to_string());
let (model, value) = parse_embedding_request(&body).unwrap();
let r = PreparedRequest::embeddings(ctx, model, body, value).unwrap();
let input = serde_json::from_slice::<Value>(&r.body).unwrap()["input"].take();
(r, input)
}
#[test]
fn embeddings_forward_text_input_as_token_ids() {
let ctx = tokenizer_ctx();
let hi = ctx.tokenizers.encode_prompt("stub-model", "hi").unwrap();
let n = hi.len();
// (input, forwarded input, fans out, (input total, longest prompt))
for (sent, forwarded, fans_out, counts) in [
(json!("hi"), json!(hi), false, (n, n)),
// The engine runs any text list as a batch, even of one.
(json!(["hi"]), json!([hi]), true, (n, n)),
(json!([[5, 6], [7]]), json!([[5, 6], [7]]), true, (3, 2)),
] {
let (r, input) = prepare_embeddings(&ctx, sent);
let got = (r.input_token_count, r.sequence_token_count);
assert_eq!((input, r.fans_out, got), (forwarded, fans_out, counts));
assert_eq!(r.tokens.is_some(), !fans_out);
}
}
#[test]
fn embeddings_estimate_fallback_text_per_prompt() {
let ctx = AppContext::stub();
for (input, total, longest) in [
(json!("abcdefgh"), 2, 2),
(json!(["abcd", "abcdefgh"]), 3, 2),
// An image's bytes are not prompt text.
(json!([{"text": "abcd", "image": "x".repeat(4096)}]), 1, 1),
] {
let (r, forwarded) = prepare_embeddings(&ctx, input.clone());
assert_eq!(forwarded, input);
assert!(r.tokens.is_none());
assert_eq!(r.input_token_count, total);
assert_eq!(r.sequence_token_count, longest);
assert_eq!(r.expected_peak_sequence_tokens, Some(longest as u64));
}
}
#[test]
fn embeddings_leave_other_input_to_the_engine() {
let mut ctx = tokenizer_ctx();
// The engine rejects blank and mixed lists, and renders multimodal items.
for sent in [
json!(["hi", " \u{1c}"]),
json!(["hi", 5]),
json!([{"text": "hi"}]),
] {
let (r, input) = prepare_embeddings(&ctx, sent.clone());
assert!(input == sent && r.tokens.is_none());
}
// Without forwarding, routing still sees the engine's tokens.
ctx.config.model.disable_input_ids_forwarding = true;
let (r, input) = prepare_embeddings(&ctx, json!("hi"));
let ids = ctx.tokenizers.encode_prompt("stub-model", "hi");
assert_eq!((input, r.tokens.map(|t| t.ids)), (json!("hi"), ids));
}
#[test]
@@ -73,11 +73,30 @@ fn engine_keeps_normalizer(class: &str, normalizer: &serde_json::Value) -> bool
}
}
/// Token ids SGLang's `tokenizer(text)` puts around a prompt and [`encode`]
/// leaves out: BOS per `add_bos_token` (default true) for [`BOS_FLAG_CLASSES`],
/// else the tokenizer.json post-processor's. Tiktoken models add none. Errs when
/// the router cannot reproduce the engine's tokens.
pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result<(Vec<u32>, Vec<u32>)> {
/// Special tokens SGLang's `tokenizer(text)` puts around a prompt and [`encode`] leaves out.
#[derive(Debug, Default, PartialEq)]
pub struct PromptAffixes {
pub prefix: Vec<u32>,
pub suffix: Vec<u32>,
/// EOS that SGLang appends to an EmbeddingGemma prompt not already ending in it.
pub eos: Option<u32>,
}
impl PromptAffixes {
/// [`encode`]'s `ids` as the engine tokenizes the same text.
pub fn apply(&self, ids: &[u32]) -> Vec<u32> {
let mut ids = [self.prefix.as_slice(), ids, &self.suffix].concat();
if let Some(eos) = self.eos.filter(|&eos| ids.last() != Some(&eos)) {
ids.push(eos);
}
ids
}
}
/// BOS per `add_bos_token` (default true) for [`BOS_FLAG_CLASSES`], else the
/// tokenizer.json post-processor's, and EmbeddingGemma's EOS. Tiktoken models add
/// none. Errs when the router cannot reproduce the engine's tokens.
pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result<PromptAffixes> {
let path = resolve(source)?;
if !path.ends_with(".json") {
return Ok(Default::default());
@@ -98,19 +117,34 @@ pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result<(Vec<u32>, Vec
let ids = |t: &dyn traits::Tokenizer, text: &str| -> Result<Vec<u32>> {
Ok(t.encode(text)?.token_ids().to_vec())
};
if BOS_FLAG_CLASSES.contains(&class) {
let bos = &config["bos_token"];
let bos = bos.as_str().or(bos["content"].as_str());
let prefix = if config["add_bos_token"].as_bool().unwrap_or(true) {
let bos = bos.context("add_bos_token is set without a bos_token")?;
match ids(plain.as_ref(), bos)?.as_slice() {
[id] => vec![*id],
_ => anyhow::bail!("bos_token {bos:?} is not a single token"),
}
} else {
Vec::new()
let special = |key: &str| -> Result<Option<u32>> {
let Some(token) = config[key].as_str().or(config[key]["content"].as_str()) else {
return Ok(None);
};
return Ok((prefix, Vec::new()));
match ids(plain.as_ref(), token)?.as_slice() {
[id] => Ok(Some(*id)),
_ => anyhow::bail!("{key} {token:?} is not a single token"),
}
};
// SGLang's `is_embedding_gemma`, whose `_tokenize_texts` ends each prompt with EOS.
files.ensure_downloaded("config.json")?;
let model = files.json("config.json")?.unwrap_or_default();
let embedding_gemma =
model["model_type"] == "gemma3_text" && model["use_bidirectional_attention"] == true;
let eos = embedding_gemma
.then(|| special("eos_token")?.context("EmbeddingGemma declares no eos_token"))
.transpose()?;
if BOS_FLAG_CLASSES.contains(&class) {
let add_bos = config["add_bos_token"].as_bool().unwrap_or(true);
let bos = add_bos
.then(|| special("bos_token")?.context("add_bos_token is set without a bos_token"))
.transpose()?;
let prefix = bos.into_iter().collect();
return Ok(PromptAffixes {
prefix,
suffix: Vec::new(),
eos,
});
}
let options = TokenizerOptions {
add_special_tokens: true,
@@ -125,7 +159,12 @@ pub fn prompt_affixes(source: &str, files: &ModelFiles) -> Result<(Vec<u32>, Vec
.windows(bare.len())
.position(|window| window == bare)
.context("the post-processor rewrites the prompt")?;
Ok((full[..start].to_vec(), full[start + bare.len()..].to_vec()))
let (prefix, suffix) = (full[..start].to_vec(), full[start + bare.len()..].to_vec());
Ok(PromptAffixes {
prefix,
suffix,
eos,
})
}
fn build(path: &str, cfg: TokenizerConfig) -> Result<(Arc<Tokenizer>, Arc<TokenizerStats>)> {
@@ -503,16 +542,18 @@ mod model_files_tests {
#[cfg(test)]
mod prompt_affix_tests {
use super::{prompt_affixes, ModelFiles};
use super::{prompt_affixes, ModelFiles, PromptAffixes};
use anyhow::Result;
use serde_json::{json, Value};
/// `prompt_affixes` of the tiny tokenizer with `normalizer`, next to `config` unless null.
/// `prompt_affixes` of the tiny tokenizer with `normalizer`, next to `config`
/// (unless null) and the `model` config.
fn affixes(
normalizer: Value,
post_processor: Value,
config: Value,
) -> Result<(Vec<u32>, Vec<u32>)> {
model: Value,
) -> Result<PromptAffixes> {
let mut data: Value =
serde_json::from_str(include_str!("../../tests/fixtures/tiny_tokenizer.json")).unwrap();
data["normalizer"] = normalizer;
@@ -523,6 +564,7 @@ mod prompt_affix_tests {
if !config.is_null() {
std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap();
}
std::fs::write(dir.path().join("config.json"), model.to_string()).unwrap();
let path = path.to_str().unwrap();
prompt_affixes(path, &ModelFiles::open(path))
}
@@ -536,20 +578,33 @@ mod prompt_affix_tests {
let llama = json!({"tokenizer_class": "LlamaTokenizerFast", "bos_token": "<|endoftext|>"});
let mut no_bos = llama.clone();
no_bos["add_bos_token"] = false.into();
for (post_processor, config, prefix) in [
(Value::Null, json!({}), vec![]),
(template.clone(), json!({}), vec![256]),
let embedding_gemma =
json!({"model_type": "gemma3_text", "use_bidirectional_attention": true});
let eos = json!({"eos_token": {"content": "<|endoftext|>"}});
assert!(affixes(Value::Null, Value::Null, json!({}), embedding_gemma.clone()).is_err());
for (post_processor, config, model, prefix, eos) in [
(Value::Null, json!({}), json!({}), vec![], None),
(template.clone(), json!({}), json!({}), vec![256], None),
// SGLang adds BOS by `add_bos_token`, ignoring the post-processor.
(Value::Null, llama, vec![256]),
(template, no_bos, vec![]),
(Value::Null, llama, json!({}), vec![256], None),
(template, no_bos, json!({}), vec![], None),
(Value::Null, eos, embedding_gemma, vec![], Some(256)),
] {
let affixes = affixes(Value::Null, post_processor, config).unwrap();
assert_eq!(affixes, (prefix, vec![]));
let affixes = affixes(Value::Null, post_processor, config, model).unwrap();
let suffix = vec![];
assert_eq!(
affixes,
PromptAffixes {
prefix,
suffix,
eos
}
);
}
// Without the config or its BOS, the engine's special tokens are unknown.
let unknown_bos = json!({"tokenizer_class": "LlamaTokenizerFast"});
for config in [Value::Null, unknown_bos] {
assert!(affixes(Value::Null, Value::Null, config).is_err());
assert!(affixes(Value::Null, Value::Null, config, json!({})).is_err());
}
}
@@ -570,10 +625,22 @@ mod prompt_affix_tests {
),
] {
let config = json!({"tokenizer_class": class, "add_bos_token": false});
let affixes = affixes(normalizer, Value::Null, config);
let affixes = affixes(normalizer, Value::Null, config, json!({}));
assert_eq!(affixes.is_ok(), kept, "{class}");
}
}
#[test]
fn embedding_gemma_eos_ends_the_prompt_once() {
let affixes = PromptAffixes {
eos: Some(1),
..Default::default()
};
assert_eq!(
(affixes.apply(&[7]), affixes.apply(&[7, 1])),
(vec![7, 1], vec![7, 1])
);
}
}
#[cfg(test)]
+5 -4
View File
@@ -65,7 +65,7 @@ pub struct TokenizerRegistry {
/// Resolved encode backend and L1 cache counters of the served model's tokenizer.
stats: Arc<stats::TokenizerStats>,
/// Special tokens the engine adds around a raw prompt; `None` if unknown.
prompt_affixes: Option<(Vec<u32>, Vec<u32>)>,
prompt_affixes: Option<adapter::PromptAffixes>,
}
impl std::fmt::Debug for TokenizerRegistry {
@@ -94,7 +94,8 @@ impl TokenizerRegistry {
me.prompt_affixes = adapter::prompt_affixes(tokenizer_path, &files)
.map_err(|e| {
tracing::warn!(model = %m.id, error = %format!("{e:#}"),
"cannot reproduce the engine's tokens; /generate forwards text")
"cannot reproduce the engine's tokens; /generate and /v1/embeddings \
forward text")
})
.ok();
match ChatFormatter::load_from(&m.id, &files) {
@@ -147,7 +148,7 @@ impl TokenizerRegistry {
/// Encode a raw prompt as the engine's `tokenizer(text)` does, special tokens included.
pub fn encode_prompt(&self, model_id: &str, text: &str) -> Option<Vec<u32>> {
let (prefix, suffix) = self.prompt_affixes.as_ref()?;
let affixes = self.prompt_affixes.as_ref()?;
let tokenizer = self.get(model_id)?;
// The supported tiktoken models use Kimi's chunked encoding, also used for chat.
let ids = if self.stats.backend() == stats::EncodeBackend::Tiktoken {
@@ -156,7 +157,7 @@ impl TokenizerRegistry {
adapter::encode(&tokenizer, text)
}
.ok()?;
(!ids.is_empty()).then(|| [prefix.as_slice(), &ids, suffix].concat())
(!ids.is_empty()).then(|| affixes.apply(&ids))
}
/// Whether this model has a chat formatter (and thus the chat-aware
@@ -990,3 +990,45 @@ async fn full_decode_group_falls_back_to_another_version_group() {
.collect();
assert_eq!(dispatched, [false, true, false, true]);
}
#[tokio::test]
async fn embeddings_fallback_batches_use_per_prompt_context_limits() {
let worker = MockWorker::start(vec![]).await;
let policy = Arc::new(FirstPolicy::default());
let mut bucket = Bucket::new("small", BucketGroups::Plain(group("w", policy)));
bucket.max_context_tokens = Some(128);
let mut ctx = Arc::try_unwrap(context(&[("w", Stage::Plain, &worker)], vec![bucket]))
.unwrap_or_else(|_| panic!("context is shared"));
// A loadable tokenizer whose engine tokenization cannot be reproduced.
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tokenizer.json");
std::fs::copy("tests/fixtures/tiny_tokenizer.json", &path).unwrap();
std::fs::write(
dir.path().join("tokenizer_config.json"),
r#"{"tokenizer_class":"CodeLlamaTokenizerFast"}"#,
)
.unwrap();
ctx.config.model.tokenizer_path = Some(path.to_str().unwrap().into());
ctx.tokenizers = Arc::new(TokenizerRegistry::load_from_config(&ctx.config).unwrap());
assert!(ctx.tokenizers.encode_prompt("tiny", "hello").is_none());
let app = build_router(Arc::new(ctx));
for (input, expected) in [
(serde_json::json!("hello"), StatusCode::OK),
(serde_json::json!(vec!["hello"; 128]), StatusCode::OK),
(
serde_json::json!(["hello", "x".repeat(600)]),
StatusCode::BAD_REQUEST,
),
] {
let req = Request::post("/v1/embeddings")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({"model":"tiny", "input":input}).to_string(),
))
.unwrap();
let response = app.clone().oneshot(req).await.unwrap();
let status = response.status();
let body = crate::common::streaming::collect_body(response.into_body()).await;
assert_eq!(status, expected, "{}", String::from_utf8_lossy(&body));
}
}
@@ -74,6 +74,7 @@ impl MockWorker {
let app = axum::Router::new()
.route("/v1/chat/completions", post(chat))
.route("/generate", post(generate))
.route("/v1/embeddings", post(embeddings))
.route("/server_info", get(serve_tiny_server_info))
.route("/abort_request", abort_request_route(abort_log.clone()))
.with_state(state);
@@ -506,3 +507,15 @@ async fn generate(
}))
.into_response()
}
/// An OpenAI embeddings response.
#[allow(dead_code)] // Used by `MockWorker::start`, only some test files need it.
async fn embeddings(
State(s): State<MockWorkerState>,
headers: HeaderMap,
body: Bytes,
) -> Response<Body> {
capture_request(&s, &headers, &body);
let data = [serde_json::json!({"object": "embedding", "embedding": [0.5], "index": 0})];
Json(serde_json::json!({"object": "list", "data": data, "model": "tiny"})).into_response()
}
@@ -241,3 +241,24 @@ async fn pd_fan_out_leaves_the_prefill_rank_to_the_engine() {
);
}
}
/// The engine's `/v1/embeddings` reads no rank, so the router pins none.
#[tokio::test]
async fn embeddings_leave_the_rank_to_the_engine() {
let worker = MockWorker::start(vec![]).await;
let app = router(
sticky_config(),
&[(&worker, WorkerMode::Plain, 4)],
Default::default(),
);
let request = Request::post("/v1/embeddings")
.header("content-type", "application/json")
.header(KEY, "conv-a")
.body(Body::from(
json!({"model": MODEL, "input": "hi"}).to_string(),
))
.unwrap();
assert!(app.oneshot(request).await.unwrap().status().is_success());
assert_eq!(worker.captured.lock().unwrap().headers.get(RANK), None);
assert!(worker.captured_json().await.get("routed_dp_rank").is_none());
}
@@ -0,0 +1,97 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
//! OpenAI `/v1/embeddings`: the engine's schema, with text input forwarded as router tokens.
use axum::body::Body;
use axum::http::{Request, StatusCode};
use axum::response::Response;
use axum::Router;
use serde_json::{json, Value};
use sgl_router::discovery::WorkerMode;
use sgl_router::state::kv_events::{compute_block_hashes, HashTree, KvWorkerId};
use sgl_router::tokenizer::TokenizerRegistry;
use tower::ServiceExt;
use crate::common::cache_aware_fixture::{config, radix_router, MODEL};
use crate::common::mock_worker::MockWorker;
use crate::common::streaming::collect_body;
async fn send(app: &Router, body: Value) -> Response {
let request = Request::post("/v1/embeddings")
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap();
app.clone().oneshot(request).await.unwrap()
}
fn prompt_ids(text: &str) -> Vec<u32> {
let tokenizers = TokenizerRegistry::load_from_config(&config()).unwrap();
tokenizers.encode_prompt(MODEL, text).unwrap()
}
#[tokio::test]
async fn embeddings_forward_text_as_input_ids() {
let engine = MockWorker::start(vec![]).await;
let app = radix_router(&[(&engine, WorkerMode::Plain)], HashTree::new());
let res = send(&app, json!({"model": MODEL, "input": "hi"})).await;
let response: Value = serde_json::from_slice(&collect_body(res.into_body()).await).unwrap();
assert_eq!(response["data"][0]["embedding"], json!([0.5]));
// A single prompt also gets an rid for abort-on-disconnect.
let forwarded = engine.captured_json().await;
assert_eq!(forwarded["input"], json!(prompt_ids("hi")));
assert!(forwarded["rid"]
.as_str()
.is_some_and(crate::common::is_engine_shaped_rid));
// A batch needs a list of rids, so none is minted.
send(&app, json!({"model": MODEL, "input": ["hi", "yo"]})).await;
let expected = json!({"model": MODEL, "input": [prompt_ids("hi"), prompt_ids("yo")]});
assert_eq!(engine.captured_json().await, expected);
}
#[tokio::test]
async fn embeddings_route_text_to_the_worker_caching_its_prefix() {
let (cached, uncached) = (
MockWorker::start(vec![]).await,
MockWorker::start(vec![]).await,
);
let tree = HashTree::new();
let hashes = compute_block_hashes(&prompt_ids("local radix cache hit"), 1);
tree.insert(&KvWorkerId::new(cached.url.clone(), 0), None, &hashes);
let app = radix_router(
&[(&uncached, WorkerMode::Plain), (&cached, WorkerMode::Plain)],
tree,
);
// Without tokens the pick is a load tie broken at random, so one request proves nothing.
for _ in 0..16 {
let res = send(
&app,
json!({"model": MODEL, "input": "local radix cache hit"}),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
}
assert!(uncached.captured.lock().unwrap().last_body.is_none());
}
#[tokio::test]
async fn embeddings_reject_prefill_decode_workers_and_other_models() {
let (prefill, decode) = (
MockWorker::start(vec![]).await,
MockWorker::start(vec![]).await,
);
let workers = [
(&prefill, WorkerMode::Prefill),
(&decode, WorkerMode::Decode),
];
let app = radix_router(&workers, HashTree::new());
let res = send(&app, json!({"model": MODEL, "input": "hi"})).await;
assert_eq!(res.status(), StatusCode::BAD_REQUEST);
let res = send(&app, json!({"model": "other", "input": "hi"})).await;
assert_eq!(res.status(), StatusCode::NOT_FOUND);
assert!(prefill.captured.lock().unwrap().last_body.is_none());
}
@@ -14,6 +14,7 @@ mod bucket_routing;
mod cache_aware_input_ids;
mod chat_routing;
mod dp_rank_routing;
mod embeddings_routing;
mod external_indexer_routing;
mod failover;
mod generate_routing;