[Rust] Implement the Rust gRPC adapter for native generation (#41766)
Signed-off-by: jain-ria <riajain@NVIDIA.com>
This commit is contained in:
Generated
+2
@@ -3809,10 +3809,12 @@ dependencies = [
|
||||
"rustix",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sglang-grpc-types",
|
||||
"sglang-mm",
|
||||
"socket2 0.6.5",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tower 0.5.3",
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
|
||||
@@ -10,6 +10,9 @@ license.workspace = true
|
||||
python-module = "sglang.srt.rust_extensions._server"
|
||||
# Always build optimized, even for an editable install.
|
||||
debug = false
|
||||
# The shared runtime.v1 crate lives outside rust/; include it in extension
|
||||
# source fingerprints so protobuf changes cannot reuse a stale _server build.
|
||||
source-inputs = ["../../proto"]
|
||||
|
||||
[lib]
|
||||
name = "sglang_server"
|
||||
@@ -60,10 +63,12 @@ numpy = "0.29.0"
|
||||
regex-syntax = "=0.8.11"
|
||||
rmp-serde = "1"
|
||||
rmpv = { version = "1", features = ["with-serde"] }
|
||||
sglang-grpc-types = { path = "../../proto" }
|
||||
# The pure-Rust core of the MM pipeline. `default-features = false` drops the
|
||||
# pyo3 bindings so it links as a plain rlib; renamed so `use sglang_mm::…` reads
|
||||
# naturally while the crate keeps its own artifact name in the shared target/.
|
||||
sglang_mm = { package = "sglang-mm", path = "../sglang-mm", default-features = false }
|
||||
tonic = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# `Router::oneshot` for handler-level router tests.
|
||||
|
||||
@@ -5,78 +5,13 @@
|
||||
//! and streams; it calls these per frame.
|
||||
|
||||
use crate::frontend::FrontendOutput;
|
||||
#[cfg(test)]
|
||||
use crate::message::response::ChunkExtras;
|
||||
|
||||
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
|
||||
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
|
||||
fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value {
|
||||
texts
|
||||
.and_then(|t| t.get(j))
|
||||
.map(|s| serde_json::json!(s))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
|
||||
/// A decoded-text column becomes the tuples' text source only when populated
|
||||
/// (`return_text_in_logprobs`); empty → `None` → null text slots.
|
||||
fn opt_texts(t: &[String]) -> Option<&[String]> {
|
||||
(!t.is_empty()).then_some(t)
|
||||
}
|
||||
|
||||
/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel.
|
||||
fn lp_value(v: f32) -> serde_json::Value {
|
||||
if v.is_nan() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
serde_json::json!(v)
|
||||
}
|
||||
}
|
||||
|
||||
/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples. `texts`
|
||||
/// (parallel to `idxs`) fills the text slot when set, else `null`.
|
||||
fn logprob_tuples(vals: &[f32], idxs: &[i32], texts: Option<&[String]>) -> serde_json::Value {
|
||||
let tuples: Vec<serde_json::Value> = vals
|
||||
.iter()
|
||||
.zip(idxs.iter())
|
||||
.enumerate()
|
||||
.map(|(j, (&v, &tid))| serde_json::json!([lp_value(v), tid, text_slot(texts, j)]))
|
||||
.collect();
|
||||
serde_json::Value::Array(tuples)
|
||||
}
|
||||
|
||||
/// Ragged top-k / token-ids shape: one entry per position — a list of
|
||||
/// `[logprob, token_id, text]` tuples, or `null` when `lens[p] == 0` (mirrors
|
||||
/// `detokenize_top_logprobs_tokens`). `texts` is parallel to `vals`/`idxs`.
|
||||
fn ragged_logprob_tuples(
|
||||
vals: &[f32],
|
||||
idxs: &[i32],
|
||||
lens: &[u32],
|
||||
texts: Option<&[String]>,
|
||||
) -> serde_json::Value {
|
||||
let mut positions = Vec::with_capacity(lens.len());
|
||||
let mut off = 0usize;
|
||||
for &l in lens {
|
||||
let l = l as usize;
|
||||
if l == 0 {
|
||||
positions.push(serde_json::Value::Null);
|
||||
} else {
|
||||
// Bounds-checked like `hidden_states_rows`: a header whose `lens` run
|
||||
// past the value buffer would otherwise panic the api thread on an
|
||||
// out-of-range index.
|
||||
let tuples: Vec<serde_json::Value> = (off..off + l)
|
||||
.filter_map(|j| {
|
||||
Some(serde_json::json!([
|
||||
lp_value(*vals.get(j)?),
|
||||
*idxs.get(j)?,
|
||||
text_slot(texts, j)
|
||||
]))
|
||||
})
|
||||
.collect();
|
||||
positions.push(serde_json::Value::Array(tuples));
|
||||
}
|
||||
off += l;
|
||||
}
|
||||
serde_json::Value::Array(positions)
|
||||
}
|
||||
use crate::native_generation::{
|
||||
hidden_states_rows, lp_value, meta_info_value, opt_texts, text_slot,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::native_generation::{logprob_tuples, ragged_logprob_tuples};
|
||||
|
||||
/// Append a flat family's `[logprob, token_id, text]` tuples to `dst`, comma
|
||||
/// separated and WITHOUT the enclosing brackets, so a cumulative frame can
|
||||
@@ -143,92 +78,17 @@ fn ragged_array_json(vals: &[f32], idxs: &[i32], lens: &[u32], texts: Option<&[S
|
||||
format!("[{body}]")
|
||||
}
|
||||
|
||||
/// Reshape flat hidden-state f32s + per-row lengths into `meta_info`'s nested
|
||||
/// `list[list[float]]` (one row per output position).
|
||||
fn hidden_states_rows(vals: &[f32], lens: &[u32]) -> serde_json::Value {
|
||||
let mut rows = Vec::with_capacity(lens.len());
|
||||
let mut off = 0usize;
|
||||
for &l in lens {
|
||||
let l = l as usize;
|
||||
// `get`, not a clamped index: clamping only the END leaves `off` past
|
||||
// `vals.len()` after one over-long row, making the next range reversed
|
||||
// (`start > end`) — which panics on the api thread rather than yielding
|
||||
// an empty row. Same reasoning as the decoder's `take_f32`.
|
||||
rows.push(serde_json::json!(vals.get(off..off + l).unwrap_or(&[])));
|
||||
off += l;
|
||||
}
|
||||
serde_json::Value::Array(rows)
|
||||
}
|
||||
|
||||
/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid`
|
||||
/// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is
|
||||
/// just the shard routing key.
|
||||
pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
|
||||
let mut v = serde_json::json!({
|
||||
"text": out.text,
|
||||
"meta_info": {
|
||||
"id": rid,
|
||||
"prompt_tokens": out.prompt_tokens,
|
||||
"completion_tokens": out.completion_tokens,
|
||||
// Full dict (type + matched + message + status_code + …), or null.
|
||||
"finish_reason": out.finish_reason,
|
||||
},
|
||||
"meta_info": meta_info_value(out, rid),
|
||||
});
|
||||
if !out.token_ids.is_empty() {
|
||||
v["output_ids"] = serde_json::json!(out.token_ids);
|
||||
}
|
||||
// Logprobs + hidden states ride behind the boxed extras (absent for a plain
|
||||
// token/text frame). `[logprob, token_id, text|null]` tuples; text
|
||||
// (`return_text_in_logprobs`) was decoded on the detok shard into `*_txt`.
|
||||
let Some(ex) = out.extras.as_deref() else {
|
||||
return v;
|
||||
};
|
||||
// Python (`add_logprob_to_meta_info`) always sets input+output token
|
||||
// logprobs together, empty lists included. A PD decode node never receives
|
||||
// input logprobs (they belong to prefill), yet its response must still
|
||||
// carry the key — the PD router keys its merge of prefill's
|
||||
// `input_token_logprobs` on its presence.
|
||||
if !ex.out_lp_val.is_empty() || !ex.in_lp_val.is_empty() {
|
||||
v["meta_info"]["output_token_logprobs"] =
|
||||
logprob_tuples(&ex.out_lp_val, &ex.out_lp_idx, opt_texts(&ex.out_lp_txt));
|
||||
v["meta_info"]["input_token_logprobs"] =
|
||||
logprob_tuples(&ex.in_lp_val, &ex.in_lp_idx, opt_texts(&ex.in_lp_txt));
|
||||
}
|
||||
if !ex.out_top_lens.is_empty() {
|
||||
v["meta_info"]["output_top_logprobs"] = ragged_logprob_tuples(
|
||||
&ex.out_top_val,
|
||||
&ex.out_top_idx,
|
||||
&ex.out_top_lens,
|
||||
opt_texts(&ex.out_top_txt),
|
||||
);
|
||||
}
|
||||
if !ex.in_top_lens.is_empty() {
|
||||
v["meta_info"]["input_top_logprobs"] = ragged_logprob_tuples(
|
||||
&ex.in_top_val,
|
||||
&ex.in_top_idx,
|
||||
&ex.in_top_lens,
|
||||
opt_texts(&ex.in_top_txt),
|
||||
);
|
||||
}
|
||||
if !ex.out_tid_lens.is_empty() {
|
||||
v["meta_info"]["output_token_ids_logprobs"] = ragged_logprob_tuples(
|
||||
&ex.out_tid_val,
|
||||
&ex.out_tid_idx,
|
||||
&ex.out_tid_lens,
|
||||
opt_texts(&ex.out_tid_txt),
|
||||
);
|
||||
}
|
||||
if !ex.in_tid_lens.is_empty() {
|
||||
v["meta_info"]["input_token_ids_logprobs"] = ragged_logprob_tuples(
|
||||
&ex.in_tid_val,
|
||||
&ex.in_tid_idx,
|
||||
&ex.in_tid_lens,
|
||||
opt_texts(&ex.in_tid_txt),
|
||||
);
|
||||
}
|
||||
if !ex.hidden_lens.is_empty() {
|
||||
v["meta_info"]["hidden_states"] = hidden_states_rows(&ex.hidden_val, &ex.hidden_lens);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
@@ -427,33 +287,10 @@ impl OutputAccumulator {
|
||||
let _ = write!(self.ids_json, "{id}");
|
||||
}
|
||||
|
||||
let o = &mut self.out;
|
||||
o.text.push_str(&d.text);
|
||||
o.token_ids.extend_from_slice(&d.token_ids); // token_ids doubles as output_ids
|
||||
o.completion_tokens += d.completion_tokens;
|
||||
o.prompt_tokens = d.prompt_tokens; // constant across the request
|
||||
if d.finish_reason.is_some() {
|
||||
o.finish_reason = d.finish_reason.clone();
|
||||
}
|
||||
// Logprobs/hidden ride behind the boxed extras — most frames have none, so
|
||||
// only allocate the accumulator's box once a delta actually carries some.
|
||||
let Some(de) = d.extras.as_deref() else {
|
||||
self.out.append_delta(d);
|
||||
return;
|
||||
};
|
||||
let oe = o
|
||||
.extras
|
||||
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
|
||||
oe.out_lp_val.extend_from_slice(&de.out_lp_val);
|
||||
oe.out_lp_idx.extend_from_slice(&de.out_lp_idx);
|
||||
oe.out_top_val.extend_from_slice(&de.out_top_val);
|
||||
oe.out_top_idx.extend_from_slice(&de.out_top_idx);
|
||||
oe.out_top_lens.extend_from_slice(&de.out_top_lens);
|
||||
oe.out_tid_val.extend_from_slice(&de.out_tid_val);
|
||||
oe.out_tid_idx.extend_from_slice(&de.out_tid_idx);
|
||||
oe.out_tid_lens.extend_from_slice(&de.out_tid_lens);
|
||||
oe.out_lp_txt.extend_from_slice(&de.out_lp_txt);
|
||||
oe.out_top_txt.extend_from_slice(&de.out_top_txt);
|
||||
oe.out_tid_txt.extend_from_slice(&de.out_tid_txt);
|
||||
// Append THIS delta's tuples, indexed within the delta — equivalent to
|
||||
// indexing the accumulated arrays only while texts stay in lockstep, which
|
||||
// the guard below verifies.
|
||||
@@ -477,6 +314,45 @@ impl OutputAccumulator {
|
||||
&de.out_tid_lens,
|
||||
opt_texts(&de.out_tid_txt),
|
||||
);
|
||||
if !de.in_lp_val.is_empty() {
|
||||
let mut body = String::new();
|
||||
push_logprob_tuples(
|
||||
&mut body,
|
||||
&de.in_lp_val,
|
||||
&de.in_lp_idx,
|
||||
opt_texts(&de.in_lp_txt),
|
||||
);
|
||||
self.in_lp_json = Some(format!("[{body}]"));
|
||||
}
|
||||
// Input families ride once (prefill); `lens` non-empty marks their arrival.
|
||||
if !de.in_top_lens.is_empty() {
|
||||
self.in_top_json = Some(ragged_array_json(
|
||||
&de.in_top_val,
|
||||
&de.in_top_idx,
|
||||
&de.in_top_lens,
|
||||
opt_texts(&de.in_top_txt),
|
||||
));
|
||||
}
|
||||
if !de.in_tid_lens.is_empty() {
|
||||
self.in_tid_json = Some(ragged_array_json(
|
||||
&de.in_tid_val,
|
||||
&de.in_tid_idx,
|
||||
&de.in_tid_lens,
|
||||
opt_texts(&de.in_tid_txt),
|
||||
));
|
||||
}
|
||||
// Hidden states are non-cumulative: the latest non-empty set wins.
|
||||
if !de.hidden_lens.is_empty() {
|
||||
self.hidden_json =
|
||||
Some(hidden_states_rows(&de.hidden_val, &de.hidden_lens).to_string());
|
||||
}
|
||||
|
||||
self.out.append_delta(d);
|
||||
let oe = self
|
||||
.out
|
||||
.extras
|
||||
.as_deref()
|
||||
.expect("a delta with extras creates cumulative extras");
|
||||
let lockstep = |txt: &Vec<String>, val: &Vec<f32>| txt.is_empty() || txt.len() == val.len();
|
||||
if !lockstep(&oe.out_lp_txt, &oe.out_lp_val)
|
||||
|| !lockstep(&oe.out_top_txt, &oe.out_top_val)
|
||||
@@ -484,51 +360,6 @@ impl OutputAccumulator {
|
||||
{
|
||||
self.extras_memo_broken = true;
|
||||
}
|
||||
if !de.in_lp_val.is_empty() {
|
||||
oe.in_lp_val = de.in_lp_val.clone();
|
||||
oe.in_lp_idx = de.in_lp_idx.clone();
|
||||
oe.in_lp_txt = de.in_lp_txt.clone();
|
||||
let mut body = String::new();
|
||||
push_logprob_tuples(
|
||||
&mut body,
|
||||
&oe.in_lp_val,
|
||||
&oe.in_lp_idx,
|
||||
opt_texts(&oe.in_lp_txt),
|
||||
);
|
||||
self.in_lp_json = Some(format!("[{body}]"));
|
||||
}
|
||||
// Input families ride once (prefill); `lens` non-empty marks their arrival.
|
||||
if !de.in_top_lens.is_empty() {
|
||||
oe.in_top_val = de.in_top_val.clone();
|
||||
oe.in_top_idx = de.in_top_idx.clone();
|
||||
oe.in_top_lens = de.in_top_lens.clone();
|
||||
oe.in_top_txt = de.in_top_txt.clone();
|
||||
self.in_top_json = Some(ragged_array_json(
|
||||
&oe.in_top_val,
|
||||
&oe.in_top_idx,
|
||||
&oe.in_top_lens,
|
||||
opt_texts(&oe.in_top_txt),
|
||||
));
|
||||
}
|
||||
if !de.in_tid_lens.is_empty() {
|
||||
oe.in_tid_val = de.in_tid_val.clone();
|
||||
oe.in_tid_idx = de.in_tid_idx.clone();
|
||||
oe.in_tid_lens = de.in_tid_lens.clone();
|
||||
oe.in_tid_txt = de.in_tid_txt.clone();
|
||||
self.in_tid_json = Some(ragged_array_json(
|
||||
&oe.in_tid_val,
|
||||
&oe.in_tid_idx,
|
||||
&oe.in_tid_lens,
|
||||
opt_texts(&oe.in_tid_txt),
|
||||
));
|
||||
}
|
||||
// Hidden states are non-cumulative: the latest non-empty set wins.
|
||||
if !de.hidden_lens.is_empty() {
|
||||
oe.hidden_val = de.hidden_val.clone();
|
||||
oe.hidden_lens = de.hidden_lens.clone();
|
||||
self.hidden_json =
|
||||
Some(hidden_states_rows(&oe.hidden_val, &oe.hidden_lens).to_string());
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrow the cumulative output for an intermediate streaming frame.
|
||||
|
||||
@@ -61,6 +61,90 @@ impl From<ChunkEvent> for FrontendOutput {
|
||||
}
|
||||
}
|
||||
|
||||
impl FrontendOutput {
|
||||
/// Fold one runtime delta into this cumulative native-generation output.
|
||||
///
|
||||
/// Runtime events are always incremental. Native HTTP and `runtime.v1`
|
||||
/// gRPC independently decide whether to expose those deltas or the
|
||||
/// cumulative result, so the fold itself lives at their shared semantic
|
||||
/// boundary rather than in either wire adapter.
|
||||
pub(crate) fn append_delta(&mut self, delta: &Self) {
|
||||
self.text.push_str(&delta.text);
|
||||
self.token_ids.extend_from_slice(&delta.token_ids);
|
||||
self.completion_tokens += delta.completion_tokens;
|
||||
self.prompt_tokens = delta.prompt_tokens;
|
||||
if delta.finish_reason.is_some() {
|
||||
self.finish_reason = delta.finish_reason.clone();
|
||||
}
|
||||
|
||||
let Some(delta_extras) = delta.extras.as_deref() else {
|
||||
return;
|
||||
};
|
||||
let extras = self
|
||||
.extras
|
||||
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
|
||||
|
||||
extras
|
||||
.out_lp_val
|
||||
.extend_from_slice(&delta_extras.out_lp_val);
|
||||
extras
|
||||
.out_lp_idx
|
||||
.extend_from_slice(&delta_extras.out_lp_idx);
|
||||
extras
|
||||
.out_top_val
|
||||
.extend_from_slice(&delta_extras.out_top_val);
|
||||
extras
|
||||
.out_top_idx
|
||||
.extend_from_slice(&delta_extras.out_top_idx);
|
||||
extras
|
||||
.out_top_lens
|
||||
.extend_from_slice(&delta_extras.out_top_lens);
|
||||
extras
|
||||
.out_tid_val
|
||||
.extend_from_slice(&delta_extras.out_tid_val);
|
||||
extras
|
||||
.out_tid_idx
|
||||
.extend_from_slice(&delta_extras.out_tid_idx);
|
||||
extras
|
||||
.out_tid_lens
|
||||
.extend_from_slice(&delta_extras.out_tid_lens);
|
||||
extras
|
||||
.out_lp_txt
|
||||
.extend_from_slice(&delta_extras.out_lp_txt);
|
||||
extras
|
||||
.out_top_txt
|
||||
.extend_from_slice(&delta_extras.out_top_txt);
|
||||
extras
|
||||
.out_tid_txt
|
||||
.extend_from_slice(&delta_extras.out_tid_txt);
|
||||
|
||||
// Input logprobs arrive once (during prefill). A later non-empty value
|
||||
// replaces the earlier one, matching the existing native HTTP fold.
|
||||
if !delta_extras.in_lp_val.is_empty() {
|
||||
extras.in_lp_val = delta_extras.in_lp_val.clone();
|
||||
extras.in_lp_idx = delta_extras.in_lp_idx.clone();
|
||||
extras.in_lp_txt = delta_extras.in_lp_txt.clone();
|
||||
}
|
||||
if !delta_extras.in_top_lens.is_empty() {
|
||||
extras.in_top_val = delta_extras.in_top_val.clone();
|
||||
extras.in_top_idx = delta_extras.in_top_idx.clone();
|
||||
extras.in_top_lens = delta_extras.in_top_lens.clone();
|
||||
extras.in_top_txt = delta_extras.in_top_txt.clone();
|
||||
}
|
||||
if !delta_extras.in_tid_lens.is_empty() {
|
||||
extras.in_tid_val = delta_extras.in_tid_val.clone();
|
||||
extras.in_tid_idx = delta_extras.in_tid_idx.clone();
|
||||
extras.in_tid_lens = delta_extras.in_tid_lens.clone();
|
||||
extras.in_tid_txt = delta_extras.in_tid_txt.clone();
|
||||
}
|
||||
// Hidden states are non-cumulative: the latest complete set wins.
|
||||
if !delta_extras.hidden_lens.is_empty() {
|
||||
extras.hidden_val = delta_extras.hidden_val.clone();
|
||||
extras.hidden_lens = delta_extras.hidden_lens.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Semantic result of a deep frontend health check.
|
||||
///
|
||||
/// Expected lifecycle states are values rather than transport errors so each
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
//! `runtime.v1` request conversion for the Rust frontend.
|
||||
//!
|
||||
//! The canonical protobuf is wider than the Rust frontend today. Fields with
|
||||
//! no internal representation are rejected explicitly instead of being
|
||||
//! silently ignored or expanding frontend behavior in this wiring change.
|
||||
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use tonic::Status;
|
||||
|
||||
use crate::frontend::FrontendRequest;
|
||||
use crate::message::config::PreferredSamplingParams;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
|
||||
type ConvertResult<T> = Result<T, ConvertError>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(super) enum ConvertError {
|
||||
#[error("{0}")]
|
||||
InvalidArgument(String),
|
||||
#[error("{0}")]
|
||||
Unsupported(String),
|
||||
#[error("{0}")]
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl ConvertError {
|
||||
fn invalid_argument(message: impl Into<String>) -> Self {
|
||||
Self::InvalidArgument(message.into())
|
||||
}
|
||||
|
||||
fn internal(message: impl Into<String>) -> Self {
|
||||
Self::Internal(message.into())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ConvertError> for Status {
|
||||
fn from(error: ConvertError) -> Self {
|
||||
match error {
|
||||
ConvertError::InvalidArgument(message) => Self::invalid_argument(message),
|
||||
ConvertError::Unsupported(message) => Self::unimplemented(message),
|
||||
ConvertError::Internal(message) => Self::internal(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn text_generate(
|
||||
request: proto::TextGenerateRequest,
|
||||
preferred: Option<&PreferredSamplingParams>,
|
||||
) -> ConvertResult<FrontendRequest> {
|
||||
let proto::TextGenerateRequest {
|
||||
text,
|
||||
sampling_params,
|
||||
stream,
|
||||
return_logprob,
|
||||
top_logprobs_num,
|
||||
logprob_start_len,
|
||||
return_text_in_logprobs,
|
||||
rid,
|
||||
lora_path,
|
||||
routing_key,
|
||||
routed_dp_rank,
|
||||
trace_headers,
|
||||
session_id,
|
||||
disaggregated_params,
|
||||
priority,
|
||||
require_reasoning,
|
||||
max_thinking_tokens,
|
||||
kv_hints,
|
||||
} = request;
|
||||
|
||||
reject_unsupported([
|
||||
("kv_hints", kv_hints.is_some()),
|
||||
("lora_path", lora_path.is_some()),
|
||||
("routing_key", routing_key.is_some()),
|
||||
("trace_headers", !trace_headers.is_empty()),
|
||||
("session_id", session_id.is_some()),
|
||||
("priority", priority.is_some()),
|
||||
("require_reasoning", require_reasoning.is_some()),
|
||||
("max_thinking_tokens", max_thinking_tokens.is_some()),
|
||||
])?;
|
||||
|
||||
let (bootstrap_host, bootstrap_port, bootstrap_room) =
|
||||
disaggregated_fields(disaggregated_params);
|
||||
Ok(FrontendRequest {
|
||||
rid: request_id(rid),
|
||||
text: Some(text),
|
||||
input_ids: None,
|
||||
skip_special_tokens: false,
|
||||
sampling_params: convert_sampling_params(sampling_params, preferred)?,
|
||||
stream: stream.unwrap_or(false),
|
||||
return_logprob: return_logprob.unwrap_or(false),
|
||||
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
|
||||
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
|
||||
token_ids_logprob: None,
|
||||
return_sampling_mask: false,
|
||||
return_hidden_states: false,
|
||||
return_text_in_logprobs,
|
||||
bootstrap_host,
|
||||
bootstrap_port,
|
||||
bootstrap_room,
|
||||
bootstrap_pair_key: None,
|
||||
decode_tp_size: None,
|
||||
routed_dp_rank: routed_dp_rank.map(i64::from),
|
||||
disagg_prefill_dp_rank: None,
|
||||
mm: None,
|
||||
mm_buffers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn generate(
|
||||
request: proto::GenerateRequest,
|
||||
preferred: Option<&PreferredSamplingParams>,
|
||||
) -> ConvertResult<FrontendRequest> {
|
||||
let proto::GenerateRequest {
|
||||
input_ids,
|
||||
sampling_params,
|
||||
stream,
|
||||
return_logprob,
|
||||
top_logprobs_num,
|
||||
logprob_start_len,
|
||||
rid,
|
||||
lora_path,
|
||||
routing_key,
|
||||
routed_dp_rank,
|
||||
trace_headers,
|
||||
session_id,
|
||||
disaggregated_params,
|
||||
priority,
|
||||
require_reasoning,
|
||||
max_thinking_tokens,
|
||||
kv_hints,
|
||||
} = request;
|
||||
|
||||
reject_unsupported([
|
||||
("kv_hints", kv_hints.is_some()),
|
||||
("lora_path", lora_path.is_some()),
|
||||
("routing_key", routing_key.is_some()),
|
||||
("trace_headers", !trace_headers.is_empty()),
|
||||
("session_id", session_id.is_some()),
|
||||
("priority", priority.is_some()),
|
||||
("require_reasoning", require_reasoning.is_some()),
|
||||
("max_thinking_tokens", max_thinking_tokens.is_some()),
|
||||
])?;
|
||||
if input_ids.is_empty() {
|
||||
return Err(ConvertError::invalid_argument("input_ids cannot be empty"));
|
||||
}
|
||||
|
||||
let (bootstrap_host, bootstrap_port, bootstrap_room) =
|
||||
disaggregated_fields(disaggregated_params);
|
||||
Ok(FrontendRequest {
|
||||
rid: request_id(rid),
|
||||
text: None,
|
||||
input_ids: Some(input_ids.into_iter().map(i64::from).collect()),
|
||||
skip_special_tokens: false,
|
||||
sampling_params: convert_sampling_params(sampling_params, preferred)?,
|
||||
stream: stream.unwrap_or(false),
|
||||
return_logprob: return_logprob.unwrap_or(false),
|
||||
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
|
||||
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
|
||||
token_ids_logprob: None,
|
||||
return_sampling_mask: false,
|
||||
return_hidden_states: false,
|
||||
return_text_in_logprobs: None,
|
||||
bootstrap_host,
|
||||
bootstrap_port,
|
||||
bootstrap_room,
|
||||
bootstrap_pair_key: None,
|
||||
decode_tp_size: None,
|
||||
routed_dp_rank: routed_dp_rank.map(i64::from),
|
||||
disagg_prefill_dp_rank: None,
|
||||
mm: None,
|
||||
mm_buffers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn request_id(rid: Option<String>) -> Rid {
|
||||
rid.map_or_else(Rid::new, |rid| Rid::from_client(&rid))
|
||||
}
|
||||
|
||||
fn reject_unsupported<const N: usize>(fields: [(&'static str, bool); N]) -> ConvertResult<()> {
|
||||
let unsupported = fields
|
||||
.into_iter()
|
||||
.filter_map(|(name, present)| present.then_some(name))
|
||||
.collect::<Vec<_>>();
|
||||
if unsupported.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
Err(ConvertError::Unsupported(format!(
|
||||
"the Rust frontend does not yet support runtime.v1 field(s): {}",
|
||||
unsupported.join(", ")
|
||||
)))
|
||||
}
|
||||
|
||||
fn disaggregated_fields(
|
||||
params: Option<proto::DisaggregatedParams>,
|
||||
) -> (Option<String>, Option<i64>, Option<i64>) {
|
||||
params.map_or((None, None, None), |params| {
|
||||
(
|
||||
Some(params.bootstrap_host),
|
||||
Some(i64::from(params.bootstrap_port)),
|
||||
Some(params.bootstrap_room),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Merge launch-time preferred values first, then overwrite only fields that
|
||||
/// are explicitly present in protobuf. This is the same precedence used by
|
||||
/// the Python-backed gRPC path; proto3 repeated fields cannot distinguish an
|
||||
/// omitted list from an explicit empty list, so empty stop lists remain absent.
|
||||
#[allow(deprecated)]
|
||||
fn convert_sampling_params(
|
||||
params: Option<proto::SamplingParams>,
|
||||
preferred: Option<&PreferredSamplingParams>,
|
||||
) -> ConvertResult<SamplingParams> {
|
||||
let mut output = SamplingParams::default();
|
||||
|
||||
let Some(params) = params else {
|
||||
if let Some(preferred) = preferred {
|
||||
output.apply_preferred(&preferred.0).map_err(|error| {
|
||||
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
|
||||
})?;
|
||||
}
|
||||
return Ok(output);
|
||||
};
|
||||
if params.guided_decoding.is_some() && (params.json_schema.is_some() || params.regex.is_some())
|
||||
{
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"legacy json_schema/regex cannot be combined with guided_decoding",
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(value) = params.temperature {
|
||||
output.temperature = f64::from(value);
|
||||
output.mark_explicit("temperature");
|
||||
}
|
||||
if let Some(value) = params.top_p {
|
||||
output.top_p = f64::from(value);
|
||||
output.mark_explicit("top_p");
|
||||
}
|
||||
if let Some(value) = params.top_k {
|
||||
output.top_k = i64::from(value);
|
||||
output.mark_explicit("top_k");
|
||||
}
|
||||
if let Some(value) = params.min_p {
|
||||
output.min_p = f64::from(value);
|
||||
output.mark_explicit("min_p");
|
||||
}
|
||||
if let Some(value) = params.frequency_penalty {
|
||||
output.frequency_penalty = f64::from(value);
|
||||
output.mark_explicit("frequency_penalty");
|
||||
}
|
||||
if let Some(value) = params.presence_penalty {
|
||||
output.presence_penalty = f64::from(value);
|
||||
output.mark_explicit("presence_penalty");
|
||||
}
|
||||
if let Some(value) = params.repetition_penalty {
|
||||
output.repetition_penalty = f64::from(value);
|
||||
output.mark_explicit("repetition_penalty");
|
||||
}
|
||||
if let Some(value) = params.max_new_tokens {
|
||||
output.max_new_tokens = Some(i64::from(value));
|
||||
output.mark_explicit("max_new_tokens");
|
||||
}
|
||||
if let Some(value) = params.min_new_tokens {
|
||||
output.min_new_tokens = i64::from(value);
|
||||
output.mark_explicit("min_new_tokens");
|
||||
}
|
||||
if !params.stop.is_empty() {
|
||||
output.stop = Some(crate::message::types::OneOrMany::Many(params.stop));
|
||||
output.mark_explicit("stop");
|
||||
}
|
||||
if !params.stop_token_ids.is_empty() {
|
||||
output.stop_token_ids = Some(params.stop_token_ids.into_iter().map(i64::from).collect());
|
||||
output.mark_explicit("stop_token_ids");
|
||||
}
|
||||
if let Some(value) = params.ignore_eos {
|
||||
output.ignore_eos = value;
|
||||
output.mark_explicit("ignore_eos");
|
||||
}
|
||||
if let Some(value) = params.n {
|
||||
output.n = i64::from(value);
|
||||
output.mark_explicit("n");
|
||||
}
|
||||
if let Some(value) = params.seed {
|
||||
output.sampling_seed = Some(value);
|
||||
output.mark_explicit("sampling_seed");
|
||||
}
|
||||
|
||||
if let Some(guided) = params.guided_decoding {
|
||||
apply_guided_decoding(&mut output, guided)?;
|
||||
} else {
|
||||
if let Some(value) = params.json_schema {
|
||||
if value.is_empty() {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"legacy json_schema must not be empty",
|
||||
));
|
||||
}
|
||||
output.json_schema = Some(value);
|
||||
output.mark_explicit("json_schema");
|
||||
}
|
||||
if let Some(value) = params.regex {
|
||||
if value.is_empty() {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"legacy regex must not be empty",
|
||||
));
|
||||
}
|
||||
output.regex = Some(value);
|
||||
output.mark_explicit("regex");
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(preferred) = preferred {
|
||||
output.apply_preferred(&preferred.0).map_err(|error| {
|
||||
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn apply_guided_decoding(
|
||||
output: &mut SamplingParams,
|
||||
guided: proto::GuidedDecoding,
|
||||
) -> ConvertResult<()> {
|
||||
use proto::guided_decoding::Constraint;
|
||||
|
||||
match guided.constraint {
|
||||
Some(Constraint::JsonSchema(value)) if !value.is_empty() => {
|
||||
output.json_schema = Some(value);
|
||||
output.mark_explicit("json_schema");
|
||||
}
|
||||
Some(Constraint::Regex(value)) if !value.is_empty() => {
|
||||
output.regex = Some(value);
|
||||
output.mark_explicit("regex");
|
||||
}
|
||||
Some(Constraint::Ebnf(value)) if !value.is_empty() => {
|
||||
output.ebnf = Some(value);
|
||||
output.mark_explicit("ebnf");
|
||||
}
|
||||
Some(Constraint::StructuralTag(value)) if !value.is_empty() => {
|
||||
output.structural_tag = Some(value);
|
||||
output.mark_explicit("structural_tag");
|
||||
}
|
||||
Some(Constraint::Choice(choice))
|
||||
if !choice.values.is_empty() && choice.values.iter().all(|value| !value.is_empty()) =>
|
||||
{
|
||||
let alternatives = choice
|
||||
.values
|
||||
.iter()
|
||||
.map(|value| regex_escape_literal(value))
|
||||
.collect::<Vec<_>>()
|
||||
.join("|");
|
||||
output.regex = Some(format!("(?:{alternatives})"));
|
||||
output.mark_explicit("regex");
|
||||
}
|
||||
Some(Constraint::Choice(_)) => {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"guided choice must contain only non-empty values",
|
||||
));
|
||||
}
|
||||
Some(_) => {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"guided decoding constraint must not be empty",
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"guided decoding constraint must be specified",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn regex_escape_literal(value: &str) -> String {
|
||||
let mut escaped = String::with_capacity(value.len());
|
||||
for character in value.chars() {
|
||||
if matches!(
|
||||
character,
|
||||
'.' | '+' | '*' | '?' | '^' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '\\'
|
||||
) {
|
||||
escaped.push('\\');
|
||||
}
|
||||
escaped.push(character);
|
||||
}
|
||||
escaped
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tonic::Code;
|
||||
|
||||
#[test]
|
||||
fn text_request_maps_supported_fields_and_preferred_sampling() {
|
||||
let preferred = PreferredSamplingParams(serde_json::json!({
|
||||
"temperature": 0.2,
|
||||
"top_p": 0.7,
|
||||
"max_new_tokens": 64,
|
||||
}));
|
||||
let request = proto::TextGenerateRequest {
|
||||
text: "hello".into(),
|
||||
sampling_params: Some(proto::SamplingParams {
|
||||
temperature: Some(0.8),
|
||||
seed: Some(7),
|
||||
..Default::default()
|
||||
}),
|
||||
stream: Some(true),
|
||||
return_logprob: Some(true),
|
||||
top_logprobs_num: Some(3),
|
||||
logprob_start_len: Some(2),
|
||||
return_text_in_logprobs: Some(true),
|
||||
rid: Some("client-id".into()),
|
||||
routed_dp_rank: Some(4),
|
||||
disaggregated_params: Some(proto::DisaggregatedParams {
|
||||
bootstrap_host: "10.0.0.1".into(),
|
||||
bootstrap_port: 8998,
|
||||
bootstrap_room: 12,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let request = text_generate(request, Some(&preferred)).unwrap();
|
||||
assert_eq!(request.rid.client_facing(), "client-id");
|
||||
assert_eq!(request.text.as_deref(), Some("hello"));
|
||||
assert!(request.input_ids.is_none());
|
||||
assert!(request.stream);
|
||||
assert!(request.return_logprob);
|
||||
assert_eq!(request.top_logprobs_num, 3);
|
||||
assert_eq!(request.logprob_start_len, 2);
|
||||
assert_eq!(request.return_text_in_logprobs, Some(true));
|
||||
assert_eq!(request.routed_dp_rank, Some(4));
|
||||
assert_eq!(request.bootstrap_host.as_deref(), Some("10.0.0.1"));
|
||||
assert_eq!(request.bootstrap_port, Some(8998));
|
||||
assert_eq!(request.bootstrap_room, Some(12));
|
||||
assert_eq!(request.sampling_params.temperature, f64::from(0.8_f32));
|
||||
assert_eq!(request.sampling_params.top_p, 0.7);
|
||||
assert_eq!(request.sampling_params.max_new_tokens, Some(64));
|
||||
assert_eq!(request.sampling_params.sampling_seed, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_request_validates_input_and_rejects_unported_fields() {
|
||||
let empty = Status::from(generate(proto::GenerateRequest::default(), None).unwrap_err());
|
||||
assert_eq!(empty.code(), Code::InvalidArgument);
|
||||
|
||||
let unsupported = Status::from(
|
||||
generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
routing_key: Some("worker-a".into()),
|
||||
priority: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap_err(),
|
||||
);
|
||||
assert_eq!(unsupported.code(), Code::Unimplemented);
|
||||
assert!(unsupported.message().contains("routing_key, priority"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_sampling_params_still_uses_preferred_values() {
|
||||
let preferred = PreferredSamplingParams(serde_json::json!({
|
||||
"temperature": 0.25,
|
||||
"max_new_tokens": 32,
|
||||
}));
|
||||
let request = generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
..Default::default()
|
||||
},
|
||||
Some(&preferred),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(request.sampling_params.temperature, 0.25);
|
||||
assert_eq!(request.sampling_params.max_new_tokens, Some(32));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guided_choice_preserves_existing_runtime_v1_mapping() {
|
||||
let request = generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
sampling_params: Some(proto::SamplingParams {
|
||||
guided_decoding: Some(proto::GuidedDecoding {
|
||||
constraint: Some(proto::guided_decoding::Constraint::Choice(
|
||||
proto::ChoiceConstraint {
|
||||
values: vec!["a+b".into(), "x.y".into()],
|
||||
},
|
||||
)),
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
request.sampling_params.regex.as_deref(),
|
||||
Some("(?:a\\+b|x\\.y)")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn guided_and_legacy_constraints_cannot_be_combined() {
|
||||
let error = Status::from(
|
||||
generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
sampling_params: Some(proto::SamplingParams {
|
||||
regex: Some("a".into()),
|
||||
guided_decoding: Some(proto::GuidedDecoding {
|
||||
constraint: Some(proto::guided_decoding::Constraint::Regex("b".into())),
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap_err(),
|
||||
);
|
||||
assert_eq!(error.code(), Code::InvalidArgument);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
//! Tonic adapter for the canonical `sglang.runtime.v1` API.
|
||||
//!
|
||||
//! This module only translates protobuf requests and responses. The shared
|
||||
//! [`FrontendHandle`] owns preprocessing, admission, runtime communication, and
|
||||
//! request cancellation; listener construction and server lifecycle are added
|
||||
//! by the next stack layer.
|
||||
//!
|
||||
//! The thin Tonic-service structure, streamed response approach, and test
|
||||
//! strategy build on Rain Jiang's multi-protocol prototype in
|
||||
//! `sgl-project/sglang#36923`. This stack keeps those ideas while using the
|
||||
//! canonical `runtime.v1` contract and the narrower [`FrontendHandle`] seam.
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::Stream;
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
|
||||
use tonic::{Request, Response, Status};
|
||||
|
||||
use crate::frontend::FrontendHandle;
|
||||
use crate::message::config::{PreferredSamplingParams, ServerArgs};
|
||||
|
||||
mod convert;
|
||||
mod response;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
#[cfg_attr(test, allow(dead_code))]
|
||||
const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
|
||||
|
||||
/// Narrow snapshot of launch policy required to preserve the existing
|
||||
/// `runtime.v1` generation behavior. It intentionally excludes listener,
|
||||
/// authentication, TLS, and lifecycle configuration.
|
||||
#[derive(Clone)]
|
||||
struct AdapterConfig {
|
||||
preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
incremental_streaming_output: bool,
|
||||
response_timeout: Duration,
|
||||
}
|
||||
|
||||
/// Tonic-facing implementation backed by the transport-neutral Rust frontend.
|
||||
///
|
||||
/// The listener follow-up constructs this service. The temporary dead-code
|
||||
/// allowance keeps this adapter-only stack layer warning-clean on its own.
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) struct GrpcService {
|
||||
frontend: FrontendHandle,
|
||||
config: AdapterConfig,
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
impl GrpcService {
|
||||
#[cfg_attr(test, allow(dead_code))]
|
||||
pub(crate) fn new(frontend: FrontendHandle, server_args: &ServerArgs) -> Self {
|
||||
Self {
|
||||
frontend,
|
||||
config: AdapterConfig {
|
||||
preferred_sampling_params: server_args.preferred_sampling_params.clone(),
|
||||
incremental_streaming_output: server_args.incremental_streaming_output,
|
||||
response_timeout: DEFAULT_RESPONSE_TIMEOUT,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn for_test(
|
||||
frontend: FrontendHandle,
|
||||
preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
incremental_streaming_output: bool,
|
||||
response_timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
frontend,
|
||||
config: AdapterConfig {
|
||||
preferred_sampling_params,
|
||||
incremental_streaming_output,
|
||||
response_timeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn unimplemented_rpc(name: &'static str) -> Status {
|
||||
Status::unimplemented(format!(
|
||||
"{name} is not implemented by the Rust frontend gRPC adapter"
|
||||
))
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl SglangService for GrpcService {
|
||||
type TextGenerateStream = ResponseStream<proto::TextGenerateResponse>;
|
||||
|
||||
async fn text_generate(
|
||||
&self,
|
||||
request: Request<proto::TextGenerateRequest>,
|
||||
) -> Result<Response<Self::TextGenerateStream>, Status> {
|
||||
let request = convert::text_generate(
|
||||
request.into_inner(),
|
||||
self.config.preferred_sampling_params.as_ref(),
|
||||
)
|
||||
.map_err(Status::from)?;
|
||||
let call = self
|
||||
.frontend
|
||||
.generate(request)
|
||||
.await
|
||||
.map_err(response::status)?;
|
||||
Ok(Response::new(response::text_generate_stream(
|
||||
call,
|
||||
self.config.incremental_streaming_output,
|
||||
self.config.response_timeout,
|
||||
)))
|
||||
}
|
||||
|
||||
type GenerateStream = ResponseStream<proto::GenerateResponse>;
|
||||
|
||||
async fn generate(
|
||||
&self,
|
||||
request: Request<proto::GenerateRequest>,
|
||||
) -> Result<Response<Self::GenerateStream>, Status> {
|
||||
let request = convert::generate(
|
||||
request.into_inner(),
|
||||
self.config.preferred_sampling_params.as_ref(),
|
||||
)
|
||||
.map_err(Status::from)?;
|
||||
let call = self
|
||||
.frontend
|
||||
.generate(request)
|
||||
.await
|
||||
.map_err(response::status)?;
|
||||
Ok(Response::new(response::generate_stream(
|
||||
call,
|
||||
self.config.incremental_streaming_output,
|
||||
self.config.response_timeout,
|
||||
)))
|
||||
}
|
||||
|
||||
// runtime.v1 is shared with the broader Python-backed service, so its
|
||||
// generated trait contains more operations than this initial Rust adapter.
|
||||
// Keep that support boundary explicit: adding an RPC is a deliberate future
|
||||
// change, never an accidental empty success.
|
||||
async fn text_embed(
|
||||
&self,
|
||||
_request: Request<proto::TextEmbedRequest>,
|
||||
) -> Result<Response<proto::TextEmbedResponse>, Status> {
|
||||
Err(unimplemented_rpc("text_embed"))
|
||||
}
|
||||
|
||||
async fn embed(
|
||||
&self,
|
||||
_request: Request<proto::EmbedRequest>,
|
||||
) -> Result<Response<proto::EmbedResponse>, Status> {
|
||||
Err(unimplemented_rpc("embed"))
|
||||
}
|
||||
|
||||
async fn classify(
|
||||
&self,
|
||||
_request: Request<proto::ClassifyRequest>,
|
||||
) -> Result<Response<proto::ClassifyResponse>, Status> {
|
||||
Err(unimplemented_rpc("classify"))
|
||||
}
|
||||
|
||||
async fn tokenize(
|
||||
&self,
|
||||
_request: Request<proto::TokenizeRequest>,
|
||||
) -> Result<Response<proto::TokenizeResponse>, Status> {
|
||||
Err(unimplemented_rpc("tokenize"))
|
||||
}
|
||||
|
||||
async fn detokenize(
|
||||
&self,
|
||||
_request: Request<proto::DetokenizeRequest>,
|
||||
) -> Result<Response<proto::DetokenizeResponse>, Status> {
|
||||
Err(unimplemented_rpc("detokenize"))
|
||||
}
|
||||
|
||||
async fn health_check(
|
||||
&self,
|
||||
_request: Request<proto::HealthCheckRequest>,
|
||||
) -> Result<Response<proto::HealthCheckResponse>, Status> {
|
||||
Err(unimplemented_rpc("health_check"))
|
||||
}
|
||||
|
||||
async fn get_model_info(
|
||||
&self,
|
||||
_request: Request<proto::GetModelInfoRequest>,
|
||||
) -> Result<Response<proto::GetModelInfoResponse>, Status> {
|
||||
Err(unimplemented_rpc("get_model_info"))
|
||||
}
|
||||
|
||||
async fn get_server_info(
|
||||
&self,
|
||||
_request: Request<proto::GetServerInfoRequest>,
|
||||
) -> Result<Response<proto::GetServerInfoResponse>, Status> {
|
||||
Err(unimplemented_rpc("get_server_info"))
|
||||
}
|
||||
|
||||
async fn list_models(
|
||||
&self,
|
||||
_request: Request<proto::ListModelsRequest>,
|
||||
) -> Result<Response<proto::ListModelsResponse>, Status> {
|
||||
Err(unimplemented_rpc("list_models"))
|
||||
}
|
||||
|
||||
async fn get_load(
|
||||
&self,
|
||||
_request: Request<proto::GetLoadRequest>,
|
||||
) -> Result<Response<proto::GetLoadResponse>, Status> {
|
||||
Err(unimplemented_rpc("get_load"))
|
||||
}
|
||||
|
||||
async fn abort(
|
||||
&self,
|
||||
_request: Request<proto::AbortRequest>,
|
||||
) -> Result<Response<proto::AbortResponse>, Status> {
|
||||
Err(unimplemented_rpc("abort"))
|
||||
}
|
||||
|
||||
async fn flush_cache(
|
||||
&self,
|
||||
_request: Request<proto::FlushCacheRequest>,
|
||||
) -> Result<Response<proto::FlushCacheResponse>, Status> {
|
||||
Err(unimplemented_rpc("flush_cache"))
|
||||
}
|
||||
|
||||
async fn pause_generation(
|
||||
&self,
|
||||
_request: Request<proto::PauseGenerationRequest>,
|
||||
) -> Result<Response<proto::PauseGenerationResponse>, Status> {
|
||||
Err(unimplemented_rpc("pause_generation"))
|
||||
}
|
||||
|
||||
type WatchEngineStateStream = ResponseStream<proto::EngineStateSnapshot>;
|
||||
|
||||
async fn watch_engine_state(
|
||||
&self,
|
||||
_request: Request<proto::WatchEngineStateRequest>,
|
||||
) -> Result<Response<Self::WatchEngineStateStream>, Status> {
|
||||
Err(unimplemented_rpc("watch_engine_state"))
|
||||
}
|
||||
|
||||
async fn continue_generation(
|
||||
&self,
|
||||
_request: Request<proto::ContinueGenerationRequest>,
|
||||
) -> Result<Response<proto::ContinueGenerationResponse>, Status> {
|
||||
Err(unimplemented_rpc("continue_generation"))
|
||||
}
|
||||
|
||||
type ChatCompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
|
||||
|
||||
async fn chat_complete(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<Self::ChatCompleteStream>, Status> {
|
||||
Err(unimplemented_rpc("chat_complete"))
|
||||
}
|
||||
|
||||
type CompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<Self::CompleteStream>, Status> {
|
||||
Err(unimplemented_rpc("complete"))
|
||||
}
|
||||
|
||||
async fn open_ai_embed(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("open_ai_embed"))
|
||||
}
|
||||
|
||||
async fn open_ai_classify(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("open_ai_classify"))
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("score"))
|
||||
}
|
||||
|
||||
async fn rerank(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("rerank"))
|
||||
}
|
||||
|
||||
async fn start_profile(
|
||||
&self,
|
||||
_request: Request<proto::StartProfileRequest>,
|
||||
) -> Result<Response<proto::StartProfileResponse>, Status> {
|
||||
Err(unimplemented_rpc("start_profile"))
|
||||
}
|
||||
|
||||
async fn stop_profile(
|
||||
&self,
|
||||
_request: Request<proto::StopProfileRequest>,
|
||||
) -> Result<Response<proto::StopProfileResponse>, Status> {
|
||||
Err(unimplemented_rpc("stop_profile"))
|
||||
}
|
||||
|
||||
async fn update_weights_from_disk(
|
||||
&self,
|
||||
_request: Request<proto::UpdateWeightsRequest>,
|
||||
) -> Result<Response<proto::UpdateWeightsResponse>, Status> {
|
||||
Err(unimplemented_rpc("update_weights_from_disk"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
//! Semantic frontend events to `runtime.v1` stream items and statuses.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use tonic::{Code, Status};
|
||||
|
||||
use super::ResponseStream;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput,
|
||||
};
|
||||
use crate::native_generation::meta_info_value;
|
||||
|
||||
pub(super) fn text_generate_stream(
|
||||
call: FrontendCall,
|
||||
incremental: bool,
|
||||
response_timeout: Duration,
|
||||
) -> ResponseStream<proto::TextGenerateResponse> {
|
||||
generation_stream(
|
||||
call,
|
||||
incremental,
|
||||
response_timeout,
|
||||
|output, meta_info, finished| {
|
||||
Ok(proto::TextGenerateResponse {
|
||||
text: output.text.clone(),
|
||||
meta_info,
|
||||
finished,
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn generate_stream(
|
||||
call: FrontendCall,
|
||||
incremental: bool,
|
||||
response_timeout: Duration,
|
||||
) -> ResponseStream<proto::GenerateResponse> {
|
||||
generation_stream(
|
||||
call,
|
||||
incremental,
|
||||
response_timeout,
|
||||
|output, meta_info, finished| {
|
||||
Ok(proto::GenerateResponse {
|
||||
output_ids: output
|
||||
.token_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.map(i32::try_from)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|_| {
|
||||
Status::internal("output token ID exceeds runtime.v1 int32 range")
|
||||
})?,
|
||||
meta_info,
|
||||
finished,
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn generation_stream<T, F>(
|
||||
mut call: FrontendCall,
|
||||
incremental: bool,
|
||||
response_timeout: Duration,
|
||||
build: F,
|
||||
) -> ResponseStream<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Fn(&FrontendOutput, HashMap<String, String>, bool) -> Result<T, Status> + Send + 'static,
|
||||
{
|
||||
let public_id = call.public_id().to_owned();
|
||||
let stream = async_stream::stream! {
|
||||
let mut cumulative = FrontendOutput::default();
|
||||
let mut cumulative_completion_tokens = 0;
|
||||
loop {
|
||||
let event = match tokio::time::timeout(response_timeout, call.recv()).await {
|
||||
Ok(Some(event)) => event,
|
||||
// The loop stops on terminal events, so this is defensive only.
|
||||
Ok(None) => break,
|
||||
Err(_) => {
|
||||
yield Err(Status::deadline_exceeded("Stream chunk timed out"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let (mut delta, finished) = match event {
|
||||
FrontendEvent::Delta(output) => (output, false),
|
||||
FrontendEvent::Finished(output) => (output, true),
|
||||
FrontendEvent::Failed(error) => {
|
||||
yield Err(status(error));
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let output = if incremental {
|
||||
// Python's incremental native stream still reports the cumulative
|
||||
// completion-token count in every frame. Do not retain the full
|
||||
// generated output just to calculate that one cumulative field.
|
||||
cumulative_completion_tokens += delta.completion_tokens;
|
||||
delta.completion_tokens = cumulative_completion_tokens;
|
||||
&delta
|
||||
} else {
|
||||
cumulative.append_delta(&delta);
|
||||
&cumulative
|
||||
};
|
||||
let meta_info = grpc_meta_info(output, &public_id);
|
||||
match build(output, meta_info, finished) {
|
||||
Ok(response) => yield Ok(response),
|
||||
Err(error) => {
|
||||
yield Err(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// `call` remains owned by this stream. Dropping a non-terminal stream or
|
||||
// timing out drops the armed FrontendCall and uses the core abort lane;
|
||||
// observing a terminal event has already disarmed it.
|
||||
};
|
||||
Box::pin(stream)
|
||||
}
|
||||
|
||||
/// runtime.v1 represents heterogeneous metadata as strings. Match the existing
|
||||
/// Python-backed server by JSON-encoding each value independently—including
|
||||
/// string values such as the request ID.
|
||||
fn grpc_meta_info(output: &FrontendOutput, public_id: &str) -> HashMap<String, String> {
|
||||
let serde_json::Value::Object(meta_info) = meta_info_value(output, public_id) else {
|
||||
unreachable!("native meta_info is always a JSON object");
|
||||
};
|
||||
meta_info
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key, value.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn status(error: FrontendError) -> Status {
|
||||
let code = match error.kind() {
|
||||
FrontendErrorKind::InvalidArgument => Code::InvalidArgument,
|
||||
FrontendErrorKind::NotFound => Code::NotFound,
|
||||
FrontendErrorKind::FailedPrecondition => Code::FailedPrecondition,
|
||||
FrontendErrorKind::ResourceExhausted => Code::ResourceExhausted,
|
||||
FrontendErrorKind::Cancelled => Code::Cancelled,
|
||||
FrontendErrorKind::DeadlineExceeded => Code::DeadlineExceeded,
|
||||
FrontendErrorKind::Unavailable => Code::Unavailable,
|
||||
FrontendErrorKind::Internal => Code::Internal,
|
||||
};
|
||||
Status::new(code, error.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn semantic_errors_map_to_canonical_grpc_codes() {
|
||||
let cases = [
|
||||
(
|
||||
FrontendError::InvalidArgument("bad".into()),
|
||||
Code::InvalidArgument,
|
||||
),
|
||||
(
|
||||
FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::NotFound,
|
||||
message: "missing".into(),
|
||||
legacy_http_status: 404,
|
||||
},
|
||||
Code::NotFound,
|
||||
),
|
||||
(
|
||||
FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::FailedPrecondition,
|
||||
message: "not ready".into(),
|
||||
legacy_http_status: 412,
|
||||
},
|
||||
Code::FailedPrecondition,
|
||||
),
|
||||
(
|
||||
FrontendError::Overloaded("full".into()),
|
||||
Code::ResourceExhausted,
|
||||
),
|
||||
(FrontendError::Cancelled("gone".into()), Code::Cancelled),
|
||||
(
|
||||
FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::DeadlineExceeded,
|
||||
message: "late".into(),
|
||||
legacy_http_status: 504,
|
||||
},
|
||||
Code::DeadlineExceeded,
|
||||
),
|
||||
(FrontendError::Unavailable, Code::Unavailable),
|
||||
(FrontendError::Internal("bug".into()), Code::Internal),
|
||||
];
|
||||
|
||||
for (error, expected) in cases {
|
||||
assert_eq!(status(error).code(), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
|
||||
use tonic::{Code, Request};
|
||||
|
||||
use super::GrpcService;
|
||||
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
|
||||
use crate::message::finish_reason::FinishReason;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind};
|
||||
use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink};
|
||||
use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
|
||||
|
||||
struct Harness {
|
||||
service: GrpcService,
|
||||
intake_rx: flume::Receiver<TmEvent>,
|
||||
abort_rx: flume::Receiver<AbortSource>,
|
||||
}
|
||||
|
||||
struct GenerationIntake {
|
||||
request: Box<GenerateRequest>,
|
||||
rid: Rid,
|
||||
sink: ResponseSink,
|
||||
admission: RequestAdmission,
|
||||
}
|
||||
|
||||
impl Harness {
|
||||
fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let frontend = FrontendHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
FrontendConfig {
|
||||
response_capacity,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: FrontendMetadata::default(),
|
||||
},
|
||||
);
|
||||
Self {
|
||||
service: GrpcService::for_test(frontend, None, incremental, response_timeout),
|
||||
intake_rx,
|
||||
abort_rx,
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_generation(&self) -> GenerationIntake {
|
||||
let TmEvent::Intake { request, admission } = self
|
||||
.intake_rx
|
||||
.recv_async()
|
||||
.await
|
||||
.expect("request submitted")
|
||||
else {
|
||||
panic!("expected generation intake");
|
||||
};
|
||||
let RuntimeRequest {
|
||||
rid,
|
||||
sink,
|
||||
kind,
|
||||
state: _,
|
||||
} = request;
|
||||
let RequestKind::Generate(request) = kind else {
|
||||
panic!("expected generation request");
|
||||
};
|
||||
GenerationIntake {
|
||||
request,
|
||||
rid,
|
||||
sink,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent {
|
||||
ChunkEvent {
|
||||
rid: rid.clone(),
|
||||
token_ids: vec![token_id],
|
||||
finish_reason: finished.then(stop_reason),
|
||||
prompt_tokens: 3,
|
||||
text: text.into(),
|
||||
completion_tokens: 1,
|
||||
extras: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_reason() -> FinishReason {
|
||||
serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn text_generate_maps_request_and_streams_cumulative_responses() {
|
||||
let harness = Harness::new(2, false, Duration::from_secs(1));
|
||||
let response = harness
|
||||
.service
|
||||
.text_generate(Request::new(proto::TextGenerateRequest {
|
||||
text: "prompt".into(),
|
||||
stream: Some(true),
|
||||
rid: Some("request-1".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut stream = response.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(intake.request.text.as_deref(), Some("prompt"));
|
||||
assert!(intake.request.input_ids.is_none());
|
||||
assert!(intake.request.stream);
|
||||
assert_eq!(intake.rid.client_facing(), "request-1");
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
|
||||
.unwrap();
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
|
||||
.unwrap();
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.text, "Hel");
|
||||
assert!(!first.finished);
|
||||
assert_eq!(first.meta_info["id"], r#""request-1""#);
|
||||
assert_eq!(first.meta_info["prompt_tokens"], "3");
|
||||
assert_eq!(first.meta_info["completion_tokens"], "1");
|
||||
assert_eq!(first.meta_info["finish_reason"], "null");
|
||||
|
||||
let finished = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(finished.text, "Hello");
|
||||
assert!(finished.finished);
|
||||
assert_eq!(finished.meta_info["completion_tokens"], "2");
|
||||
assert!(finished.meta_info["finish_reason"].contains(r#""type":"stop""#));
|
||||
assert!(stream.next().await.is_none());
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_streams_incremental_token_ids_with_cumulative_count() {
|
||||
let harness = Harness::new(2, true, Duration::from_secs(1));
|
||||
let response = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![4, 5],
|
||||
stream: Some(true),
|
||||
rid: Some("tokens".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut stream = response.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..]));
|
||||
assert!(intake.request.text.is_none());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false)))
|
||||
.unwrap();
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true)))
|
||||
.unwrap();
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
let finished = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.output_ids, vec![10]);
|
||||
assert_eq!(finished.output_ids, vec![11]);
|
||||
assert_eq!(finished.meta_info["completion_tokens"], "2");
|
||||
assert!(finished.finished);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_drop_aborts_admitted_request() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
rid: Some("cancel-me".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
|
||||
.unwrap();
|
||||
assert!(stream.next().await.unwrap().is_ok());
|
||||
drop(stream);
|
||||
let abort = harness.abort_rx.recv_async().await.unwrap();
|
||||
assert_eq!(abort.rid().client_facing(), "cancel-me");
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_failure_is_an_in_stream_status_and_disarms_abort() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
|
||||
"bad sampling".into(),
|
||||
)))
|
||||
.unwrap();
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert_eq!(error.code(), Code::InvalidArgument);
|
||||
drop(stream);
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() {
|
||||
let harness = Harness::new(1, false, Duration::from_millis(10));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
rid: Some("too-slow".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
|
||||
.unwrap();
|
||||
assert!(stream.next().await.unwrap().is_ok());
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert_eq!(error.code(), Code::DeadlineExceeded);
|
||||
drop(stream);
|
||||
let abort = harness.abort_rx.recv_async().await.unwrap();
|
||||
assert_eq!(abort.rid().client_facing(), "too-slow");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_intake_is_a_top_level_unavailable_status() {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
drop(intake_rx);
|
||||
let (abort_tx, _abort_rx) = flume::unbounded();
|
||||
let frontend = FrontendHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
FrontendConfig {
|
||||
response_capacity: 1,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: FrontendMetadata::default(),
|
||||
},
|
||||
);
|
||||
let service = GrpcService::for_test(frontend, None, false, Duration::from_secs(1));
|
||||
|
||||
let result = service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
..Default::default()
|
||||
}))
|
||||
.await;
|
||||
let error = match result {
|
||||
Ok(_) => panic!("closed intake must reject the RPC"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), Code::Unavailable);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_proto_fields_and_engine_watch_are_explicitly_unsupported() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let text_error = harness
|
||||
.service
|
||||
.text_generate(Request::new(proto::TextGenerateRequest {
|
||||
text: "hello".into(),
|
||||
kv_hints: Some(Default::default()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.err()
|
||||
.expect("KV hints must not be silently ignored");
|
||||
assert_eq!(text_error.code(), Code::Unimplemented);
|
||||
let token_error = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
kv_hints: Some(Default::default()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.err()
|
||||
.expect("KV hints must not be silently ignored");
|
||||
assert_eq!(token_error.code(), Code::Unimplemented);
|
||||
let watch_error = harness
|
||||
.service
|
||||
.watch_engine_state(Request::new(proto::WatchEngineStateRequest {}))
|
||||
.await
|
||||
.err()
|
||||
.expect("engine watch is outside the generation adapter");
|
||||
assert_eq!(watch_error.code(), Code::Unimplemented);
|
||||
assert!(harness.intake_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn output_token_overflow_returns_error_instead_of_truncating() {
|
||||
let harness = Harness::new(1, true, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![i32::MAX],
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(
|
||||
intake.request.input_ids.as_deref(),
|
||||
Some(&[i64::from(i32::MAX)][..])
|
||||
);
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(
|
||||
&intake.rid,
|
||||
"",
|
||||
i64::from(i32::MAX) + 1,
|
||||
false,
|
||||
)))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
stream.next().await.unwrap().unwrap_err().code(),
|
||||
Code::Internal
|
||||
);
|
||||
assert!(stream.next().await.is_none());
|
||||
assert!(
|
||||
matches!(harness.abort_rx.recv_async().await.unwrap(), AbortSource::Guard(rid) if rid == intake.rid)
|
||||
);
|
||||
}
|
||||
@@ -13,8 +13,10 @@
|
||||
|
||||
mod api_server;
|
||||
mod frontend;
|
||||
mod grpc;
|
||||
mod message;
|
||||
mod multi_modality;
|
||||
mod native_generation;
|
||||
mod tokenizer_manager;
|
||||
mod utils;
|
||||
|
||||
|
||||
@@ -315,10 +315,10 @@ impl SamplingParamsInput {
|
||||
/// wins even when it explicitly carries the type's default or null.
|
||||
pub fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
|
||||
match self {
|
||||
Self::One(params) => apply_preferred_to_one(params, preferred),
|
||||
Self::One(params) => params.apply_preferred(preferred),
|
||||
Self::Many(params) => params
|
||||
.iter_mut()
|
||||
.try_for_each(|params| apply_preferred_to_one(params, preferred)),
|
||||
.try_for_each(|params| params.apply_preferred(preferred)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,27 +327,6 @@ impl SamplingParamsInput {
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_preferred_to_one(
|
||||
params: &mut SamplingParams,
|
||||
preferred: &serde_json::Value,
|
||||
) -> Result<(), String> {
|
||||
let mut merged = preferred
|
||||
.as_object()
|
||||
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?
|
||||
.clone();
|
||||
let request_value = serde_json::to_value(&*params).map_err(|e| e.to_string())?;
|
||||
let request = request_value
|
||||
.as_object()
|
||||
.ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?;
|
||||
for field in ¶ms.explicit_fields {
|
||||
if let Some(value) = request.get(field) {
|
||||
merged.insert(field.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
*params = sampling_params_from_value(serde_json::Value::Object(merged))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Default for SamplingParams {
|
||||
fn default() -> Self {
|
||||
// Each field reads the same `default()` the serde attribute above names,
|
||||
@@ -393,6 +372,33 @@ impl Default for SamplingParams {
|
||||
}
|
||||
|
||||
impl SamplingParams {
|
||||
/// Record a field supplied by a non-Serde adapter. Preferred launch values
|
||||
/// are merged underneath these fields just as they are underneath keys
|
||||
/// present in an HTTP sampling object.
|
||||
pub(crate) fn mark_explicit(&mut self, field: &'static str) {
|
||||
self.explicit_fields.insert(field.to_owned());
|
||||
}
|
||||
|
||||
/// Merge operator-provided sampling defaults beneath explicitly supplied
|
||||
/// request fields. HTTP and gRPC adapters use the same precedence policy.
|
||||
pub(crate) fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
|
||||
let mut merged = preferred
|
||||
.as_object()
|
||||
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?
|
||||
.clone();
|
||||
let request_value = serde_json::to_value(&*self).map_err(|e| e.to_string())?;
|
||||
let request = request_value
|
||||
.as_object()
|
||||
.ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?;
|
||||
for field in &self.explicit_fields {
|
||||
if let Some(value) = request.get(field) {
|
||||
merged.insert(field.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
*self = sampling_params_from_value(serde_json::Value::Object(merged))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `__post_init__` → `normalize` → `verify`, the order
|
||||
/// `TokenizerManager._create_tokenized_object` runs them in. `Err` is a
|
||||
/// request-local 400. `skip_tokenizer_init` stands in for Python's
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
//! Shared wire semantics for SGLang-native generation responses.
|
||||
//!
|
||||
//! Native HTTP places these values in a JSON `meta_info` object, while
|
||||
//! `runtime.v1` gRPC JSON-encodes each value into `map<string, string>`. Keeping
|
||||
//! the logical shape here prevents the sibling adapters from drifting without
|
||||
//! making either adapter depend on the other.
|
||||
|
||||
use crate::frontend::FrontendOutput;
|
||||
|
||||
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
|
||||
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
|
||||
pub(crate) fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value {
|
||||
texts
|
||||
.and_then(|texts| texts.get(j))
|
||||
.map(|text| serde_json::json!(text))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
|
||||
/// A decoded-text column becomes the tuples' text source only when populated.
|
||||
pub(crate) fn opt_texts(texts: &[String]) -> Option<&[String]> {
|
||||
(!texts.is_empty()).then_some(texts)
|
||||
}
|
||||
|
||||
/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel.
|
||||
pub(crate) fn lp_value(value: f32) -> serde_json::Value {
|
||||
if value.is_nan() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
serde_json::json!(value)
|
||||
}
|
||||
}
|
||||
|
||||
/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples.
|
||||
pub(crate) fn logprob_tuples(
|
||||
values: &[f32],
|
||||
token_ids: &[i32],
|
||||
texts: Option<&[String]>,
|
||||
) -> serde_json::Value {
|
||||
let tuples = values
|
||||
.iter()
|
||||
.zip(token_ids.iter())
|
||||
.enumerate()
|
||||
.map(|(index, (&value, &token_id))| {
|
||||
serde_json::json!([lp_value(value), token_id, text_slot(texts, index)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::Value::Array(tuples)
|
||||
}
|
||||
|
||||
/// Ragged top-k / requested-token shape: one entry per position, or `null`
|
||||
/// where that position's length is zero.
|
||||
pub(crate) fn ragged_logprob_tuples(
|
||||
values: &[f32],
|
||||
token_ids: &[i32],
|
||||
lengths: &[u32],
|
||||
texts: Option<&[String]>,
|
||||
) -> serde_json::Value {
|
||||
let mut positions = Vec::with_capacity(lengths.len());
|
||||
let mut offset = 0usize;
|
||||
for &length in lengths {
|
||||
let length = length as usize;
|
||||
if length == 0 {
|
||||
positions.push(serde_json::Value::Null);
|
||||
} else {
|
||||
// Bounds checks turn malformed parallel columns into a shortened
|
||||
// row instead of panicking a transport worker.
|
||||
let tuples = (offset..offset + length)
|
||||
.filter_map(|index| {
|
||||
Some(serde_json::json!([
|
||||
lp_value(*values.get(index)?),
|
||||
*token_ids.get(index)?,
|
||||
text_slot(texts, index)
|
||||
]))
|
||||
})
|
||||
.collect();
|
||||
positions.push(serde_json::Value::Array(tuples));
|
||||
}
|
||||
offset += length;
|
||||
}
|
||||
serde_json::Value::Array(positions)
|
||||
}
|
||||
|
||||
/// Reshape flat hidden-state values and per-row lengths into nested rows.
|
||||
pub(crate) fn hidden_states_rows(values: &[f32], lengths: &[u32]) -> serde_json::Value {
|
||||
let mut rows = Vec::with_capacity(lengths.len());
|
||||
let mut offset = 0usize;
|
||||
for &length in lengths {
|
||||
let length = length as usize;
|
||||
rows.push(serde_json::json!(
|
||||
values.get(offset..offset + length).unwrap_or(&[])
|
||||
));
|
||||
offset += length;
|
||||
}
|
||||
serde_json::Value::Array(rows)
|
||||
}
|
||||
|
||||
/// Build the heterogeneous native `meta_info` object before either adapter
|
||||
/// chooses its wire encoding.
|
||||
pub(crate) fn meta_info_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
|
||||
let mut meta = serde_json::json!({
|
||||
"id": rid,
|
||||
"prompt_tokens": out.prompt_tokens,
|
||||
"completion_tokens": out.completion_tokens,
|
||||
"finish_reason": out.finish_reason,
|
||||
});
|
||||
let Some(extras) = out.extras.as_deref() else {
|
||||
return meta;
|
||||
};
|
||||
|
||||
// Python always sets input and output token logprobs together, including
|
||||
// the empty input list emitted by a disaggregated decode node.
|
||||
if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() {
|
||||
meta["output_token_logprobs"] = logprob_tuples(
|
||||
&extras.out_lp_val,
|
||||
&extras.out_lp_idx,
|
||||
opt_texts(&extras.out_lp_txt),
|
||||
);
|
||||
meta["input_token_logprobs"] = logprob_tuples(
|
||||
&extras.in_lp_val,
|
||||
&extras.in_lp_idx,
|
||||
opt_texts(&extras.in_lp_txt),
|
||||
);
|
||||
}
|
||||
if !extras.out_top_lens.is_empty() {
|
||||
meta["output_top_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.out_top_val,
|
||||
&extras.out_top_idx,
|
||||
&extras.out_top_lens,
|
||||
opt_texts(&extras.out_top_txt),
|
||||
);
|
||||
}
|
||||
if !extras.in_top_lens.is_empty() {
|
||||
meta["input_top_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.in_top_val,
|
||||
&extras.in_top_idx,
|
||||
&extras.in_top_lens,
|
||||
opt_texts(&extras.in_top_txt),
|
||||
);
|
||||
}
|
||||
if !extras.out_tid_lens.is_empty() {
|
||||
meta["output_token_ids_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.out_tid_val,
|
||||
&extras.out_tid_idx,
|
||||
&extras.out_tid_lens,
|
||||
opt_texts(&extras.out_tid_txt),
|
||||
);
|
||||
}
|
||||
if !extras.in_tid_lens.is_empty() {
|
||||
meta["input_token_ids_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.in_tid_val,
|
||||
&extras.in_tid_idx,
|
||||
&extras.in_tid_lens,
|
||||
opt_texts(&extras.in_tid_txt),
|
||||
);
|
||||
}
|
||||
if !extras.hidden_lens.is_empty() {
|
||||
meta["hidden_states"] = hidden_states_rows(&extras.hidden_val, &extras.hidden_lens);
|
||||
}
|
||||
meta
|
||||
}
|
||||
@@ -565,7 +565,7 @@ crate-type = ["cdylib"]
|
||||
"sglang-server",
|
||||
"sglang_server",
|
||||
(),
|
||||
(),
|
||||
(grpc_proto,),
|
||||
),
|
||||
(
|
||||
"sglang.srt.rust_extensions._grpc",
|
||||
|
||||
Reference in New Issue
Block a user