[Rust] Implement the Rust gRPC adapter for native generation (#41766)

Signed-off-by: jain-ria <riajain@NVIDIA.com>
This commit is contained in:
jain-ria
2026-09-30 18:17:42 -07:00
committed by GitHub
parent 6af651ea0c
commit ebccddce19
12 changed files with 1740 additions and 240 deletions
+2
View File
@@ -3809,10 +3809,12 @@ dependencies = [
"rustix",
"serde",
"serde_json",
"sglang-grpc-types",
"sglang-mm",
"socket2 0.6.5",
"thiserror",
"tokio",
"tonic",
"tower 0.5.3",
"tracing",
"tracing-appender",
+5
View File
@@ -10,6 +10,9 @@ license.workspace = true
python-module = "sglang.srt.rust_extensions._server"
# Always build optimized, even for an editable install.
debug = false
# The shared runtime.v1 crate lives outside rust/; include it in extension
# source fingerprints so protobuf changes cannot reuse a stale _server build.
source-inputs = ["../../proto"]
[lib]
name = "sglang_server"
@@ -60,10 +63,12 @@ numpy = "0.29.0"
regex-syntax = "=0.8.11"
rmp-serde = "1"
rmpv = { version = "1", features = ["with-serde"] }
sglang-grpc-types = { path = "../../proto" }
# The pure-Rust core of the MM pipeline. `default-features = false` drops the
# pyo3 bindings so it links as a plain rlib; renamed so `use sglang_mm::…` reads
# naturally while the crate keeps its own artifact name in the shared target/.
sglang_mm = { package = "sglang-mm", path = "../sglang-mm", default-features = false }
tonic = { workspace = true }
[dev-dependencies]
# `Router::oneshot` for handler-level router tests.
+47 -216
View File
@@ -5,78 +5,13 @@
//! and streams; it calls these per frame.
use crate::frontend::FrontendOutput;
#[cfg(test)]
use crate::message::response::ChunkExtras;
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value {
texts
.and_then(|t| t.get(j))
.map(|s| serde_json::json!(s))
.unwrap_or(serde_json::Value::Null)
}
/// A decoded-text column becomes the tuples' text source only when populated
/// (`return_text_in_logprobs`); empty → `None` → null text slots.
fn opt_texts(t: &[String]) -> Option<&[String]> {
(!t.is_empty()).then_some(t)
}
/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel.
fn lp_value(v: f32) -> serde_json::Value {
if v.is_nan() {
serde_json::Value::Null
} else {
serde_json::json!(v)
}
}
/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples. `texts`
/// (parallel to `idxs`) fills the text slot when set, else `null`.
fn logprob_tuples(vals: &[f32], idxs: &[i32], texts: Option<&[String]>) -> serde_json::Value {
let tuples: Vec<serde_json::Value> = vals
.iter()
.zip(idxs.iter())
.enumerate()
.map(|(j, (&v, &tid))| serde_json::json!([lp_value(v), tid, text_slot(texts, j)]))
.collect();
serde_json::Value::Array(tuples)
}
/// Ragged top-k / token-ids shape: one entry per position — a list of
/// `[logprob, token_id, text]` tuples, or `null` when `lens[p] == 0` (mirrors
/// `detokenize_top_logprobs_tokens`). `texts` is parallel to `vals`/`idxs`.
fn ragged_logprob_tuples(
vals: &[f32],
idxs: &[i32],
lens: &[u32],
texts: Option<&[String]>,
) -> serde_json::Value {
let mut positions = Vec::with_capacity(lens.len());
let mut off = 0usize;
for &l in lens {
let l = l as usize;
if l == 0 {
positions.push(serde_json::Value::Null);
} else {
// Bounds-checked like `hidden_states_rows`: a header whose `lens` run
// past the value buffer would otherwise panic the api thread on an
// out-of-range index.
let tuples: Vec<serde_json::Value> = (off..off + l)
.filter_map(|j| {
Some(serde_json::json!([
lp_value(*vals.get(j)?),
*idxs.get(j)?,
text_slot(texts, j)
]))
})
.collect();
positions.push(serde_json::Value::Array(tuples));
}
off += l;
}
serde_json::Value::Array(positions)
}
use crate::native_generation::{
hidden_states_rows, lp_value, meta_info_value, opt_texts, text_slot,
};
#[cfg(test)]
use crate::native_generation::{logprob_tuples, ragged_logprob_tuples};
/// Append a flat family's `[logprob, token_id, text]` tuples to `dst`, comma
/// separated and WITHOUT the enclosing brackets, so a cumulative frame can
@@ -143,92 +78,17 @@ fn ragged_array_json(vals: &[f32], idxs: &[i32], lens: &[u32], texts: Option<&[S
format!("[{body}]")
}
/// Reshape flat hidden-state f32s + per-row lengths into `meta_info`'s nested
/// `list[list[float]]` (one row per output position).
fn hidden_states_rows(vals: &[f32], lens: &[u32]) -> serde_json::Value {
let mut rows = Vec::with_capacity(lens.len());
let mut off = 0usize;
for &l in lens {
let l = l as usize;
// `get`, not a clamped index: clamping only the END leaves `off` past
// `vals.len()` after one over-long row, making the next range reversed
// (`start > end`) — which panics on the api thread rather than yielding
// an empty row. Same reasoning as the decoder's `take_f32`.
rows.push(serde_json::json!(vals.get(off..off + l).unwrap_or(&[])));
off += l;
}
serde_json::Value::Array(rows)
}
/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid`
/// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is
/// just the shard routing key.
pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
let mut v = serde_json::json!({
"text": out.text,
"meta_info": {
"id": rid,
"prompt_tokens": out.prompt_tokens,
"completion_tokens": out.completion_tokens,
// Full dict (type + matched + message + status_code + …), or null.
"finish_reason": out.finish_reason,
},
"meta_info": meta_info_value(out, rid),
});
if !out.token_ids.is_empty() {
v["output_ids"] = serde_json::json!(out.token_ids);
}
// Logprobs + hidden states ride behind the boxed extras (absent for a plain
// token/text frame). `[logprob, token_id, text|null]` tuples; text
// (`return_text_in_logprobs`) was decoded on the detok shard into `*_txt`.
let Some(ex) = out.extras.as_deref() else {
return v;
};
// Python (`add_logprob_to_meta_info`) always sets input+output token
// logprobs together, empty lists included. A PD decode node never receives
// input logprobs (they belong to prefill), yet its response must still
// carry the key — the PD router keys its merge of prefill's
// `input_token_logprobs` on its presence.
if !ex.out_lp_val.is_empty() || !ex.in_lp_val.is_empty() {
v["meta_info"]["output_token_logprobs"] =
logprob_tuples(&ex.out_lp_val, &ex.out_lp_idx, opt_texts(&ex.out_lp_txt));
v["meta_info"]["input_token_logprobs"] =
logprob_tuples(&ex.in_lp_val, &ex.in_lp_idx, opt_texts(&ex.in_lp_txt));
}
if !ex.out_top_lens.is_empty() {
v["meta_info"]["output_top_logprobs"] = ragged_logprob_tuples(
&ex.out_top_val,
&ex.out_top_idx,
&ex.out_top_lens,
opt_texts(&ex.out_top_txt),
);
}
if !ex.in_top_lens.is_empty() {
v["meta_info"]["input_top_logprobs"] = ragged_logprob_tuples(
&ex.in_top_val,
&ex.in_top_idx,
&ex.in_top_lens,
opt_texts(&ex.in_top_txt),
);
}
if !ex.out_tid_lens.is_empty() {
v["meta_info"]["output_token_ids_logprobs"] = ragged_logprob_tuples(
&ex.out_tid_val,
&ex.out_tid_idx,
&ex.out_tid_lens,
opt_texts(&ex.out_tid_txt),
);
}
if !ex.in_tid_lens.is_empty() {
v["meta_info"]["input_token_ids_logprobs"] = ragged_logprob_tuples(
&ex.in_tid_val,
&ex.in_tid_idx,
&ex.in_tid_lens,
opt_texts(&ex.in_tid_txt),
);
}
if !ex.hidden_lens.is_empty() {
v["meta_info"]["hidden_states"] = hidden_states_rows(&ex.hidden_val, &ex.hidden_lens);
}
v
}
@@ -427,33 +287,10 @@ impl OutputAccumulator {
let _ = write!(self.ids_json, "{id}");
}
let o = &mut self.out;
o.text.push_str(&d.text);
o.token_ids.extend_from_slice(&d.token_ids); // token_ids doubles as output_ids
o.completion_tokens += d.completion_tokens;
o.prompt_tokens = d.prompt_tokens; // constant across the request
if d.finish_reason.is_some() {
o.finish_reason = d.finish_reason.clone();
}
// Logprobs/hidden ride behind the boxed extras — most frames have none, so
// only allocate the accumulator's box once a delta actually carries some.
let Some(de) = d.extras.as_deref() else {
self.out.append_delta(d);
return;
};
let oe = o
.extras
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
oe.out_lp_val.extend_from_slice(&de.out_lp_val);
oe.out_lp_idx.extend_from_slice(&de.out_lp_idx);
oe.out_top_val.extend_from_slice(&de.out_top_val);
oe.out_top_idx.extend_from_slice(&de.out_top_idx);
oe.out_top_lens.extend_from_slice(&de.out_top_lens);
oe.out_tid_val.extend_from_slice(&de.out_tid_val);
oe.out_tid_idx.extend_from_slice(&de.out_tid_idx);
oe.out_tid_lens.extend_from_slice(&de.out_tid_lens);
oe.out_lp_txt.extend_from_slice(&de.out_lp_txt);
oe.out_top_txt.extend_from_slice(&de.out_top_txt);
oe.out_tid_txt.extend_from_slice(&de.out_tid_txt);
// Append THIS delta's tuples, indexed within the delta — equivalent to
// indexing the accumulated arrays only while texts stay in lockstep, which
// the guard below verifies.
@@ -477,6 +314,45 @@ impl OutputAccumulator {
&de.out_tid_lens,
opt_texts(&de.out_tid_txt),
);
if !de.in_lp_val.is_empty() {
let mut body = String::new();
push_logprob_tuples(
&mut body,
&de.in_lp_val,
&de.in_lp_idx,
opt_texts(&de.in_lp_txt),
);
self.in_lp_json = Some(format!("[{body}]"));
}
// Input families ride once (prefill); `lens` non-empty marks their arrival.
if !de.in_top_lens.is_empty() {
self.in_top_json = Some(ragged_array_json(
&de.in_top_val,
&de.in_top_idx,
&de.in_top_lens,
opt_texts(&de.in_top_txt),
));
}
if !de.in_tid_lens.is_empty() {
self.in_tid_json = Some(ragged_array_json(
&de.in_tid_val,
&de.in_tid_idx,
&de.in_tid_lens,
opt_texts(&de.in_tid_txt),
));
}
// Hidden states are non-cumulative: the latest non-empty set wins.
if !de.hidden_lens.is_empty() {
self.hidden_json =
Some(hidden_states_rows(&de.hidden_val, &de.hidden_lens).to_string());
}
self.out.append_delta(d);
let oe = self
.out
.extras
.as_deref()
.expect("a delta with extras creates cumulative extras");
let lockstep = |txt: &Vec<String>, val: &Vec<f32>| txt.is_empty() || txt.len() == val.len();
if !lockstep(&oe.out_lp_txt, &oe.out_lp_val)
|| !lockstep(&oe.out_top_txt, &oe.out_top_val)
@@ -484,51 +360,6 @@ impl OutputAccumulator {
{
self.extras_memo_broken = true;
}
if !de.in_lp_val.is_empty() {
oe.in_lp_val = de.in_lp_val.clone();
oe.in_lp_idx = de.in_lp_idx.clone();
oe.in_lp_txt = de.in_lp_txt.clone();
let mut body = String::new();
push_logprob_tuples(
&mut body,
&oe.in_lp_val,
&oe.in_lp_idx,
opt_texts(&oe.in_lp_txt),
);
self.in_lp_json = Some(format!("[{body}]"));
}
// Input families ride once (prefill); `lens` non-empty marks their arrival.
if !de.in_top_lens.is_empty() {
oe.in_top_val = de.in_top_val.clone();
oe.in_top_idx = de.in_top_idx.clone();
oe.in_top_lens = de.in_top_lens.clone();
oe.in_top_txt = de.in_top_txt.clone();
self.in_top_json = Some(ragged_array_json(
&oe.in_top_val,
&oe.in_top_idx,
&oe.in_top_lens,
opt_texts(&oe.in_top_txt),
));
}
if !de.in_tid_lens.is_empty() {
oe.in_tid_val = de.in_tid_val.clone();
oe.in_tid_idx = de.in_tid_idx.clone();
oe.in_tid_lens = de.in_tid_lens.clone();
oe.in_tid_txt = de.in_tid_txt.clone();
self.in_tid_json = Some(ragged_array_json(
&oe.in_tid_val,
&oe.in_tid_idx,
&oe.in_tid_lens,
opt_texts(&oe.in_tid_txt),
));
}
// Hidden states are non-cumulative: the latest non-empty set wins.
if !de.hidden_lens.is_empty() {
oe.hidden_val = de.hidden_val.clone();
oe.hidden_lens = de.hidden_lens.clone();
self.hidden_json =
Some(hidden_states_rows(&oe.hidden_val, &oe.hidden_lens).to_string());
}
}
/// Borrow the cumulative output for an intermediate streaming frame.
@@ -61,6 +61,90 @@ impl From<ChunkEvent> for FrontendOutput {
}
}
impl FrontendOutput {
/// Fold one runtime delta into this cumulative native-generation output.
///
/// Runtime events are always incremental. Native HTTP and `runtime.v1`
/// gRPC independently decide whether to expose those deltas or the
/// cumulative result, so the fold itself lives at their shared semantic
/// boundary rather than in either wire adapter.
pub(crate) fn append_delta(&mut self, delta: &Self) {
self.text.push_str(&delta.text);
self.token_ids.extend_from_slice(&delta.token_ids);
self.completion_tokens += delta.completion_tokens;
self.prompt_tokens = delta.prompt_tokens;
if delta.finish_reason.is_some() {
self.finish_reason = delta.finish_reason.clone();
}
let Some(delta_extras) = delta.extras.as_deref() else {
return;
};
let extras = self
.extras
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
extras
.out_lp_val
.extend_from_slice(&delta_extras.out_lp_val);
extras
.out_lp_idx
.extend_from_slice(&delta_extras.out_lp_idx);
extras
.out_top_val
.extend_from_slice(&delta_extras.out_top_val);
extras
.out_top_idx
.extend_from_slice(&delta_extras.out_top_idx);
extras
.out_top_lens
.extend_from_slice(&delta_extras.out_top_lens);
extras
.out_tid_val
.extend_from_slice(&delta_extras.out_tid_val);
extras
.out_tid_idx
.extend_from_slice(&delta_extras.out_tid_idx);
extras
.out_tid_lens
.extend_from_slice(&delta_extras.out_tid_lens);
extras
.out_lp_txt
.extend_from_slice(&delta_extras.out_lp_txt);
extras
.out_top_txt
.extend_from_slice(&delta_extras.out_top_txt);
extras
.out_tid_txt
.extend_from_slice(&delta_extras.out_tid_txt);
// Input logprobs arrive once (during prefill). A later non-empty value
// replaces the earlier one, matching the existing native HTTP fold.
if !delta_extras.in_lp_val.is_empty() {
extras.in_lp_val = delta_extras.in_lp_val.clone();
extras.in_lp_idx = delta_extras.in_lp_idx.clone();
extras.in_lp_txt = delta_extras.in_lp_txt.clone();
}
if !delta_extras.in_top_lens.is_empty() {
extras.in_top_val = delta_extras.in_top_val.clone();
extras.in_top_idx = delta_extras.in_top_idx.clone();
extras.in_top_lens = delta_extras.in_top_lens.clone();
extras.in_top_txt = delta_extras.in_top_txt.clone();
}
if !delta_extras.in_tid_lens.is_empty() {
extras.in_tid_val = delta_extras.in_tid_val.clone();
extras.in_tid_idx = delta_extras.in_tid_idx.clone();
extras.in_tid_lens = delta_extras.in_tid_lens.clone();
extras.in_tid_txt = delta_extras.in_tid_txt.clone();
}
// Hidden states are non-cumulative: the latest complete set wins.
if !delta_extras.hidden_lens.is_empty() {
extras.hidden_val = delta_extras.hidden_val.clone();
extras.hidden_lens = delta_extras.hidden_lens.clone();
}
}
}
/// Semantic result of a deep frontend health check.
///
/// Expected lifecycle states are values rather than transport errors so each
+531
View File
@@ -0,0 +1,531 @@
//! `runtime.v1` request conversion for the Rust frontend.
//!
//! The canonical protobuf is wider than the Rust frontend today. Fields with
//! no internal representation are rejected explicitly instead of being
//! silently ignored or expanding frontend behavior in this wiring change.
use sglang_grpc_types::sglang::runtime::v1 as proto;
use tonic::Status;
use crate::frontend::FrontendRequest;
use crate::message::config::PreferredSamplingParams;
use crate::message::ids::Rid;
use crate::message::sampling::SamplingParams;
type ConvertResult<T> = Result<T, ConvertError>;
#[derive(Debug, thiserror::Error)]
pub(super) enum ConvertError {
#[error("{0}")]
InvalidArgument(String),
#[error("{0}")]
Unsupported(String),
#[error("{0}")]
Internal(String),
}
impl ConvertError {
fn invalid_argument(message: impl Into<String>) -> Self {
Self::InvalidArgument(message.into())
}
fn internal(message: impl Into<String>) -> Self {
Self::Internal(message.into())
}
}
impl From<ConvertError> for Status {
fn from(error: ConvertError) -> Self {
match error {
ConvertError::InvalidArgument(message) => Self::invalid_argument(message),
ConvertError::Unsupported(message) => Self::unimplemented(message),
ConvertError::Internal(message) => Self::internal(message),
}
}
}
pub(super) fn text_generate(
request: proto::TextGenerateRequest,
preferred: Option<&PreferredSamplingParams>,
) -> ConvertResult<FrontendRequest> {
let proto::TextGenerateRequest {
text,
sampling_params,
stream,
return_logprob,
top_logprobs_num,
logprob_start_len,
return_text_in_logprobs,
rid,
lora_path,
routing_key,
routed_dp_rank,
trace_headers,
session_id,
disaggregated_params,
priority,
require_reasoning,
max_thinking_tokens,
kv_hints,
} = request;
reject_unsupported([
("kv_hints", kv_hints.is_some()),
("lora_path", lora_path.is_some()),
("routing_key", routing_key.is_some()),
("trace_headers", !trace_headers.is_empty()),
("session_id", session_id.is_some()),
("priority", priority.is_some()),
("require_reasoning", require_reasoning.is_some()),
("max_thinking_tokens", max_thinking_tokens.is_some()),
])?;
let (bootstrap_host, bootstrap_port, bootstrap_room) =
disaggregated_fields(disaggregated_params);
Ok(FrontendRequest {
rid: request_id(rid),
text: Some(text),
input_ids: None,
skip_special_tokens: false,
sampling_params: convert_sampling_params(sampling_params, preferred)?,
stream: stream.unwrap_or(false),
return_logprob: return_logprob.unwrap_or(false),
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
token_ids_logprob: None,
return_sampling_mask: false,
return_hidden_states: false,
return_text_in_logprobs,
bootstrap_host,
bootstrap_port,
bootstrap_room,
bootstrap_pair_key: None,
decode_tp_size: None,
routed_dp_rank: routed_dp_rank.map(i64::from),
disagg_prefill_dp_rank: None,
mm: None,
mm_buffers: Vec::new(),
})
}
pub(super) fn generate(
request: proto::GenerateRequest,
preferred: Option<&PreferredSamplingParams>,
) -> ConvertResult<FrontendRequest> {
let proto::GenerateRequest {
input_ids,
sampling_params,
stream,
return_logprob,
top_logprobs_num,
logprob_start_len,
rid,
lora_path,
routing_key,
routed_dp_rank,
trace_headers,
session_id,
disaggregated_params,
priority,
require_reasoning,
max_thinking_tokens,
kv_hints,
} = request;
reject_unsupported([
("kv_hints", kv_hints.is_some()),
("lora_path", lora_path.is_some()),
("routing_key", routing_key.is_some()),
("trace_headers", !trace_headers.is_empty()),
("session_id", session_id.is_some()),
("priority", priority.is_some()),
("require_reasoning", require_reasoning.is_some()),
("max_thinking_tokens", max_thinking_tokens.is_some()),
])?;
if input_ids.is_empty() {
return Err(ConvertError::invalid_argument("input_ids cannot be empty"));
}
let (bootstrap_host, bootstrap_port, bootstrap_room) =
disaggregated_fields(disaggregated_params);
Ok(FrontendRequest {
rid: request_id(rid),
text: None,
input_ids: Some(input_ids.into_iter().map(i64::from).collect()),
skip_special_tokens: false,
sampling_params: convert_sampling_params(sampling_params, preferred)?,
stream: stream.unwrap_or(false),
return_logprob: return_logprob.unwrap_or(false),
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
token_ids_logprob: None,
return_sampling_mask: false,
return_hidden_states: false,
return_text_in_logprobs: None,
bootstrap_host,
bootstrap_port,
bootstrap_room,
bootstrap_pair_key: None,
decode_tp_size: None,
routed_dp_rank: routed_dp_rank.map(i64::from),
disagg_prefill_dp_rank: None,
mm: None,
mm_buffers: Vec::new(),
})
}
fn request_id(rid: Option<String>) -> Rid {
rid.map_or_else(Rid::new, |rid| Rid::from_client(&rid))
}
fn reject_unsupported<const N: usize>(fields: [(&'static str, bool); N]) -> ConvertResult<()> {
let unsupported = fields
.into_iter()
.filter_map(|(name, present)| present.then_some(name))
.collect::<Vec<_>>();
if unsupported.is_empty() {
return Ok(());
}
Err(ConvertError::Unsupported(format!(
"the Rust frontend does not yet support runtime.v1 field(s): {}",
unsupported.join(", ")
)))
}
fn disaggregated_fields(
params: Option<proto::DisaggregatedParams>,
) -> (Option<String>, Option<i64>, Option<i64>) {
params.map_or((None, None, None), |params| {
(
Some(params.bootstrap_host),
Some(i64::from(params.bootstrap_port)),
Some(params.bootstrap_room),
)
})
}
/// Merge launch-time preferred values first, then overwrite only fields that
/// are explicitly present in protobuf. This is the same precedence used by
/// the Python-backed gRPC path; proto3 repeated fields cannot distinguish an
/// omitted list from an explicit empty list, so empty stop lists remain absent.
#[allow(deprecated)]
fn convert_sampling_params(
params: Option<proto::SamplingParams>,
preferred: Option<&PreferredSamplingParams>,
) -> ConvertResult<SamplingParams> {
let mut output = SamplingParams::default();
let Some(params) = params else {
if let Some(preferred) = preferred {
output.apply_preferred(&preferred.0).map_err(|error| {
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
})?;
}
return Ok(output);
};
if params.guided_decoding.is_some() && (params.json_schema.is_some() || params.regex.is_some())
{
return Err(ConvertError::invalid_argument(
"legacy json_schema/regex cannot be combined with guided_decoding",
));
}
if let Some(value) = params.temperature {
output.temperature = f64::from(value);
output.mark_explicit("temperature");
}
if let Some(value) = params.top_p {
output.top_p = f64::from(value);
output.mark_explicit("top_p");
}
if let Some(value) = params.top_k {
output.top_k = i64::from(value);
output.mark_explicit("top_k");
}
if let Some(value) = params.min_p {
output.min_p = f64::from(value);
output.mark_explicit("min_p");
}
if let Some(value) = params.frequency_penalty {
output.frequency_penalty = f64::from(value);
output.mark_explicit("frequency_penalty");
}
if let Some(value) = params.presence_penalty {
output.presence_penalty = f64::from(value);
output.mark_explicit("presence_penalty");
}
if let Some(value) = params.repetition_penalty {
output.repetition_penalty = f64::from(value);
output.mark_explicit("repetition_penalty");
}
if let Some(value) = params.max_new_tokens {
output.max_new_tokens = Some(i64::from(value));
output.mark_explicit("max_new_tokens");
}
if let Some(value) = params.min_new_tokens {
output.min_new_tokens = i64::from(value);
output.mark_explicit("min_new_tokens");
}
if !params.stop.is_empty() {
output.stop = Some(crate::message::types::OneOrMany::Many(params.stop));
output.mark_explicit("stop");
}
if !params.stop_token_ids.is_empty() {
output.stop_token_ids = Some(params.stop_token_ids.into_iter().map(i64::from).collect());
output.mark_explicit("stop_token_ids");
}
if let Some(value) = params.ignore_eos {
output.ignore_eos = value;
output.mark_explicit("ignore_eos");
}
if let Some(value) = params.n {
output.n = i64::from(value);
output.mark_explicit("n");
}
if let Some(value) = params.seed {
output.sampling_seed = Some(value);
output.mark_explicit("sampling_seed");
}
if let Some(guided) = params.guided_decoding {
apply_guided_decoding(&mut output, guided)?;
} else {
if let Some(value) = params.json_schema {
if value.is_empty() {
return Err(ConvertError::invalid_argument(
"legacy json_schema must not be empty",
));
}
output.json_schema = Some(value);
output.mark_explicit("json_schema");
}
if let Some(value) = params.regex {
if value.is_empty() {
return Err(ConvertError::invalid_argument(
"legacy regex must not be empty",
));
}
output.regex = Some(value);
output.mark_explicit("regex");
}
}
if let Some(preferred) = preferred {
output.apply_preferred(&preferred.0).map_err(|error| {
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
})?;
}
Ok(output)
}
fn apply_guided_decoding(
output: &mut SamplingParams,
guided: proto::GuidedDecoding,
) -> ConvertResult<()> {
use proto::guided_decoding::Constraint;
match guided.constraint {
Some(Constraint::JsonSchema(value)) if !value.is_empty() => {
output.json_schema = Some(value);
output.mark_explicit("json_schema");
}
Some(Constraint::Regex(value)) if !value.is_empty() => {
output.regex = Some(value);
output.mark_explicit("regex");
}
Some(Constraint::Ebnf(value)) if !value.is_empty() => {
output.ebnf = Some(value);
output.mark_explicit("ebnf");
}
Some(Constraint::StructuralTag(value)) if !value.is_empty() => {
output.structural_tag = Some(value);
output.mark_explicit("structural_tag");
}
Some(Constraint::Choice(choice))
if !choice.values.is_empty() && choice.values.iter().all(|value| !value.is_empty()) =>
{
let alternatives = choice
.values
.iter()
.map(|value| regex_escape_literal(value))
.collect::<Vec<_>>()
.join("|");
output.regex = Some(format!("(?:{alternatives})"));
output.mark_explicit("regex");
}
Some(Constraint::Choice(_)) => {
return Err(ConvertError::invalid_argument(
"guided choice must contain only non-empty values",
));
}
Some(_) => {
return Err(ConvertError::invalid_argument(
"guided decoding constraint must not be empty",
));
}
None => {
return Err(ConvertError::invalid_argument(
"guided decoding constraint must be specified",
));
}
}
Ok(())
}
fn regex_escape_literal(value: &str) -> String {
let mut escaped = String::with_capacity(value.len());
for character in value.chars() {
if matches!(
character,
'.' | '+' | '*' | '?' | '^' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '\\'
) {
escaped.push('\\');
}
escaped.push(character);
}
escaped
}
#[cfg(test)]
mod tests {
use super::*;
use tonic::Code;
#[test]
fn text_request_maps_supported_fields_and_preferred_sampling() {
let preferred = PreferredSamplingParams(serde_json::json!({
"temperature": 0.2,
"top_p": 0.7,
"max_new_tokens": 64,
}));
let request = proto::TextGenerateRequest {
text: "hello".into(),
sampling_params: Some(proto::SamplingParams {
temperature: Some(0.8),
seed: Some(7),
..Default::default()
}),
stream: Some(true),
return_logprob: Some(true),
top_logprobs_num: Some(3),
logprob_start_len: Some(2),
return_text_in_logprobs: Some(true),
rid: Some("client-id".into()),
routed_dp_rank: Some(4),
disaggregated_params: Some(proto::DisaggregatedParams {
bootstrap_host: "10.0.0.1".into(),
bootstrap_port: 8998,
bootstrap_room: 12,
}),
..Default::default()
};
let request = text_generate(request, Some(&preferred)).unwrap();
assert_eq!(request.rid.client_facing(), "client-id");
assert_eq!(request.text.as_deref(), Some("hello"));
assert!(request.input_ids.is_none());
assert!(request.stream);
assert!(request.return_logprob);
assert_eq!(request.top_logprobs_num, 3);
assert_eq!(request.logprob_start_len, 2);
assert_eq!(request.return_text_in_logprobs, Some(true));
assert_eq!(request.routed_dp_rank, Some(4));
assert_eq!(request.bootstrap_host.as_deref(), Some("10.0.0.1"));
assert_eq!(request.bootstrap_port, Some(8998));
assert_eq!(request.bootstrap_room, Some(12));
assert_eq!(request.sampling_params.temperature, f64::from(0.8_f32));
assert_eq!(request.sampling_params.top_p, 0.7);
assert_eq!(request.sampling_params.max_new_tokens, Some(64));
assert_eq!(request.sampling_params.sampling_seed, Some(7));
}
#[test]
fn token_request_validates_input_and_rejects_unported_fields() {
let empty = Status::from(generate(proto::GenerateRequest::default(), None).unwrap_err());
assert_eq!(empty.code(), Code::InvalidArgument);
let unsupported = Status::from(
generate(
proto::GenerateRequest {
input_ids: vec![1],
routing_key: Some("worker-a".into()),
priority: Some(1),
..Default::default()
},
None,
)
.unwrap_err(),
);
assert_eq!(unsupported.code(), Code::Unimplemented);
assert!(unsupported.message().contains("routing_key, priority"));
}
#[test]
fn absent_sampling_params_still_uses_preferred_values() {
let preferred = PreferredSamplingParams(serde_json::json!({
"temperature": 0.25,
"max_new_tokens": 32,
}));
let request = generate(
proto::GenerateRequest {
input_ids: vec![1],
..Default::default()
},
Some(&preferred),
)
.unwrap();
assert_eq!(request.sampling_params.temperature, 0.25);
assert_eq!(request.sampling_params.max_new_tokens, Some(32));
}
#[test]
fn guided_choice_preserves_existing_runtime_v1_mapping() {
let request = generate(
proto::GenerateRequest {
input_ids: vec![1],
sampling_params: Some(proto::SamplingParams {
guided_decoding: Some(proto::GuidedDecoding {
constraint: Some(proto::guided_decoding::Constraint::Choice(
proto::ChoiceConstraint {
values: vec!["a+b".into(), "x.y".into()],
},
)),
}),
..Default::default()
}),
..Default::default()
},
None,
)
.unwrap();
assert_eq!(
request.sampling_params.regex.as_deref(),
Some("(?:a\\+b|x\\.y)")
);
}
#[test]
#[allow(deprecated)]
fn guided_and_legacy_constraints_cannot_be_combined() {
let error = Status::from(
generate(
proto::GenerateRequest {
input_ids: vec![1],
sampling_params: Some(proto::SamplingParams {
regex: Some("a".into()),
guided_decoding: Some(proto::GuidedDecoding {
constraint: Some(proto::guided_decoding::Constraint::Regex("b".into())),
}),
..Default::default()
}),
..Default::default()
},
None,
)
.unwrap_err(),
);
assert_eq!(error.code(), Code::InvalidArgument);
}
}
+318
View File
@@ -0,0 +1,318 @@
//! Tonic adapter for the canonical `sglang.runtime.v1` API.
//!
//! This module only translates protobuf requests and responses. The shared
//! [`FrontendHandle`] owns preprocessing, admission, runtime communication, and
//! request cancellation; listener construction and server lifecycle are added
//! by the next stack layer.
//!
//! The thin Tonic-service structure, streamed response approach, and test
//! strategy build on Rain Jiang's multi-protocol prototype in
//! `sgl-project/sglang#36923`. This stack keeps those ideas while using the
//! canonical `runtime.v1` contract and the narrower [`FrontendHandle`] seam.
use std::pin::Pin;
use std::time::Duration;
use futures::Stream;
use sglang_grpc_types::sglang::runtime::v1 as proto;
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
use tonic::{Request, Response, Status};
use crate::frontend::FrontendHandle;
use crate::message::config::{PreferredSamplingParams, ServerArgs};
mod convert;
mod response;
#[cfg(test)]
mod tests;
#[cfg_attr(test, allow(dead_code))]
const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300);
type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
/// Narrow snapshot of launch policy required to preserve the existing
/// `runtime.v1` generation behavior. It intentionally excludes listener,
/// authentication, TLS, and lifecycle configuration.
#[derive(Clone)]
struct AdapterConfig {
preferred_sampling_params: Option<PreferredSamplingParams>,
incremental_streaming_output: bool,
response_timeout: Duration,
}
/// Tonic-facing implementation backed by the transport-neutral Rust frontend.
///
/// The listener follow-up constructs this service. The temporary dead-code
/// allowance keeps this adapter-only stack layer warning-clean on its own.
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) struct GrpcService {
frontend: FrontendHandle,
config: AdapterConfig,
}
#[cfg_attr(not(test), allow(dead_code))]
impl GrpcService {
#[cfg_attr(test, allow(dead_code))]
pub(crate) fn new(frontend: FrontendHandle, server_args: &ServerArgs) -> Self {
Self {
frontend,
config: AdapterConfig {
preferred_sampling_params: server_args.preferred_sampling_params.clone(),
incremental_streaming_output: server_args.incremental_streaming_output,
response_timeout: DEFAULT_RESPONSE_TIMEOUT,
},
}
}
#[cfg(test)]
fn for_test(
frontend: FrontendHandle,
preferred_sampling_params: Option<PreferredSamplingParams>,
incremental_streaming_output: bool,
response_timeout: Duration,
) -> Self {
Self {
frontend,
config: AdapterConfig {
preferred_sampling_params,
incremental_streaming_output,
response_timeout,
},
}
}
}
fn unimplemented_rpc(name: &'static str) -> Status {
Status::unimplemented(format!(
"{name} is not implemented by the Rust frontend gRPC adapter"
))
}
#[tonic::async_trait]
impl SglangService for GrpcService {
type TextGenerateStream = ResponseStream<proto::TextGenerateResponse>;
async fn text_generate(
&self,
request: Request<proto::TextGenerateRequest>,
) -> Result<Response<Self::TextGenerateStream>, Status> {
let request = convert::text_generate(
request.into_inner(),
self.config.preferred_sampling_params.as_ref(),
)
.map_err(Status::from)?;
let call = self
.frontend
.generate(request)
.await
.map_err(response::status)?;
Ok(Response::new(response::text_generate_stream(
call,
self.config.incremental_streaming_output,
self.config.response_timeout,
)))
}
type GenerateStream = ResponseStream<proto::GenerateResponse>;
async fn generate(
&self,
request: Request<proto::GenerateRequest>,
) -> Result<Response<Self::GenerateStream>, Status> {
let request = convert::generate(
request.into_inner(),
self.config.preferred_sampling_params.as_ref(),
)
.map_err(Status::from)?;
let call = self
.frontend
.generate(request)
.await
.map_err(response::status)?;
Ok(Response::new(response::generate_stream(
call,
self.config.incremental_streaming_output,
self.config.response_timeout,
)))
}
// runtime.v1 is shared with the broader Python-backed service, so its
// generated trait contains more operations than this initial Rust adapter.
// Keep that support boundary explicit: adding an RPC is a deliberate future
// change, never an accidental empty success.
async fn text_embed(
&self,
_request: Request<proto::TextEmbedRequest>,
) -> Result<Response<proto::TextEmbedResponse>, Status> {
Err(unimplemented_rpc("text_embed"))
}
async fn embed(
&self,
_request: Request<proto::EmbedRequest>,
) -> Result<Response<proto::EmbedResponse>, Status> {
Err(unimplemented_rpc("embed"))
}
async fn classify(
&self,
_request: Request<proto::ClassifyRequest>,
) -> Result<Response<proto::ClassifyResponse>, Status> {
Err(unimplemented_rpc("classify"))
}
async fn tokenize(
&self,
_request: Request<proto::TokenizeRequest>,
) -> Result<Response<proto::TokenizeResponse>, Status> {
Err(unimplemented_rpc("tokenize"))
}
async fn detokenize(
&self,
_request: Request<proto::DetokenizeRequest>,
) -> Result<Response<proto::DetokenizeResponse>, Status> {
Err(unimplemented_rpc("detokenize"))
}
async fn health_check(
&self,
_request: Request<proto::HealthCheckRequest>,
) -> Result<Response<proto::HealthCheckResponse>, Status> {
Err(unimplemented_rpc("health_check"))
}
async fn get_model_info(
&self,
_request: Request<proto::GetModelInfoRequest>,
) -> Result<Response<proto::GetModelInfoResponse>, Status> {
Err(unimplemented_rpc("get_model_info"))
}
async fn get_server_info(
&self,
_request: Request<proto::GetServerInfoRequest>,
) -> Result<Response<proto::GetServerInfoResponse>, Status> {
Err(unimplemented_rpc("get_server_info"))
}
async fn list_models(
&self,
_request: Request<proto::ListModelsRequest>,
) -> Result<Response<proto::ListModelsResponse>, Status> {
Err(unimplemented_rpc("list_models"))
}
async fn get_load(
&self,
_request: Request<proto::GetLoadRequest>,
) -> Result<Response<proto::GetLoadResponse>, Status> {
Err(unimplemented_rpc("get_load"))
}
async fn abort(
&self,
_request: Request<proto::AbortRequest>,
) -> Result<Response<proto::AbortResponse>, Status> {
Err(unimplemented_rpc("abort"))
}
async fn flush_cache(
&self,
_request: Request<proto::FlushCacheRequest>,
) -> Result<Response<proto::FlushCacheResponse>, Status> {
Err(unimplemented_rpc("flush_cache"))
}
async fn pause_generation(
&self,
_request: Request<proto::PauseGenerationRequest>,
) -> Result<Response<proto::PauseGenerationResponse>, Status> {
Err(unimplemented_rpc("pause_generation"))
}
type WatchEngineStateStream = ResponseStream<proto::EngineStateSnapshot>;
async fn watch_engine_state(
&self,
_request: Request<proto::WatchEngineStateRequest>,
) -> Result<Response<Self::WatchEngineStateStream>, Status> {
Err(unimplemented_rpc("watch_engine_state"))
}
async fn continue_generation(
&self,
_request: Request<proto::ContinueGenerationRequest>,
) -> Result<Response<proto::ContinueGenerationResponse>, Status> {
Err(unimplemented_rpc("continue_generation"))
}
type ChatCompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
async fn chat_complete(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<Self::ChatCompleteStream>, Status> {
Err(unimplemented_rpc("chat_complete"))
}
type CompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
async fn complete(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<Self::CompleteStream>, Status> {
Err(unimplemented_rpc("complete"))
}
async fn open_ai_embed(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("open_ai_embed"))
}
async fn open_ai_classify(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("open_ai_classify"))
}
async fn score(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("score"))
}
async fn rerank(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("rerank"))
}
async fn start_profile(
&self,
_request: Request<proto::StartProfileRequest>,
) -> Result<Response<proto::StartProfileResponse>, Status> {
Err(unimplemented_rpc("start_profile"))
}
async fn stop_profile(
&self,
_request: Request<proto::StopProfileRequest>,
) -> Result<Response<proto::StopProfileResponse>, Status> {
Err(unimplemented_rpc("stop_profile"))
}
async fn update_weights_from_disk(
&self,
_request: Request<proto::UpdateWeightsRequest>,
) -> Result<Response<proto::UpdateWeightsResponse>, Status> {
Err(unimplemented_rpc("update_weights_from_disk"))
}
}
+200
View File
@@ -0,0 +1,200 @@
//! Semantic frontend events to `runtime.v1` stream items and statuses.
use std::collections::HashMap;
use std::time::Duration;
use sglang_grpc_types::sglang::runtime::v1 as proto;
use tonic::{Code, Status};
use super::ResponseStream;
use crate::frontend::{
FrontendCall, FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput,
};
use crate::native_generation::meta_info_value;
pub(super) fn text_generate_stream(
call: FrontendCall,
incremental: bool,
response_timeout: Duration,
) -> ResponseStream<proto::TextGenerateResponse> {
generation_stream(
call,
incremental,
response_timeout,
|output, meta_info, finished| {
Ok(proto::TextGenerateResponse {
text: output.text.clone(),
meta_info,
finished,
})
},
)
}
pub(super) fn generate_stream(
call: FrontendCall,
incremental: bool,
response_timeout: Duration,
) -> ResponseStream<proto::GenerateResponse> {
generation_stream(
call,
incremental,
response_timeout,
|output, meta_info, finished| {
Ok(proto::GenerateResponse {
output_ids: output
.token_ids
.iter()
.copied()
.map(i32::try_from)
.collect::<Result<Vec<_>, _>>()
.map_err(|_| {
Status::internal("output token ID exceeds runtime.v1 int32 range")
})?,
meta_info,
finished,
})
},
)
}
fn generation_stream<T, F>(
mut call: FrontendCall,
incremental: bool,
response_timeout: Duration,
build: F,
) -> ResponseStream<T>
where
T: Send + 'static,
F: Fn(&FrontendOutput, HashMap<String, String>, bool) -> Result<T, Status> + Send + 'static,
{
let public_id = call.public_id().to_owned();
let stream = async_stream::stream! {
let mut cumulative = FrontendOutput::default();
let mut cumulative_completion_tokens = 0;
loop {
let event = match tokio::time::timeout(response_timeout, call.recv()).await {
Ok(Some(event)) => event,
// The loop stops on terminal events, so this is defensive only.
Ok(None) => break,
Err(_) => {
yield Err(Status::deadline_exceeded("Stream chunk timed out"));
break;
}
};
let (mut delta, finished) = match event {
FrontendEvent::Delta(output) => (output, false),
FrontendEvent::Finished(output) => (output, true),
FrontendEvent::Failed(error) => {
yield Err(status(error));
break;
}
};
let output = if incremental {
// Python's incremental native stream still reports the cumulative
// completion-token count in every frame. Do not retain the full
// generated output just to calculate that one cumulative field.
cumulative_completion_tokens += delta.completion_tokens;
delta.completion_tokens = cumulative_completion_tokens;
&delta
} else {
cumulative.append_delta(&delta);
&cumulative
};
let meta_info = grpc_meta_info(output, &public_id);
match build(output, meta_info, finished) {
Ok(response) => yield Ok(response),
Err(error) => {
yield Err(error);
break;
}
}
if finished {
break;
}
}
// `call` remains owned by this stream. Dropping a non-terminal stream or
// timing out drops the armed FrontendCall and uses the core abort lane;
// observing a terminal event has already disarmed it.
};
Box::pin(stream)
}
/// runtime.v1 represents heterogeneous metadata as strings. Match the existing
/// Python-backed server by JSON-encoding each value independently—including
/// string values such as the request ID.
fn grpc_meta_info(output: &FrontendOutput, public_id: &str) -> HashMap<String, String> {
let serde_json::Value::Object(meta_info) = meta_info_value(output, public_id) else {
unreachable!("native meta_info is always a JSON object");
};
meta_info
.into_iter()
.map(|(key, value)| (key, value.to_string()))
.collect()
}
pub(super) fn status(error: FrontendError) -> Status {
let code = match error.kind() {
FrontendErrorKind::InvalidArgument => Code::InvalidArgument,
FrontendErrorKind::NotFound => Code::NotFound,
FrontendErrorKind::FailedPrecondition => Code::FailedPrecondition,
FrontendErrorKind::ResourceExhausted => Code::ResourceExhausted,
FrontendErrorKind::Cancelled => Code::Cancelled,
FrontendErrorKind::DeadlineExceeded => Code::DeadlineExceeded,
FrontendErrorKind::Unavailable => Code::Unavailable,
FrontendErrorKind::Internal => Code::Internal,
};
Status::new(code, error.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn semantic_errors_map_to_canonical_grpc_codes() {
let cases = [
(
FrontendError::InvalidArgument("bad".into()),
Code::InvalidArgument,
),
(
FrontendError::RuntimeRejected {
kind: FrontendErrorKind::NotFound,
message: "missing".into(),
legacy_http_status: 404,
},
Code::NotFound,
),
(
FrontendError::RuntimeRejected {
kind: FrontendErrorKind::FailedPrecondition,
message: "not ready".into(),
legacy_http_status: 412,
},
Code::FailedPrecondition,
),
(
FrontendError::Overloaded("full".into()),
Code::ResourceExhausted,
),
(FrontendError::Cancelled("gone".into()), Code::Cancelled),
(
FrontendError::RuntimeRejected {
kind: FrontendErrorKind::DeadlineExceeded,
message: "late".into(),
legacy_http_status: 504,
},
Code::DeadlineExceeded,
),
(FrontendError::Unavailable, Code::Unavailable),
(FrontendError::Internal("bug".into()), Code::Internal),
];
for (error, expected) in cases {
assert_eq!(status(error).code(), expected);
}
}
}
+361
View File
@@ -0,0 +1,361 @@
use std::time::Duration;
use futures::StreamExt;
use sglang_grpc_types::sglang::runtime::v1 as proto;
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
use tonic::{Code, Request};
use super::GrpcService;
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
use crate::message::finish_reason::FinishReason;
use crate::message::ids::Rid;
use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind};
use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink};
use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
struct Harness {
service: GrpcService,
intake_rx: flume::Receiver<TmEvent>,
abort_rx: flume::Receiver<AbortSource>,
}
struct GenerationIntake {
request: Box<GenerateRequest>,
rid: Rid,
sink: ResponseSink,
admission: RequestAdmission,
}
impl Harness {
fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self {
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let frontend = FrontendHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
response_capacity,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: FrontendMetadata::default(),
},
);
Self {
service: GrpcService::for_test(frontend, None, incremental, response_timeout),
intake_rx,
abort_rx,
}
}
async fn next_generation(&self) -> GenerationIntake {
let TmEvent::Intake { request, admission } = self
.intake_rx
.recv_async()
.await
.expect("request submitted")
else {
panic!("expected generation intake");
};
let RuntimeRequest {
rid,
sink,
kind,
state: _,
} = request;
let RequestKind::Generate(request) = kind else {
panic!("expected generation request");
};
GenerationIntake {
request,
rid,
sink,
admission,
}
}
}
fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent {
ChunkEvent {
rid: rid.clone(),
token_ids: vec![token_id],
finish_reason: finished.then(stop_reason),
prompt_tokens: 3,
text: text.into(),
completion_tokens: 1,
extras: None,
}
}
fn stop_reason() -> FinishReason {
serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap()
}
#[tokio::test]
async fn text_generate_maps_request_and_streams_cumulative_responses() {
let harness = Harness::new(2, false, Duration::from_secs(1));
let response = harness
.service
.text_generate(Request::new(proto::TextGenerateRequest {
text: "prompt".into(),
stream: Some(true),
rid: Some("request-1".into()),
..Default::default()
}))
.await
.unwrap();
let mut stream = response.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(intake.request.text.as_deref(), Some("prompt"));
assert!(intake.request.input_ids.is_none());
assert!(intake.request.stream);
assert_eq!(intake.rid.client_facing(), "request-1");
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
.unwrap();
intake
.sink
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
.unwrap();
let first = stream.next().await.unwrap().unwrap();
assert_eq!(first.text, "Hel");
assert!(!first.finished);
assert_eq!(first.meta_info["id"], r#""request-1""#);
assert_eq!(first.meta_info["prompt_tokens"], "3");
assert_eq!(first.meta_info["completion_tokens"], "1");
assert_eq!(first.meta_info["finish_reason"], "null");
let finished = stream.next().await.unwrap().unwrap();
assert_eq!(finished.text, "Hello");
assert!(finished.finished);
assert_eq!(finished.meta_info["completion_tokens"], "2");
assert!(finished.meta_info["finish_reason"].contains(r#""type":"stop""#));
assert!(stream.next().await.is_none());
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn generate_streams_incremental_token_ids_with_cumulative_count() {
let harness = Harness::new(2, true, Duration::from_secs(1));
let response = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![4, 5],
stream: Some(true),
rid: Some("tokens".into()),
..Default::default()
}))
.await
.unwrap();
let mut stream = response.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..]));
assert!(intake.request.text.is_none());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false)))
.unwrap();
intake
.sink
.try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true)))
.unwrap();
let first = stream.next().await.unwrap().unwrap();
let finished = stream.next().await.unwrap().unwrap();
assert_eq!(first.output_ids, vec![10]);
assert_eq!(finished.output_ids, vec![11]);
assert_eq!(finished.meta_info["completion_tokens"], "2");
assert!(finished.finished);
}
#[tokio::test]
async fn stream_drop_aborts_admitted_request() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
rid: Some("cancel-me".into()),
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
.unwrap();
assert!(stream.next().await.unwrap().is_ok());
drop(stream);
let abort = harness.abort_rx.recv_async().await.unwrap();
assert_eq!(abort.rid().client_facing(), "cancel-me");
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn runtime_failure_is_an_in_stream_status_and_disarms_abort() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
"bad sampling".into(),
)))
.unwrap();
let error = stream.next().await.unwrap().unwrap_err();
assert_eq!(error.code(), Code::InvalidArgument);
drop(stream);
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() {
let harness = Harness::new(1, false, Duration::from_millis(10));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
rid: Some("too-slow".into()),
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
.unwrap();
assert!(stream.next().await.unwrap().is_ok());
let error = stream.next().await.unwrap().unwrap_err();
assert_eq!(error.code(), Code::DeadlineExceeded);
drop(stream);
let abort = harness.abort_rx.recv_async().await.unwrap();
assert_eq!(abort.rid().client_facing(), "too-slow");
}
#[tokio::test]
async fn closed_intake_is_a_top_level_unavailable_status() {
let (intake_tx, intake_rx) = flume::unbounded();
drop(intake_rx);
let (abort_tx, _abort_rx) = flume::unbounded();
let frontend = FrontendHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
response_capacity: 1,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: FrontendMetadata::default(),
},
);
let service = GrpcService::for_test(frontend, None, false, Duration::from_secs(1));
let result = service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
..Default::default()
}))
.await;
let error = match result {
Ok(_) => panic!("closed intake must reject the RPC"),
Err(error) => error,
};
assert_eq!(error.code(), Code::Unavailable);
}
#[tokio::test]
async fn new_proto_fields_and_engine_watch_are_explicitly_unsupported() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let text_error = harness
.service
.text_generate(Request::new(proto::TextGenerateRequest {
text: "hello".into(),
kv_hints: Some(Default::default()),
..Default::default()
}))
.await
.err()
.expect("KV hints must not be silently ignored");
assert_eq!(text_error.code(), Code::Unimplemented);
let token_error = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
kv_hints: Some(Default::default()),
..Default::default()
}))
.await
.err()
.expect("KV hints must not be silently ignored");
assert_eq!(token_error.code(), Code::Unimplemented);
let watch_error = harness
.service
.watch_engine_state(Request::new(proto::WatchEngineStateRequest {}))
.await
.err()
.expect("engine watch is outside the generation adapter");
assert_eq!(watch_error.code(), Code::Unimplemented);
assert!(harness.intake_rx.try_recv().is_err());
}
#[tokio::test]
async fn output_token_overflow_returns_error_instead_of_truncating() {
let harness = Harness::new(1, true, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![i32::MAX],
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(
intake.request.input_ids.as_deref(),
Some(&[i64::from(i32::MAX)][..])
);
intake
.sink
.try_send(ResponseItem::Frame(chunk(
&intake.rid,
"",
i64::from(i32::MAX) + 1,
false,
)))
.unwrap();
assert_eq!(
stream.next().await.unwrap().unwrap_err().code(),
Code::Internal
);
assert!(stream.next().await.is_none());
assert!(
matches!(harness.abort_rx.recv_async().await.unwrap(), AbortSource::Guard(rid) if rid == intake.rid)
);
}
+2
View File
@@ -13,8 +13,10 @@
mod api_server;
mod frontend;
mod grpc;
mod message;
mod multi_modality;
mod native_generation;
mod tokenizer_manager;
mod utils;
+29 -23
View File
@@ -315,10 +315,10 @@ impl SamplingParamsInput {
/// wins even when it explicitly carries the type's default or null.
pub fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
match self {
Self::One(params) => apply_preferred_to_one(params, preferred),
Self::One(params) => params.apply_preferred(preferred),
Self::Many(params) => params
.iter_mut()
.try_for_each(|params| apply_preferred_to_one(params, preferred)),
.try_for_each(|params| params.apply_preferred(preferred)),
}
}
@@ -327,27 +327,6 @@ impl SamplingParamsInput {
}
}
fn apply_preferred_to_one(
params: &mut SamplingParams,
preferred: &serde_json::Value,
) -> Result<(), String> {
let mut merged = preferred
.as_object()
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?
.clone();
let request_value = serde_json::to_value(&*params).map_err(|e| e.to_string())?;
let request = request_value
.as_object()
.ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?;
for field in &params.explicit_fields {
if let Some(value) = request.get(field) {
merged.insert(field.clone(), value.clone());
}
}
*params = sampling_params_from_value(serde_json::Value::Object(merged))?;
Ok(())
}
impl Default for SamplingParams {
fn default() -> Self {
// Each field reads the same `default()` the serde attribute above names,
@@ -393,6 +372,33 @@ impl Default for SamplingParams {
}
impl SamplingParams {
/// Record a field supplied by a non-Serde adapter. Preferred launch values
/// are merged underneath these fields just as they are underneath keys
/// present in an HTTP sampling object.
pub(crate) fn mark_explicit(&mut self, field: &'static str) {
self.explicit_fields.insert(field.to_owned());
}
/// Merge operator-provided sampling defaults beneath explicitly supplied
/// request fields. HTTP and gRPC adapters use the same precedence policy.
pub(crate) fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
let mut merged = preferred
.as_object()
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?
.clone();
let request_value = serde_json::to_value(&*self).map_err(|e| e.to_string())?;
let request = request_value
.as_object()
.ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?;
for field in &self.explicit_fields {
if let Some(value) = request.get(field) {
merged.insert(field.clone(), value.clone());
}
}
*self = sampling_params_from_value(serde_json::Value::Object(merged))?;
Ok(())
}
/// `__post_init__` → `normalize` → `verify`, the order
/// `TokenizerManager._create_tokenized_object` runs them in. `Err` is a
/// request-local 400. `skip_tokenizer_init` stands in for Python's
+160
View File
@@ -0,0 +1,160 @@
//! Shared wire semantics for SGLang-native generation responses.
//!
//! Native HTTP places these values in a JSON `meta_info` object, while
//! `runtime.v1` gRPC JSON-encodes each value into `map<string, string>`. Keeping
//! the logical shape here prevents the sibling adapters from drifting without
//! making either adapter depend on the other.
use crate::frontend::FrontendOutput;
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
pub(crate) fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value {
texts
.and_then(|texts| texts.get(j))
.map(|text| serde_json::json!(text))
.unwrap_or(serde_json::Value::Null)
}
/// A decoded-text column becomes the tuples' text source only when populated.
pub(crate) fn opt_texts(texts: &[String]) -> Option<&[String]> {
(!texts.is_empty()).then_some(texts)
}
/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel.
pub(crate) fn lp_value(value: f32) -> serde_json::Value {
if value.is_nan() {
serde_json::Value::Null
} else {
serde_json::json!(value)
}
}
/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples.
pub(crate) fn logprob_tuples(
values: &[f32],
token_ids: &[i32],
texts: Option<&[String]>,
) -> serde_json::Value {
let tuples = values
.iter()
.zip(token_ids.iter())
.enumerate()
.map(|(index, (&value, &token_id))| {
serde_json::json!([lp_value(value), token_id, text_slot(texts, index)])
})
.collect();
serde_json::Value::Array(tuples)
}
/// Ragged top-k / requested-token shape: one entry per position, or `null`
/// where that position's length is zero.
pub(crate) fn ragged_logprob_tuples(
values: &[f32],
token_ids: &[i32],
lengths: &[u32],
texts: Option<&[String]>,
) -> serde_json::Value {
let mut positions = Vec::with_capacity(lengths.len());
let mut offset = 0usize;
for &length in lengths {
let length = length as usize;
if length == 0 {
positions.push(serde_json::Value::Null);
} else {
// Bounds checks turn malformed parallel columns into a shortened
// row instead of panicking a transport worker.
let tuples = (offset..offset + length)
.filter_map(|index| {
Some(serde_json::json!([
lp_value(*values.get(index)?),
*token_ids.get(index)?,
text_slot(texts, index)
]))
})
.collect();
positions.push(serde_json::Value::Array(tuples));
}
offset += length;
}
serde_json::Value::Array(positions)
}
/// Reshape flat hidden-state values and per-row lengths into nested rows.
pub(crate) fn hidden_states_rows(values: &[f32], lengths: &[u32]) -> serde_json::Value {
let mut rows = Vec::with_capacity(lengths.len());
let mut offset = 0usize;
for &length in lengths {
let length = length as usize;
rows.push(serde_json::json!(
values.get(offset..offset + length).unwrap_or(&[])
));
offset += length;
}
serde_json::Value::Array(rows)
}
/// Build the heterogeneous native `meta_info` object before either adapter
/// chooses its wire encoding.
pub(crate) fn meta_info_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
let mut meta = serde_json::json!({
"id": rid,
"prompt_tokens": out.prompt_tokens,
"completion_tokens": out.completion_tokens,
"finish_reason": out.finish_reason,
});
let Some(extras) = out.extras.as_deref() else {
return meta;
};
// Python always sets input and output token logprobs together, including
// the empty input list emitted by a disaggregated decode node.
if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() {
meta["output_token_logprobs"] = logprob_tuples(
&extras.out_lp_val,
&extras.out_lp_idx,
opt_texts(&extras.out_lp_txt),
);
meta["input_token_logprobs"] = logprob_tuples(
&extras.in_lp_val,
&extras.in_lp_idx,
opt_texts(&extras.in_lp_txt),
);
}
if !extras.out_top_lens.is_empty() {
meta["output_top_logprobs"] = ragged_logprob_tuples(
&extras.out_top_val,
&extras.out_top_idx,
&extras.out_top_lens,
opt_texts(&extras.out_top_txt),
);
}
if !extras.in_top_lens.is_empty() {
meta["input_top_logprobs"] = ragged_logprob_tuples(
&extras.in_top_val,
&extras.in_top_idx,
&extras.in_top_lens,
opt_texts(&extras.in_top_txt),
);
}
if !extras.out_tid_lens.is_empty() {
meta["output_token_ids_logprobs"] = ragged_logprob_tuples(
&extras.out_tid_val,
&extras.out_tid_idx,
&extras.out_tid_lens,
opt_texts(&extras.out_tid_txt),
);
}
if !extras.in_tid_lens.is_empty() {
meta["input_token_ids_logprobs"] = ragged_logprob_tuples(
&extras.in_tid_val,
&extras.in_tid_idx,
&extras.in_tid_lens,
opt_texts(&extras.in_tid_txt),
);
}
if !extras.hidden_lens.is_empty() {
meta["hidden_states"] = hidden_states_rows(&extras.hidden_val, &extras.hidden_lens);
}
meta
}
+1 -1
View File
@@ -565,7 +565,7 @@ crate-type = ["cdylib"]
"sglang-server",
"sglang_server",
(),
(),
(grpc_proto,),
),
(
"sglang.srt.rust_extensions._grpc",