diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 7a19d7fc6e..0860fcf145 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -3809,10 +3809,12 @@ dependencies = [ "rustix", "serde", "serde_json", + "sglang-grpc-types", "sglang-mm", "socket2 0.6.5", "thiserror", "tokio", + "tonic", "tower 0.5.3", "tracing", "tracing-appender", diff --git a/rust/sglang-server/Cargo.toml b/rust/sglang-server/Cargo.toml index 80e1a5635f..e97f5cf66f 100644 --- a/rust/sglang-server/Cargo.toml +++ b/rust/sglang-server/Cargo.toml @@ -10,6 +10,9 @@ license.workspace = true python-module = "sglang.srt.rust_extensions._server" # Always build optimized, even for an editable install. debug = false +# The shared runtime.v1 crate lives outside rust/; include it in extension +# source fingerprints so protobuf changes cannot reuse a stale _server build. +source-inputs = ["../../proto"] [lib] name = "sglang_server" @@ -60,10 +63,12 @@ numpy = "0.29.0" regex-syntax = "=0.8.11" rmp-serde = "1" rmpv = { version = "1", features = ["with-serde"] } +sglang-grpc-types = { path = "../../proto" } # The pure-Rust core of the MM pipeline. `default-features = false` drops the # pyo3 bindings so it links as a plain rlib; renamed so `use sglang_mm::…` reads # naturally while the crate keeps its own artifact name in the shared target/. sglang_mm = { package = "sglang-mm", path = "../sglang-mm", default-features = false } +tonic = { workspace = true } [dev-dependencies] # `Router::oneshot` for handler-level router tests. diff --git a/rust/sglang-server/src/api_server/frame.rs b/rust/sglang-server/src/api_server/frame.rs index b669ceecaa..eae8c18c2d 100644 --- a/rust/sglang-server/src/api_server/frame.rs +++ b/rust/sglang-server/src/api_server/frame.rs @@ -5,78 +5,13 @@ //! and streams; it calls these per frame. use crate::frontend::FrontendOutput; +#[cfg(test)] use crate::message::response::ChunkExtras; - -/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when -/// `return_text_in_logprobs` supplied a text buffer, else `null`. -fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value { - texts - .and_then(|t| t.get(j)) - .map(|s| serde_json::json!(s)) - .unwrap_or(serde_json::Value::Null) -} - -/// A decoded-text column becomes the tuples' text source only when populated -/// (`return_text_in_logprobs`); empty → `None` → null text slots. -fn opt_texts(t: &[String]) -> Option<&[String]> { - (!t.is_empty()).then_some(t) -} - -/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel. -fn lp_value(v: f32) -> serde_json::Value { - if v.is_nan() { - serde_json::Value::Null - } else { - serde_json::json!(v) - } -} - -/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples. `texts` -/// (parallel to `idxs`) fills the text slot when set, else `null`. -fn logprob_tuples(vals: &[f32], idxs: &[i32], texts: Option<&[String]>) -> serde_json::Value { - let tuples: Vec = vals - .iter() - .zip(idxs.iter()) - .enumerate() - .map(|(j, (&v, &tid))| serde_json::json!([lp_value(v), tid, text_slot(texts, j)])) - .collect(); - serde_json::Value::Array(tuples) -} - -/// Ragged top-k / token-ids shape: one entry per position — a list of -/// `[logprob, token_id, text]` tuples, or `null` when `lens[p] == 0` (mirrors -/// `detokenize_top_logprobs_tokens`). `texts` is parallel to `vals`/`idxs`. -fn ragged_logprob_tuples( - vals: &[f32], - idxs: &[i32], - lens: &[u32], - texts: Option<&[String]>, -) -> serde_json::Value { - let mut positions = Vec::with_capacity(lens.len()); - let mut off = 0usize; - for &l in lens { - let l = l as usize; - if l == 0 { - positions.push(serde_json::Value::Null); - } else { - // Bounds-checked like `hidden_states_rows`: a header whose `lens` run - // past the value buffer would otherwise panic the api thread on an - // out-of-range index. - let tuples: Vec = (off..off + l) - .filter_map(|j| { - Some(serde_json::json!([ - lp_value(*vals.get(j)?), - *idxs.get(j)?, - text_slot(texts, j) - ])) - }) - .collect(); - positions.push(serde_json::Value::Array(tuples)); - } - off += l; - } - serde_json::Value::Array(positions) -} +use crate::native_generation::{ + hidden_states_rows, lp_value, meta_info_value, opt_texts, text_slot, +}; +#[cfg(test)] +use crate::native_generation::{logprob_tuples, ragged_logprob_tuples}; /// Append a flat family's `[logprob, token_id, text]` tuples to `dst`, comma /// separated and WITHOUT the enclosing brackets, so a cumulative frame can @@ -143,92 +78,17 @@ fn ragged_array_json(vals: &[f32], idxs: &[i32], lens: &[u32], texts: Option<&[S format!("[{body}]") } -/// Reshape flat hidden-state f32s + per-row lengths into `meta_info`'s nested -/// `list[list[float]]` (one row per output position). -fn hidden_states_rows(vals: &[f32], lens: &[u32]) -> serde_json::Value { - let mut rows = Vec::with_capacity(lens.len()); - let mut off = 0usize; - for &l in lens { - let l = l as usize; - // `get`, not a clamped index: clamping only the END leaves `off` past - // `vals.len()` after one over-long row, making the next range reversed - // (`start > end`) — which panics on the api thread rather than yielding - // an empty row. Same reasoning as the decoder's `take_f32`. - rows.push(serde_json::json!(vals.get(off..off + l).unwrap_or(&[]))); - off += l; - } - serde_json::Value::Array(rows) -} - /// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid` /// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is /// just the shard routing key. pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value { let mut v = serde_json::json!({ "text": out.text, - "meta_info": { - "id": rid, - "prompt_tokens": out.prompt_tokens, - "completion_tokens": out.completion_tokens, - // Full dict (type + matched + message + status_code + …), or null. - "finish_reason": out.finish_reason, - }, + "meta_info": meta_info_value(out, rid), }); if !out.token_ids.is_empty() { v["output_ids"] = serde_json::json!(out.token_ids); } - // Logprobs + hidden states ride behind the boxed extras (absent for a plain - // token/text frame). `[logprob, token_id, text|null]` tuples; text - // (`return_text_in_logprobs`) was decoded on the detok shard into `*_txt`. - let Some(ex) = out.extras.as_deref() else { - return v; - }; - // Python (`add_logprob_to_meta_info`) always sets input+output token - // logprobs together, empty lists included. A PD decode node never receives - // input logprobs (they belong to prefill), yet its response must still - // carry the key — the PD router keys its merge of prefill's - // `input_token_logprobs` on its presence. - if !ex.out_lp_val.is_empty() || !ex.in_lp_val.is_empty() { - v["meta_info"]["output_token_logprobs"] = - logprob_tuples(&ex.out_lp_val, &ex.out_lp_idx, opt_texts(&ex.out_lp_txt)); - v["meta_info"]["input_token_logprobs"] = - logprob_tuples(&ex.in_lp_val, &ex.in_lp_idx, opt_texts(&ex.in_lp_txt)); - } - if !ex.out_top_lens.is_empty() { - v["meta_info"]["output_top_logprobs"] = ragged_logprob_tuples( - &ex.out_top_val, - &ex.out_top_idx, - &ex.out_top_lens, - opt_texts(&ex.out_top_txt), - ); - } - if !ex.in_top_lens.is_empty() { - v["meta_info"]["input_top_logprobs"] = ragged_logprob_tuples( - &ex.in_top_val, - &ex.in_top_idx, - &ex.in_top_lens, - opt_texts(&ex.in_top_txt), - ); - } - if !ex.out_tid_lens.is_empty() { - v["meta_info"]["output_token_ids_logprobs"] = ragged_logprob_tuples( - &ex.out_tid_val, - &ex.out_tid_idx, - &ex.out_tid_lens, - opt_texts(&ex.out_tid_txt), - ); - } - if !ex.in_tid_lens.is_empty() { - v["meta_info"]["input_token_ids_logprobs"] = ragged_logprob_tuples( - &ex.in_tid_val, - &ex.in_tid_idx, - &ex.in_tid_lens, - opt_texts(&ex.in_tid_txt), - ); - } - if !ex.hidden_lens.is_empty() { - v["meta_info"]["hidden_states"] = hidden_states_rows(&ex.hidden_val, &ex.hidden_lens); - } v } @@ -427,33 +287,10 @@ impl OutputAccumulator { let _ = write!(self.ids_json, "{id}"); } - let o = &mut self.out; - o.text.push_str(&d.text); - o.token_ids.extend_from_slice(&d.token_ids); // token_ids doubles as output_ids - o.completion_tokens += d.completion_tokens; - o.prompt_tokens = d.prompt_tokens; // constant across the request - if d.finish_reason.is_some() { - o.finish_reason = d.finish_reason.clone(); - } - // Logprobs/hidden ride behind the boxed extras — most frames have none, so - // only allocate the accumulator's box once a delta actually carries some. let Some(de) = d.extras.as_deref() else { + self.out.append_delta(d); return; }; - let oe = o - .extras - .get_or_insert_with(|| Box::new(ChunkExtras::default())); - oe.out_lp_val.extend_from_slice(&de.out_lp_val); - oe.out_lp_idx.extend_from_slice(&de.out_lp_idx); - oe.out_top_val.extend_from_slice(&de.out_top_val); - oe.out_top_idx.extend_from_slice(&de.out_top_idx); - oe.out_top_lens.extend_from_slice(&de.out_top_lens); - oe.out_tid_val.extend_from_slice(&de.out_tid_val); - oe.out_tid_idx.extend_from_slice(&de.out_tid_idx); - oe.out_tid_lens.extend_from_slice(&de.out_tid_lens); - oe.out_lp_txt.extend_from_slice(&de.out_lp_txt); - oe.out_top_txt.extend_from_slice(&de.out_top_txt); - oe.out_tid_txt.extend_from_slice(&de.out_tid_txt); // Append THIS delta's tuples, indexed within the delta — equivalent to // indexing the accumulated arrays only while texts stay in lockstep, which // the guard below verifies. @@ -477,6 +314,45 @@ impl OutputAccumulator { &de.out_tid_lens, opt_texts(&de.out_tid_txt), ); + if !de.in_lp_val.is_empty() { + let mut body = String::new(); + push_logprob_tuples( + &mut body, + &de.in_lp_val, + &de.in_lp_idx, + opt_texts(&de.in_lp_txt), + ); + self.in_lp_json = Some(format!("[{body}]")); + } + // Input families ride once (prefill); `lens` non-empty marks their arrival. + if !de.in_top_lens.is_empty() { + self.in_top_json = Some(ragged_array_json( + &de.in_top_val, + &de.in_top_idx, + &de.in_top_lens, + opt_texts(&de.in_top_txt), + )); + } + if !de.in_tid_lens.is_empty() { + self.in_tid_json = Some(ragged_array_json( + &de.in_tid_val, + &de.in_tid_idx, + &de.in_tid_lens, + opt_texts(&de.in_tid_txt), + )); + } + // Hidden states are non-cumulative: the latest non-empty set wins. + if !de.hidden_lens.is_empty() { + self.hidden_json = + Some(hidden_states_rows(&de.hidden_val, &de.hidden_lens).to_string()); + } + + self.out.append_delta(d); + let oe = self + .out + .extras + .as_deref() + .expect("a delta with extras creates cumulative extras"); let lockstep = |txt: &Vec, val: &Vec| txt.is_empty() || txt.len() == val.len(); if !lockstep(&oe.out_lp_txt, &oe.out_lp_val) || !lockstep(&oe.out_top_txt, &oe.out_top_val) @@ -484,51 +360,6 @@ impl OutputAccumulator { { self.extras_memo_broken = true; } - if !de.in_lp_val.is_empty() { - oe.in_lp_val = de.in_lp_val.clone(); - oe.in_lp_idx = de.in_lp_idx.clone(); - oe.in_lp_txt = de.in_lp_txt.clone(); - let mut body = String::new(); - push_logprob_tuples( - &mut body, - &oe.in_lp_val, - &oe.in_lp_idx, - opt_texts(&oe.in_lp_txt), - ); - self.in_lp_json = Some(format!("[{body}]")); - } - // Input families ride once (prefill); `lens` non-empty marks their arrival. - if !de.in_top_lens.is_empty() { - oe.in_top_val = de.in_top_val.clone(); - oe.in_top_idx = de.in_top_idx.clone(); - oe.in_top_lens = de.in_top_lens.clone(); - oe.in_top_txt = de.in_top_txt.clone(); - self.in_top_json = Some(ragged_array_json( - &oe.in_top_val, - &oe.in_top_idx, - &oe.in_top_lens, - opt_texts(&oe.in_top_txt), - )); - } - if !de.in_tid_lens.is_empty() { - oe.in_tid_val = de.in_tid_val.clone(); - oe.in_tid_idx = de.in_tid_idx.clone(); - oe.in_tid_lens = de.in_tid_lens.clone(); - oe.in_tid_txt = de.in_tid_txt.clone(); - self.in_tid_json = Some(ragged_array_json( - &oe.in_tid_val, - &oe.in_tid_idx, - &oe.in_tid_lens, - opt_texts(&oe.in_tid_txt), - )); - } - // Hidden states are non-cumulative: the latest non-empty set wins. - if !de.hidden_lens.is_empty() { - oe.hidden_val = de.hidden_val.clone(); - oe.hidden_lens = de.hidden_lens.clone(); - self.hidden_json = - Some(hidden_states_rows(&oe.hidden_val, &oe.hidden_lens).to_string()); - } } /// Borrow the cumulative output for an intermediate streaming frame. diff --git a/rust/sglang-server/src/frontend/contract.rs b/rust/sglang-server/src/frontend/contract.rs index b8b544f408..037c3b7583 100644 --- a/rust/sglang-server/src/frontend/contract.rs +++ b/rust/sglang-server/src/frontend/contract.rs @@ -61,6 +61,90 @@ impl From for FrontendOutput { } } +impl FrontendOutput { + /// Fold one runtime delta into this cumulative native-generation output. + /// + /// Runtime events are always incremental. Native HTTP and `runtime.v1` + /// gRPC independently decide whether to expose those deltas or the + /// cumulative result, so the fold itself lives at their shared semantic + /// boundary rather than in either wire adapter. + pub(crate) fn append_delta(&mut self, delta: &Self) { + self.text.push_str(&delta.text); + self.token_ids.extend_from_slice(&delta.token_ids); + self.completion_tokens += delta.completion_tokens; + self.prompt_tokens = delta.prompt_tokens; + if delta.finish_reason.is_some() { + self.finish_reason = delta.finish_reason.clone(); + } + + let Some(delta_extras) = delta.extras.as_deref() else { + return; + }; + let extras = self + .extras + .get_or_insert_with(|| Box::new(ChunkExtras::default())); + + extras + .out_lp_val + .extend_from_slice(&delta_extras.out_lp_val); + extras + .out_lp_idx + .extend_from_slice(&delta_extras.out_lp_idx); + extras + .out_top_val + .extend_from_slice(&delta_extras.out_top_val); + extras + .out_top_idx + .extend_from_slice(&delta_extras.out_top_idx); + extras + .out_top_lens + .extend_from_slice(&delta_extras.out_top_lens); + extras + .out_tid_val + .extend_from_slice(&delta_extras.out_tid_val); + extras + .out_tid_idx + .extend_from_slice(&delta_extras.out_tid_idx); + extras + .out_tid_lens + .extend_from_slice(&delta_extras.out_tid_lens); + extras + .out_lp_txt + .extend_from_slice(&delta_extras.out_lp_txt); + extras + .out_top_txt + .extend_from_slice(&delta_extras.out_top_txt); + extras + .out_tid_txt + .extend_from_slice(&delta_extras.out_tid_txt); + + // Input logprobs arrive once (during prefill). A later non-empty value + // replaces the earlier one, matching the existing native HTTP fold. + if !delta_extras.in_lp_val.is_empty() { + extras.in_lp_val = delta_extras.in_lp_val.clone(); + extras.in_lp_idx = delta_extras.in_lp_idx.clone(); + extras.in_lp_txt = delta_extras.in_lp_txt.clone(); + } + if !delta_extras.in_top_lens.is_empty() { + extras.in_top_val = delta_extras.in_top_val.clone(); + extras.in_top_idx = delta_extras.in_top_idx.clone(); + extras.in_top_lens = delta_extras.in_top_lens.clone(); + extras.in_top_txt = delta_extras.in_top_txt.clone(); + } + if !delta_extras.in_tid_lens.is_empty() { + extras.in_tid_val = delta_extras.in_tid_val.clone(); + extras.in_tid_idx = delta_extras.in_tid_idx.clone(); + extras.in_tid_lens = delta_extras.in_tid_lens.clone(); + extras.in_tid_txt = delta_extras.in_tid_txt.clone(); + } + // Hidden states are non-cumulative: the latest complete set wins. + if !delta_extras.hidden_lens.is_empty() { + extras.hidden_val = delta_extras.hidden_val.clone(); + extras.hidden_lens = delta_extras.hidden_lens.clone(); + } + } +} + /// Semantic result of a deep frontend health check. /// /// Expected lifecycle states are values rather than transport errors so each diff --git a/rust/sglang-server/src/grpc/convert.rs b/rust/sglang-server/src/grpc/convert.rs new file mode 100644 index 0000000000..ff1aa9884e --- /dev/null +++ b/rust/sglang-server/src/grpc/convert.rs @@ -0,0 +1,531 @@ +//! `runtime.v1` request conversion for the Rust frontend. +//! +//! The canonical protobuf is wider than the Rust frontend today. Fields with +//! no internal representation are rejected explicitly instead of being +//! silently ignored or expanding frontend behavior in this wiring change. + +use sglang_grpc_types::sglang::runtime::v1 as proto; +use tonic::Status; + +use crate::frontend::FrontendRequest; +use crate::message::config::PreferredSamplingParams; +use crate::message::ids::Rid; +use crate::message::sampling::SamplingParams; + +type ConvertResult = Result; + +#[derive(Debug, thiserror::Error)] +pub(super) enum ConvertError { + #[error("{0}")] + InvalidArgument(String), + #[error("{0}")] + Unsupported(String), + #[error("{0}")] + Internal(String), +} + +impl ConvertError { + fn invalid_argument(message: impl Into) -> Self { + Self::InvalidArgument(message.into()) + } + + fn internal(message: impl Into) -> Self { + Self::Internal(message.into()) + } +} + +impl From for Status { + fn from(error: ConvertError) -> Self { + match error { + ConvertError::InvalidArgument(message) => Self::invalid_argument(message), + ConvertError::Unsupported(message) => Self::unimplemented(message), + ConvertError::Internal(message) => Self::internal(message), + } + } +} + +pub(super) fn text_generate( + request: proto::TextGenerateRequest, + preferred: Option<&PreferredSamplingParams>, +) -> ConvertResult { + let proto::TextGenerateRequest { + text, + sampling_params, + stream, + return_logprob, + top_logprobs_num, + logprob_start_len, + return_text_in_logprobs, + rid, + lora_path, + routing_key, + routed_dp_rank, + trace_headers, + session_id, + disaggregated_params, + priority, + require_reasoning, + max_thinking_tokens, + kv_hints, + } = request; + + reject_unsupported([ + ("kv_hints", kv_hints.is_some()), + ("lora_path", lora_path.is_some()), + ("routing_key", routing_key.is_some()), + ("trace_headers", !trace_headers.is_empty()), + ("session_id", session_id.is_some()), + ("priority", priority.is_some()), + ("require_reasoning", require_reasoning.is_some()), + ("max_thinking_tokens", max_thinking_tokens.is_some()), + ])?; + + let (bootstrap_host, bootstrap_port, bootstrap_room) = + disaggregated_fields(disaggregated_params); + Ok(FrontendRequest { + rid: request_id(rid), + text: Some(text), + input_ids: None, + skip_special_tokens: false, + sampling_params: convert_sampling_params(sampling_params, preferred)?, + stream: stream.unwrap_or(false), + return_logprob: return_logprob.unwrap_or(false), + logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1), + top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0), + token_ids_logprob: None, + return_sampling_mask: false, + return_hidden_states: false, + return_text_in_logprobs, + bootstrap_host, + bootstrap_port, + bootstrap_room, + bootstrap_pair_key: None, + decode_tp_size: None, + routed_dp_rank: routed_dp_rank.map(i64::from), + disagg_prefill_dp_rank: None, + mm: None, + mm_buffers: Vec::new(), + }) +} + +pub(super) fn generate( + request: proto::GenerateRequest, + preferred: Option<&PreferredSamplingParams>, +) -> ConvertResult { + let proto::GenerateRequest { + input_ids, + sampling_params, + stream, + return_logprob, + top_logprobs_num, + logprob_start_len, + rid, + lora_path, + routing_key, + routed_dp_rank, + trace_headers, + session_id, + disaggregated_params, + priority, + require_reasoning, + max_thinking_tokens, + kv_hints, + } = request; + + reject_unsupported([ + ("kv_hints", kv_hints.is_some()), + ("lora_path", lora_path.is_some()), + ("routing_key", routing_key.is_some()), + ("trace_headers", !trace_headers.is_empty()), + ("session_id", session_id.is_some()), + ("priority", priority.is_some()), + ("require_reasoning", require_reasoning.is_some()), + ("max_thinking_tokens", max_thinking_tokens.is_some()), + ])?; + if input_ids.is_empty() { + return Err(ConvertError::invalid_argument("input_ids cannot be empty")); + } + + let (bootstrap_host, bootstrap_port, bootstrap_room) = + disaggregated_fields(disaggregated_params); + Ok(FrontendRequest { + rid: request_id(rid), + text: None, + input_ids: Some(input_ids.into_iter().map(i64::from).collect()), + skip_special_tokens: false, + sampling_params: convert_sampling_params(sampling_params, preferred)?, + stream: stream.unwrap_or(false), + return_logprob: return_logprob.unwrap_or(false), + logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1), + top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0), + token_ids_logprob: None, + return_sampling_mask: false, + return_hidden_states: false, + return_text_in_logprobs: None, + bootstrap_host, + bootstrap_port, + bootstrap_room, + bootstrap_pair_key: None, + decode_tp_size: None, + routed_dp_rank: routed_dp_rank.map(i64::from), + disagg_prefill_dp_rank: None, + mm: None, + mm_buffers: Vec::new(), + }) +} + +fn request_id(rid: Option) -> Rid { + rid.map_or_else(Rid::new, |rid| Rid::from_client(&rid)) +} + +fn reject_unsupported(fields: [(&'static str, bool); N]) -> ConvertResult<()> { + let unsupported = fields + .into_iter() + .filter_map(|(name, present)| present.then_some(name)) + .collect::>(); + if unsupported.is_empty() { + return Ok(()); + } + Err(ConvertError::Unsupported(format!( + "the Rust frontend does not yet support runtime.v1 field(s): {}", + unsupported.join(", ") + ))) +} + +fn disaggregated_fields( + params: Option, +) -> (Option, Option, Option) { + params.map_or((None, None, None), |params| { + ( + Some(params.bootstrap_host), + Some(i64::from(params.bootstrap_port)), + Some(params.bootstrap_room), + ) + }) +} + +/// Merge launch-time preferred values first, then overwrite only fields that +/// are explicitly present in protobuf. This is the same precedence used by +/// the Python-backed gRPC path; proto3 repeated fields cannot distinguish an +/// omitted list from an explicit empty list, so empty stop lists remain absent. +#[allow(deprecated)] +fn convert_sampling_params( + params: Option, + preferred: Option<&PreferredSamplingParams>, +) -> ConvertResult { + let mut output = SamplingParams::default(); + + let Some(params) = params else { + if let Some(preferred) = preferred { + output.apply_preferred(&preferred.0).map_err(|error| { + ConvertError::internal(format!("invalid preferred_sampling_params: {error}")) + })?; + } + return Ok(output); + }; + if params.guided_decoding.is_some() && (params.json_schema.is_some() || params.regex.is_some()) + { + return Err(ConvertError::invalid_argument( + "legacy json_schema/regex cannot be combined with guided_decoding", + )); + } + + if let Some(value) = params.temperature { + output.temperature = f64::from(value); + output.mark_explicit("temperature"); + } + if let Some(value) = params.top_p { + output.top_p = f64::from(value); + output.mark_explicit("top_p"); + } + if let Some(value) = params.top_k { + output.top_k = i64::from(value); + output.mark_explicit("top_k"); + } + if let Some(value) = params.min_p { + output.min_p = f64::from(value); + output.mark_explicit("min_p"); + } + if let Some(value) = params.frequency_penalty { + output.frequency_penalty = f64::from(value); + output.mark_explicit("frequency_penalty"); + } + if let Some(value) = params.presence_penalty { + output.presence_penalty = f64::from(value); + output.mark_explicit("presence_penalty"); + } + if let Some(value) = params.repetition_penalty { + output.repetition_penalty = f64::from(value); + output.mark_explicit("repetition_penalty"); + } + if let Some(value) = params.max_new_tokens { + output.max_new_tokens = Some(i64::from(value)); + output.mark_explicit("max_new_tokens"); + } + if let Some(value) = params.min_new_tokens { + output.min_new_tokens = i64::from(value); + output.mark_explicit("min_new_tokens"); + } + if !params.stop.is_empty() { + output.stop = Some(crate::message::types::OneOrMany::Many(params.stop)); + output.mark_explicit("stop"); + } + if !params.stop_token_ids.is_empty() { + output.stop_token_ids = Some(params.stop_token_ids.into_iter().map(i64::from).collect()); + output.mark_explicit("stop_token_ids"); + } + if let Some(value) = params.ignore_eos { + output.ignore_eos = value; + output.mark_explicit("ignore_eos"); + } + if let Some(value) = params.n { + output.n = i64::from(value); + output.mark_explicit("n"); + } + if let Some(value) = params.seed { + output.sampling_seed = Some(value); + output.mark_explicit("sampling_seed"); + } + + if let Some(guided) = params.guided_decoding { + apply_guided_decoding(&mut output, guided)?; + } else { + if let Some(value) = params.json_schema { + if value.is_empty() { + return Err(ConvertError::invalid_argument( + "legacy json_schema must not be empty", + )); + } + output.json_schema = Some(value); + output.mark_explicit("json_schema"); + } + if let Some(value) = params.regex { + if value.is_empty() { + return Err(ConvertError::invalid_argument( + "legacy regex must not be empty", + )); + } + output.regex = Some(value); + output.mark_explicit("regex"); + } + } + + if let Some(preferred) = preferred { + output.apply_preferred(&preferred.0).map_err(|error| { + ConvertError::internal(format!("invalid preferred_sampling_params: {error}")) + })?; + } + + Ok(output) +} + +fn apply_guided_decoding( + output: &mut SamplingParams, + guided: proto::GuidedDecoding, +) -> ConvertResult<()> { + use proto::guided_decoding::Constraint; + + match guided.constraint { + Some(Constraint::JsonSchema(value)) if !value.is_empty() => { + output.json_schema = Some(value); + output.mark_explicit("json_schema"); + } + Some(Constraint::Regex(value)) if !value.is_empty() => { + output.regex = Some(value); + output.mark_explicit("regex"); + } + Some(Constraint::Ebnf(value)) if !value.is_empty() => { + output.ebnf = Some(value); + output.mark_explicit("ebnf"); + } + Some(Constraint::StructuralTag(value)) if !value.is_empty() => { + output.structural_tag = Some(value); + output.mark_explicit("structural_tag"); + } + Some(Constraint::Choice(choice)) + if !choice.values.is_empty() && choice.values.iter().all(|value| !value.is_empty()) => + { + let alternatives = choice + .values + .iter() + .map(|value| regex_escape_literal(value)) + .collect::>() + .join("|"); + output.regex = Some(format!("(?:{alternatives})")); + output.mark_explicit("regex"); + } + Some(Constraint::Choice(_)) => { + return Err(ConvertError::invalid_argument( + "guided choice must contain only non-empty values", + )); + } + Some(_) => { + return Err(ConvertError::invalid_argument( + "guided decoding constraint must not be empty", + )); + } + None => { + return Err(ConvertError::invalid_argument( + "guided decoding constraint must be specified", + )); + } + } + Ok(()) +} + +fn regex_escape_literal(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + if matches!( + character, + '.' | '+' | '*' | '?' | '^' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '\\' + ) { + escaped.push('\\'); + } + escaped.push(character); + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use tonic::Code; + + #[test] + fn text_request_maps_supported_fields_and_preferred_sampling() { + let preferred = PreferredSamplingParams(serde_json::json!({ + "temperature": 0.2, + "top_p": 0.7, + "max_new_tokens": 64, + })); + let request = proto::TextGenerateRequest { + text: "hello".into(), + sampling_params: Some(proto::SamplingParams { + temperature: Some(0.8), + seed: Some(7), + ..Default::default() + }), + stream: Some(true), + return_logprob: Some(true), + top_logprobs_num: Some(3), + logprob_start_len: Some(2), + return_text_in_logprobs: Some(true), + rid: Some("client-id".into()), + routed_dp_rank: Some(4), + disaggregated_params: Some(proto::DisaggregatedParams { + bootstrap_host: "10.0.0.1".into(), + bootstrap_port: 8998, + bootstrap_room: 12, + }), + ..Default::default() + }; + + let request = text_generate(request, Some(&preferred)).unwrap(); + assert_eq!(request.rid.client_facing(), "client-id"); + assert_eq!(request.text.as_deref(), Some("hello")); + assert!(request.input_ids.is_none()); + assert!(request.stream); + assert!(request.return_logprob); + assert_eq!(request.top_logprobs_num, 3); + assert_eq!(request.logprob_start_len, 2); + assert_eq!(request.return_text_in_logprobs, Some(true)); + assert_eq!(request.routed_dp_rank, Some(4)); + assert_eq!(request.bootstrap_host.as_deref(), Some("10.0.0.1")); + assert_eq!(request.bootstrap_port, Some(8998)); + assert_eq!(request.bootstrap_room, Some(12)); + assert_eq!(request.sampling_params.temperature, f64::from(0.8_f32)); + assert_eq!(request.sampling_params.top_p, 0.7); + assert_eq!(request.sampling_params.max_new_tokens, Some(64)); + assert_eq!(request.sampling_params.sampling_seed, Some(7)); + } + + #[test] + fn token_request_validates_input_and_rejects_unported_fields() { + let empty = Status::from(generate(proto::GenerateRequest::default(), None).unwrap_err()); + assert_eq!(empty.code(), Code::InvalidArgument); + + let unsupported = Status::from( + generate( + proto::GenerateRequest { + input_ids: vec![1], + routing_key: Some("worker-a".into()), + priority: Some(1), + ..Default::default() + }, + None, + ) + .unwrap_err(), + ); + assert_eq!(unsupported.code(), Code::Unimplemented); + assert!(unsupported.message().contains("routing_key, priority")); + } + + #[test] + fn absent_sampling_params_still_uses_preferred_values() { + let preferred = PreferredSamplingParams(serde_json::json!({ + "temperature": 0.25, + "max_new_tokens": 32, + })); + let request = generate( + proto::GenerateRequest { + input_ids: vec![1], + ..Default::default() + }, + Some(&preferred), + ) + .unwrap(); + + assert_eq!(request.sampling_params.temperature, 0.25); + assert_eq!(request.sampling_params.max_new_tokens, Some(32)); + } + + #[test] + fn guided_choice_preserves_existing_runtime_v1_mapping() { + let request = generate( + proto::GenerateRequest { + input_ids: vec![1], + sampling_params: Some(proto::SamplingParams { + guided_decoding: Some(proto::GuidedDecoding { + constraint: Some(proto::guided_decoding::Constraint::Choice( + proto::ChoiceConstraint { + values: vec!["a+b".into(), "x.y".into()], + }, + )), + }), + ..Default::default() + }), + ..Default::default() + }, + None, + ) + .unwrap(); + assert_eq!( + request.sampling_params.regex.as_deref(), + Some("(?:a\\+b|x\\.y)") + ); + } + + #[test] + #[allow(deprecated)] + fn guided_and_legacy_constraints_cannot_be_combined() { + let error = Status::from( + generate( + proto::GenerateRequest { + input_ids: vec![1], + sampling_params: Some(proto::SamplingParams { + regex: Some("a".into()), + guided_decoding: Some(proto::GuidedDecoding { + constraint: Some(proto::guided_decoding::Constraint::Regex("b".into())), + }), + ..Default::default() + }), + ..Default::default() + }, + None, + ) + .unwrap_err(), + ); + assert_eq!(error.code(), Code::InvalidArgument); + } +} diff --git a/rust/sglang-server/src/grpc/mod.rs b/rust/sglang-server/src/grpc/mod.rs new file mode 100644 index 0000000000..fdeb9bb6f0 --- /dev/null +++ b/rust/sglang-server/src/grpc/mod.rs @@ -0,0 +1,318 @@ +//! Tonic adapter for the canonical `sglang.runtime.v1` API. +//! +//! This module only translates protobuf requests and responses. The shared +//! [`FrontendHandle`] owns preprocessing, admission, runtime communication, and +//! request cancellation; listener construction and server lifecycle are added +//! by the next stack layer. +//! +//! The thin Tonic-service structure, streamed response approach, and test +//! strategy build on Rain Jiang's multi-protocol prototype in +//! `sgl-project/sglang#36923`. This stack keeps those ideas while using the +//! canonical `runtime.v1` contract and the narrower [`FrontendHandle`] seam. + +use std::pin::Pin; +use std::time::Duration; + +use futures::Stream; +use sglang_grpc_types::sglang::runtime::v1 as proto; +use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService; +use tonic::{Request, Response, Status}; + +use crate::frontend::FrontendHandle; +use crate::message::config::{PreferredSamplingParams, ServerArgs}; + +mod convert; +mod response; + +#[cfg(test)] +mod tests; + +#[cfg_attr(test, allow(dead_code))] +const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300); + +type ResponseStream = Pin> + Send + 'static>>; + +/// Narrow snapshot of launch policy required to preserve the existing +/// `runtime.v1` generation behavior. It intentionally excludes listener, +/// authentication, TLS, and lifecycle configuration. +#[derive(Clone)] +struct AdapterConfig { + preferred_sampling_params: Option, + incremental_streaming_output: bool, + response_timeout: Duration, +} + +/// Tonic-facing implementation backed by the transport-neutral Rust frontend. +/// +/// The listener follow-up constructs this service. The temporary dead-code +/// allowance keeps this adapter-only stack layer warning-clean on its own. +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) struct GrpcService { + frontend: FrontendHandle, + config: AdapterConfig, +} + +#[cfg_attr(not(test), allow(dead_code))] +impl GrpcService { + #[cfg_attr(test, allow(dead_code))] + pub(crate) fn new(frontend: FrontendHandle, server_args: &ServerArgs) -> Self { + Self { + frontend, + config: AdapterConfig { + preferred_sampling_params: server_args.preferred_sampling_params.clone(), + incremental_streaming_output: server_args.incremental_streaming_output, + response_timeout: DEFAULT_RESPONSE_TIMEOUT, + }, + } + } + + #[cfg(test)] + fn for_test( + frontend: FrontendHandle, + preferred_sampling_params: Option, + incremental_streaming_output: bool, + response_timeout: Duration, + ) -> Self { + Self { + frontend, + config: AdapterConfig { + preferred_sampling_params, + incremental_streaming_output, + response_timeout, + }, + } + } +} + +fn unimplemented_rpc(name: &'static str) -> Status { + Status::unimplemented(format!( + "{name} is not implemented by the Rust frontend gRPC adapter" + )) +} + +#[tonic::async_trait] +impl SglangService for GrpcService { + type TextGenerateStream = ResponseStream; + + async fn text_generate( + &self, + request: Request, + ) -> Result, Status> { + let request = convert::text_generate( + request.into_inner(), + self.config.preferred_sampling_params.as_ref(), + ) + .map_err(Status::from)?; + let call = self + .frontend + .generate(request) + .await + .map_err(response::status)?; + Ok(Response::new(response::text_generate_stream( + call, + self.config.incremental_streaming_output, + self.config.response_timeout, + ))) + } + + type GenerateStream = ResponseStream; + + async fn generate( + &self, + request: Request, + ) -> Result, Status> { + let request = convert::generate( + request.into_inner(), + self.config.preferred_sampling_params.as_ref(), + ) + .map_err(Status::from)?; + let call = self + .frontend + .generate(request) + .await + .map_err(response::status)?; + Ok(Response::new(response::generate_stream( + call, + self.config.incremental_streaming_output, + self.config.response_timeout, + ))) + } + + // runtime.v1 is shared with the broader Python-backed service, so its + // generated trait contains more operations than this initial Rust adapter. + // Keep that support boundary explicit: adding an RPC is a deliberate future + // change, never an accidental empty success. + async fn text_embed( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("text_embed")) + } + + async fn embed( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("embed")) + } + + async fn classify( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("classify")) + } + + async fn tokenize( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("tokenize")) + } + + async fn detokenize( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("detokenize")) + } + + async fn health_check( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("health_check")) + } + + async fn get_model_info( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("get_model_info")) + } + + async fn get_server_info( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("get_server_info")) + } + + async fn list_models( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("list_models")) + } + + async fn get_load( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("get_load")) + } + + async fn abort( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("abort")) + } + + async fn flush_cache( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("flush_cache")) + } + + async fn pause_generation( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("pause_generation")) + } + + type WatchEngineStateStream = ResponseStream; + + async fn watch_engine_state( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("watch_engine_state")) + } + + async fn continue_generation( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("continue_generation")) + } + + type ChatCompleteStream = ResponseStream; + + async fn chat_complete( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("chat_complete")) + } + + type CompleteStream = ResponseStream; + + async fn complete( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("complete")) + } + + async fn open_ai_embed( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("open_ai_embed")) + } + + async fn open_ai_classify( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("open_ai_classify")) + } + + async fn score( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("score")) + } + + async fn rerank( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("rerank")) + } + + async fn start_profile( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("start_profile")) + } + + async fn stop_profile( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("stop_profile")) + } + + async fn update_weights_from_disk( + &self, + _request: Request, + ) -> Result, Status> { + Err(unimplemented_rpc("update_weights_from_disk")) + } +} diff --git a/rust/sglang-server/src/grpc/response.rs b/rust/sglang-server/src/grpc/response.rs new file mode 100644 index 0000000000..93ee25b2c9 --- /dev/null +++ b/rust/sglang-server/src/grpc/response.rs @@ -0,0 +1,200 @@ +//! Semantic frontend events to `runtime.v1` stream items and statuses. + +use std::collections::HashMap; +use std::time::Duration; + +use sglang_grpc_types::sglang::runtime::v1 as proto; +use tonic::{Code, Status}; + +use super::ResponseStream; +use crate::frontend::{ + FrontendCall, FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, +}; +use crate::native_generation::meta_info_value; + +pub(super) fn text_generate_stream( + call: FrontendCall, + incremental: bool, + response_timeout: Duration, +) -> ResponseStream { + generation_stream( + call, + incremental, + response_timeout, + |output, meta_info, finished| { + Ok(proto::TextGenerateResponse { + text: output.text.clone(), + meta_info, + finished, + }) + }, + ) +} + +pub(super) fn generate_stream( + call: FrontendCall, + incremental: bool, + response_timeout: Duration, +) -> ResponseStream { + generation_stream( + call, + incremental, + response_timeout, + |output, meta_info, finished| { + Ok(proto::GenerateResponse { + output_ids: output + .token_ids + .iter() + .copied() + .map(i32::try_from) + .collect::, _>>() + .map_err(|_| { + Status::internal("output token ID exceeds runtime.v1 int32 range") + })?, + meta_info, + finished, + }) + }, + ) +} + +fn generation_stream( + mut call: FrontendCall, + incremental: bool, + response_timeout: Duration, + build: F, +) -> ResponseStream +where + T: Send + 'static, + F: Fn(&FrontendOutput, HashMap, bool) -> Result + Send + 'static, +{ + let public_id = call.public_id().to_owned(); + let stream = async_stream::stream! { + let mut cumulative = FrontendOutput::default(); + let mut cumulative_completion_tokens = 0; + loop { + let event = match tokio::time::timeout(response_timeout, call.recv()).await { + Ok(Some(event)) => event, + // The loop stops on terminal events, so this is defensive only. + Ok(None) => break, + Err(_) => { + yield Err(Status::deadline_exceeded("Stream chunk timed out")); + break; + } + }; + + let (mut delta, finished) = match event { + FrontendEvent::Delta(output) => (output, false), + FrontendEvent::Finished(output) => (output, true), + FrontendEvent::Failed(error) => { + yield Err(status(error)); + break; + } + }; + + let output = if incremental { + // Python's incremental native stream still reports the cumulative + // completion-token count in every frame. Do not retain the full + // generated output just to calculate that one cumulative field. + cumulative_completion_tokens += delta.completion_tokens; + delta.completion_tokens = cumulative_completion_tokens; + &delta + } else { + cumulative.append_delta(&delta); + &cumulative + }; + let meta_info = grpc_meta_info(output, &public_id); + match build(output, meta_info, finished) { + Ok(response) => yield Ok(response), + Err(error) => { + yield Err(error); + break; + } + } + if finished { + break; + } + } + // `call` remains owned by this stream. Dropping a non-terminal stream or + // timing out drops the armed FrontendCall and uses the core abort lane; + // observing a terminal event has already disarmed it. + }; + Box::pin(stream) +} + +/// runtime.v1 represents heterogeneous metadata as strings. Match the existing +/// Python-backed server by JSON-encoding each value independently—including +/// string values such as the request ID. +fn grpc_meta_info(output: &FrontendOutput, public_id: &str) -> HashMap { + let serde_json::Value::Object(meta_info) = meta_info_value(output, public_id) else { + unreachable!("native meta_info is always a JSON object"); + }; + meta_info + .into_iter() + .map(|(key, value)| (key, value.to_string())) + .collect() +} + +pub(super) fn status(error: FrontendError) -> Status { + let code = match error.kind() { + FrontendErrorKind::InvalidArgument => Code::InvalidArgument, + FrontendErrorKind::NotFound => Code::NotFound, + FrontendErrorKind::FailedPrecondition => Code::FailedPrecondition, + FrontendErrorKind::ResourceExhausted => Code::ResourceExhausted, + FrontendErrorKind::Cancelled => Code::Cancelled, + FrontendErrorKind::DeadlineExceeded => Code::DeadlineExceeded, + FrontendErrorKind::Unavailable => Code::Unavailable, + FrontendErrorKind::Internal => Code::Internal, + }; + Status::new(code, error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn semantic_errors_map_to_canonical_grpc_codes() { + let cases = [ + ( + FrontendError::InvalidArgument("bad".into()), + Code::InvalidArgument, + ), + ( + FrontendError::RuntimeRejected { + kind: FrontendErrorKind::NotFound, + message: "missing".into(), + legacy_http_status: 404, + }, + Code::NotFound, + ), + ( + FrontendError::RuntimeRejected { + kind: FrontendErrorKind::FailedPrecondition, + message: "not ready".into(), + legacy_http_status: 412, + }, + Code::FailedPrecondition, + ), + ( + FrontendError::Overloaded("full".into()), + Code::ResourceExhausted, + ), + (FrontendError::Cancelled("gone".into()), Code::Cancelled), + ( + FrontendError::RuntimeRejected { + kind: FrontendErrorKind::DeadlineExceeded, + message: "late".into(), + legacy_http_status: 504, + }, + Code::DeadlineExceeded, + ), + (FrontendError::Unavailable, Code::Unavailable), + (FrontendError::Internal("bug".into()), Code::Internal), + ]; + + for (error, expected) in cases { + assert_eq!(status(error).code(), expected); + } + } +} diff --git a/rust/sglang-server/src/grpc/tests.rs b/rust/sglang-server/src/grpc/tests.rs new file mode 100644 index 0000000000..ff6b718297 --- /dev/null +++ b/rust/sglang-server/src/grpc/tests.rs @@ -0,0 +1,361 @@ +use std::time::Duration; + +use futures::StreamExt; +use sglang_grpc_types::sglang::runtime::v1 as proto; +use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService; +use tonic::{Code, Request}; + +use super::GrpcService; +use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata}; +use crate::message::finish_reason::FinishReason; +use crate::message::ids::Rid; +use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind}; +use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink}; +use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent}; + +struct Harness { + service: GrpcService, + intake_rx: flume::Receiver, + abort_rx: flume::Receiver, +} + +struct GenerationIntake { + request: Box, + rid: Rid, + sink: ResponseSink, + admission: RequestAdmission, +} + +impl Harness { + fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self { + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, abort_rx) = flume::unbounded(); + let frontend = FrontendHandle::new( + intake_tx, + abort_tx, + FrontendConfig { + response_capacity, + response_activity: Default::default(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: FrontendMetadata::default(), + }, + ); + Self { + service: GrpcService::for_test(frontend, None, incremental, response_timeout), + intake_rx, + abort_rx, + } + } + + async fn next_generation(&self) -> GenerationIntake { + let TmEvent::Intake { request, admission } = self + .intake_rx + .recv_async() + .await + .expect("request submitted") + else { + panic!("expected generation intake"); + }; + let RuntimeRequest { + rid, + sink, + kind, + state: _, + } = request; + let RequestKind::Generate(request) = kind else { + panic!("expected generation request"); + }; + GenerationIntake { + request, + rid, + sink, + admission, + } + } +} + +fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent { + ChunkEvent { + rid: rid.clone(), + token_ids: vec![token_id], + finish_reason: finished.then(stop_reason), + prompt_tokens: 3, + text: text.into(), + completion_tokens: 1, + extras: None, + } +} + +fn stop_reason() -> FinishReason { + serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap() +} + +#[tokio::test] +async fn text_generate_maps_request_and_streams_cumulative_responses() { + let harness = Harness::new(2, false, Duration::from_secs(1)); + let response = harness + .service + .text_generate(Request::new(proto::TextGenerateRequest { + text: "prompt".into(), + stream: Some(true), + rid: Some("request-1".into()), + ..Default::default() + })) + .await + .unwrap(); + let mut stream = response.into_inner(); + + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + assert_eq!(intake.request.text.as_deref(), Some("prompt")); + assert!(intake.request.input_ids.is_none()); + assert!(intake.request.stream); + assert_eq!(intake.rid.client_facing(), "request-1"); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false))) + .unwrap(); + intake + .sink + .try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true))) + .unwrap(); + + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.text, "Hel"); + assert!(!first.finished); + assert_eq!(first.meta_info["id"], r#""request-1""#); + assert_eq!(first.meta_info["prompt_tokens"], "3"); + assert_eq!(first.meta_info["completion_tokens"], "1"); + assert_eq!(first.meta_info["finish_reason"], "null"); + + let finished = stream.next().await.unwrap().unwrap(); + assert_eq!(finished.text, "Hello"); + assert!(finished.finished); + assert_eq!(finished.meta_info["completion_tokens"], "2"); + assert!(finished.meta_info["finish_reason"].contains(r#""type":"stop""#)); + assert!(stream.next().await.is_none()); + assert!(harness.abort_rx.try_recv().is_err()); +} + +#[tokio::test] +async fn generate_streams_incremental_token_ids_with_cumulative_count() { + let harness = Harness::new(2, true, Duration::from_secs(1)); + let response = harness + .service + .generate(Request::new(proto::GenerateRequest { + input_ids: vec![4, 5], + stream: Some(true), + rid: Some("tokens".into()), + ..Default::default() + })) + .await + .unwrap(); + let mut stream = response.into_inner(); + + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..])); + assert!(intake.request.text.is_none()); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false))) + .unwrap(); + intake + .sink + .try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true))) + .unwrap(); + + let first = stream.next().await.unwrap().unwrap(); + let finished = stream.next().await.unwrap().unwrap(); + assert_eq!(first.output_ids, vec![10]); + assert_eq!(finished.output_ids, vec![11]); + assert_eq!(finished.meta_info["completion_tokens"], "2"); + assert!(finished.finished); +} + +#[tokio::test] +async fn stream_drop_aborts_admitted_request() { + let harness = Harness::new(1, false, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(proto::GenerateRequest { + input_ids: vec![1], + rid: Some("cancel-me".into()), + ..Default::default() + })) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false))) + .unwrap(); + assert!(stream.next().await.unwrap().is_ok()); + drop(stream); + let abort = harness.abort_rx.recv_async().await.unwrap(); + assert_eq!(abort.rid().client_facing(), "cancel-me"); + assert!(harness.abort_rx.try_recv().is_err()); +} + +#[tokio::test] +async fn runtime_failure_is_an_in_stream_status_and_disarms_abort() { + let harness = Harness::new(1, false, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(proto::GenerateRequest { + input_ids: vec![1], + ..Default::default() + })) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + intake + .sink + .try_send(ResponseItem::Error(crate::utils::error::Error::Validation( + "bad sampling".into(), + ))) + .unwrap(); + + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument); + drop(stream); + assert!(harness.abort_rx.try_recv().is_err()); +} + +#[tokio::test] +async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() { + let harness = Harness::new(1, false, Duration::from_millis(10)); + let mut stream = harness + .service + .generate(Request::new(proto::GenerateRequest { + input_ids: vec![1], + rid: Some("too-slow".into()), + ..Default::default() + })) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false))) + .unwrap(); + assert!(stream.next().await.unwrap().is_ok()); + + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.code(), Code::DeadlineExceeded); + drop(stream); + let abort = harness.abort_rx.recv_async().await.unwrap(); + assert_eq!(abort.rid().client_facing(), "too-slow"); +} + +#[tokio::test] +async fn closed_intake_is_a_top_level_unavailable_status() { + let (intake_tx, intake_rx) = flume::unbounded(); + drop(intake_rx); + let (abort_tx, _abort_rx) = flume::unbounded(); + let frontend = FrontendHandle::new( + intake_tx, + abort_tx, + FrontendConfig { + response_capacity: 1, + response_activity: Default::default(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: FrontendMetadata::default(), + }, + ); + let service = GrpcService::for_test(frontend, None, false, Duration::from_secs(1)); + + let result = service + .generate(Request::new(proto::GenerateRequest { + input_ids: vec![1], + ..Default::default() + })) + .await; + let error = match result { + Ok(_) => panic!("closed intake must reject the RPC"), + Err(error) => error, + }; + assert_eq!(error.code(), Code::Unavailable); +} + +#[tokio::test] +async fn new_proto_fields_and_engine_watch_are_explicitly_unsupported() { + let harness = Harness::new(1, false, Duration::from_secs(1)); + let text_error = harness + .service + .text_generate(Request::new(proto::TextGenerateRequest { + text: "hello".into(), + kv_hints: Some(Default::default()), + ..Default::default() + })) + .await + .err() + .expect("KV hints must not be silently ignored"); + assert_eq!(text_error.code(), Code::Unimplemented); + let token_error = harness + .service + .generate(Request::new(proto::GenerateRequest { + input_ids: vec![1], + kv_hints: Some(Default::default()), + ..Default::default() + })) + .await + .err() + .expect("KV hints must not be silently ignored"); + assert_eq!(token_error.code(), Code::Unimplemented); + let watch_error = harness + .service + .watch_engine_state(Request::new(proto::WatchEngineStateRequest {})) + .await + .err() + .expect("engine watch is outside the generation adapter"); + assert_eq!(watch_error.code(), Code::Unimplemented); + assert!(harness.intake_rx.try_recv().is_err()); +} + +#[tokio::test] +async fn output_token_overflow_returns_error_instead_of_truncating() { + let harness = Harness::new(1, true, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(proto::GenerateRequest { + input_ids: vec![i32::MAX], + ..Default::default() + })) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + assert_eq!( + intake.request.input_ids.as_deref(), + Some(&[i64::from(i32::MAX)][..]) + ); + intake + .sink + .try_send(ResponseItem::Frame(chunk( + &intake.rid, + "", + i64::from(i32::MAX) + 1, + false, + ))) + .unwrap(); + assert_eq!( + stream.next().await.unwrap().unwrap_err().code(), + Code::Internal + ); + assert!(stream.next().await.is_none()); + assert!( + matches!(harness.abort_rx.recv_async().await.unwrap(), AbortSource::Guard(rid) if rid == intake.rid) + ); +} diff --git a/rust/sglang-server/src/lib.rs b/rust/sglang-server/src/lib.rs index 4f2592916a..e6d9a18d18 100644 --- a/rust/sglang-server/src/lib.rs +++ b/rust/sglang-server/src/lib.rs @@ -13,8 +13,10 @@ mod api_server; mod frontend; +mod grpc; mod message; mod multi_modality; +mod native_generation; mod tokenizer_manager; mod utils; diff --git a/rust/sglang-server/src/message/sampling.rs b/rust/sglang-server/src/message/sampling.rs index 2491004d3e..7c8c9a2b6b 100644 --- a/rust/sglang-server/src/message/sampling.rs +++ b/rust/sglang-server/src/message/sampling.rs @@ -315,10 +315,10 @@ impl SamplingParamsInput { /// wins even when it explicitly carries the type's default or null. pub fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> { match self { - Self::One(params) => apply_preferred_to_one(params, preferred), + Self::One(params) => params.apply_preferred(preferred), Self::Many(params) => params .iter_mut() - .try_for_each(|params| apply_preferred_to_one(params, preferred)), + .try_for_each(|params| params.apply_preferred(preferred)), } } @@ -327,27 +327,6 @@ impl SamplingParamsInput { } } -fn apply_preferred_to_one( - params: &mut SamplingParams, - preferred: &serde_json::Value, -) -> Result<(), String> { - let mut merged = preferred - .as_object() - .ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())? - .clone(); - let request_value = serde_json::to_value(&*params).map_err(|e| e.to_string())?; - let request = request_value - .as_object() - .ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?; - for field in ¶ms.explicit_fields { - if let Some(value) = request.get(field) { - merged.insert(field.clone(), value.clone()); - } - } - *params = sampling_params_from_value(serde_json::Value::Object(merged))?; - Ok(()) -} - impl Default for SamplingParams { fn default() -> Self { // Each field reads the same `default()` the serde attribute above names, @@ -393,6 +372,33 @@ impl Default for SamplingParams { } impl SamplingParams { + /// Record a field supplied by a non-Serde adapter. Preferred launch values + /// are merged underneath these fields just as they are underneath keys + /// present in an HTTP sampling object. + pub(crate) fn mark_explicit(&mut self, field: &'static str) { + self.explicit_fields.insert(field.to_owned()); + } + + /// Merge operator-provided sampling defaults beneath explicitly supplied + /// request fields. HTTP and gRPC adapters use the same precedence policy. + pub(crate) fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> { + let mut merged = preferred + .as_object() + .ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())? + .clone(); + let request_value = serde_json::to_value(&*self).map_err(|e| e.to_string())?; + let request = request_value + .as_object() + .ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?; + for field in &self.explicit_fields { + if let Some(value) = request.get(field) { + merged.insert(field.clone(), value.clone()); + } + } + *self = sampling_params_from_value(serde_json::Value::Object(merged))?; + Ok(()) + } + /// `__post_init__` → `normalize` → `verify`, the order /// `TokenizerManager._create_tokenized_object` runs them in. `Err` is a /// request-local 400. `skip_tokenizer_init` stands in for Python's diff --git a/rust/sglang-server/src/native_generation.rs b/rust/sglang-server/src/native_generation.rs new file mode 100644 index 0000000000..2366bfb7f2 --- /dev/null +++ b/rust/sglang-server/src/native_generation.rs @@ -0,0 +1,160 @@ +//! Shared wire semantics for SGLang-native generation responses. +//! +//! Native HTTP places these values in a JSON `meta_info` object, while +//! `runtime.v1` gRPC JSON-encodes each value into `map`. Keeping +//! the logical shape here prevents the sibling adapters from drifting without +//! making either adapter depend on the other. + +use crate::frontend::FrontendOutput; + +/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when +/// `return_text_in_logprobs` supplied a text buffer, else `null`. +pub(crate) fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value { + texts + .and_then(|texts| texts.get(j)) + .map(|text| serde_json::json!(text)) + .unwrap_or(serde_json::Value::Null) +} + +/// A decoded-text column becomes the tuples' text source only when populated. +pub(crate) fn opt_texts(texts: &[String]) -> Option<&[String]> { + (!texts.is_empty()).then_some(texts) +} + +/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel. +pub(crate) fn lp_value(value: f32) -> serde_json::Value { + if value.is_nan() { + serde_json::Value::Null + } else { + serde_json::json!(value) + } +} + +/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples. +pub(crate) fn logprob_tuples( + values: &[f32], + token_ids: &[i32], + texts: Option<&[String]>, +) -> serde_json::Value { + let tuples = values + .iter() + .zip(token_ids.iter()) + .enumerate() + .map(|(index, (&value, &token_id))| { + serde_json::json!([lp_value(value), token_id, text_slot(texts, index)]) + }) + .collect(); + serde_json::Value::Array(tuples) +} + +/// Ragged top-k / requested-token shape: one entry per position, or `null` +/// where that position's length is zero. +pub(crate) fn ragged_logprob_tuples( + values: &[f32], + token_ids: &[i32], + lengths: &[u32], + texts: Option<&[String]>, +) -> serde_json::Value { + let mut positions = Vec::with_capacity(lengths.len()); + let mut offset = 0usize; + for &length in lengths { + let length = length as usize; + if length == 0 { + positions.push(serde_json::Value::Null); + } else { + // Bounds checks turn malformed parallel columns into a shortened + // row instead of panicking a transport worker. + let tuples = (offset..offset + length) + .filter_map(|index| { + Some(serde_json::json!([ + lp_value(*values.get(index)?), + *token_ids.get(index)?, + text_slot(texts, index) + ])) + }) + .collect(); + positions.push(serde_json::Value::Array(tuples)); + } + offset += length; + } + serde_json::Value::Array(positions) +} + +/// Reshape flat hidden-state values and per-row lengths into nested rows. +pub(crate) fn hidden_states_rows(values: &[f32], lengths: &[u32]) -> serde_json::Value { + let mut rows = Vec::with_capacity(lengths.len()); + let mut offset = 0usize; + for &length in lengths { + let length = length as usize; + rows.push(serde_json::json!( + values.get(offset..offset + length).unwrap_or(&[]) + )); + offset += length; + } + serde_json::Value::Array(rows) +} + +/// Build the heterogeneous native `meta_info` object before either adapter +/// chooses its wire encoding. +pub(crate) fn meta_info_value(out: &FrontendOutput, rid: &str) -> serde_json::Value { + let mut meta = serde_json::json!({ + "id": rid, + "prompt_tokens": out.prompt_tokens, + "completion_tokens": out.completion_tokens, + "finish_reason": out.finish_reason, + }); + let Some(extras) = out.extras.as_deref() else { + return meta; + }; + + // Python always sets input and output token logprobs together, including + // the empty input list emitted by a disaggregated decode node. + if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() { + meta["output_token_logprobs"] = logprob_tuples( + &extras.out_lp_val, + &extras.out_lp_idx, + opt_texts(&extras.out_lp_txt), + ); + meta["input_token_logprobs"] = logprob_tuples( + &extras.in_lp_val, + &extras.in_lp_idx, + opt_texts(&extras.in_lp_txt), + ); + } + if !extras.out_top_lens.is_empty() { + meta["output_top_logprobs"] = ragged_logprob_tuples( + &extras.out_top_val, + &extras.out_top_idx, + &extras.out_top_lens, + opt_texts(&extras.out_top_txt), + ); + } + if !extras.in_top_lens.is_empty() { + meta["input_top_logprobs"] = ragged_logprob_tuples( + &extras.in_top_val, + &extras.in_top_idx, + &extras.in_top_lens, + opt_texts(&extras.in_top_txt), + ); + } + if !extras.out_tid_lens.is_empty() { + meta["output_token_ids_logprobs"] = ragged_logprob_tuples( + &extras.out_tid_val, + &extras.out_tid_idx, + &extras.out_tid_lens, + opt_texts(&extras.out_tid_txt), + ); + } + if !extras.in_tid_lens.is_empty() { + meta["input_token_ids_logprobs"] = ragged_logprob_tuples( + &extras.in_tid_val, + &extras.in_tid_idx, + &extras.in_tid_lens, + opt_texts(&extras.in_tid_txt), + ); + } + if !extras.hidden_lens.is_empty() { + meta["hidden_states"] = hidden_states_rows(&extras.hidden_val, &extras.hidden_lens); + } + meta +} diff --git a/test/registered/rust/test_rust_extension.py b/test/registered/rust/test_rust_extension.py index 5320813791..10b5712d62 100644 --- a/test/registered/rust/test_rust_extension.py +++ b/test/registered/rust/test_rust_extension.py @@ -565,7 +565,7 @@ crate-type = ["cdylib"] "sglang-server", "sglang_server", (), - (), + (grpc_proto,), ), ( "sglang.srt.rust_extensions._grpc",