diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto index 75cf98de90..88b91265bc 100644 --- a/proto/sglang/api/v1/generate.proto +++ b/proto/sglang/api/v1/generate.proto @@ -1,9 +1,7 @@ -// The native `/generate` surface — the ground truth for -// rust/sglang-server/src/message/request.rs's GenerateBody (request) and -// api_core/frame.rs's frame shape (response). Unknown JSON keys on the -// request are IGNORED: `deny_unknown_fields` broke real clients that send -// Python-only `GenerateReqInput` fields (session_id, -// ...), pinned by request.rs's unported-fields test. +// Shared `/generate` wire contract for HTTP, gRPC, and OpenAI lowering. +// Native Rust serving supports a subset of Python's fields; its normalization +// explicitly ignores the Python-only extensions. Unknown JSON keys remain +// ignored for compatibility with clients sending additional Python fields. syntax = "proto3"; @@ -78,6 +76,24 @@ message GenerateRequest { optional string routing_key = 31; optional uint64 max_thinking_tokens = 32; KvHintsEnvelopeOrList kv_hints = 29; + + // Python serving extensions forwarded by the shared OpenAI processor. + // Their presence in the wire contract does not imply native Rust support. + optional string session_id = 33; + OptionalStringOrList custom_logit_processor = 34; + map custom_labels = 35; + ImagesConfig images_config = 36; + bool return_prompt_token_ids = 37; + int64 routed_experts_start_len = 38; + BoolOrList return_sampling_mask = 39; + OptionalStringList modalities = 40; +} + +// Per-request Unlimited-OCR options. The processor owns mode validation and +// defaults: an absent image_mode uses "gundam", an empty prefix adds no tokens. +message ImagesConfig { + optional string image_mode = 1; + string prefix = 2; } message KvHintsEnvelopeOrList { @@ -184,6 +200,17 @@ message GenerateMetaInfo { // reasoning so far; reported on every frame. uint32 cached_tokens = 13; uint64 reasoning_tokens = 14; + // Python reports the cumulative count even for incremental logprob frames. + optional uint64 output_token_logprobs_length = 15; + optional string weight_version = 16; + // Omitted until terminal output; reported spans are always nonempty. + repeated WeightVersionSpan weight_versions = 17; +} + +message WeightVersionSpan { + string version = 1; + int64 start = 2; + int64 end = 3; } message GenerateResponse { @@ -196,6 +223,7 @@ message GenerateResponse { TokenIds output_ids = 3; // Batch position; only on batch responses/frames. optional uint32 index = 4; + TokenIds prompt_token_ids = 5; } // One request's failure inside a Generate stream; carries its batch position diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index 5e8791fab1..b35bb21041 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -267,6 +267,14 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): kv_hints: Optional[KvHintsEnvelopeOrList] = None routing_key: Optional[str] = None max_thinking_tokens: Optional[int] = None + session_id: Optional[str] = None + custom_logit_processor: Optional[OptionalStringOrList] = None + custom_labels: Dict[str, str] = {} + images_config: Optional[ImagesConfig] = None + return_prompt_token_ids: bool = False + routed_experts_start_len: int = 0 + return_sampling_mask: Optional[BoolOrList] = None + modalities: Optional[OptionalStringList] = None @classmethod def from_json_value(cls, value: Any) -> "GenerateRequest": @@ -278,6 +286,22 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): return encode_GenerateRequest(self) +class ImagesConfig(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.ImagesConfig: JSON object (unknown keys ignored).""" + + image_mode: Optional[str] = None + prefix: str = "" + + @classmethod + def from_json_value(cls, value: Any) -> "ImagesConfig": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_ImagesConfig(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_ImagesConfig(self) + + class KvHintsEnvelope(msgspec.Struct, kw_only=True, frozen=True): """sglang.api.v1.KvHintsEnvelope: JSON object (unknown keys ignored).""" @@ -347,6 +371,9 @@ class GenerateMetaInfo(msgspec.Struct, kw_only=True, frozen=True): e2e_latency: Optional[float] = None cached_tokens: int = 0 reasoning_tokens: int = 0 + output_token_logprobs_length: Optional[int] = None + weight_version: Optional[str] = None + weight_versions: List[WeightVersionSpan] = [] @classmethod def from_json_value(cls, value: Any) -> "GenerateMetaInfo": @@ -358,6 +385,23 @@ class GenerateMetaInfo(msgspec.Struct, kw_only=True, frozen=True): return encode_GenerateMetaInfo(self) +class WeightVersionSpan(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.WeightVersionSpan: JSON object (unknown keys ignored).""" + + version: str = "" + start: int = 0 + end: int = 0 + + @classmethod + def from_json_value(cls, value: Any) -> "WeightVersionSpan": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_WeightVersionSpan(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_WeightVersionSpan(self) + + class GenerateResponse(msgspec.Struct, kw_only=True, frozen=True): """sglang.api.v1.GenerateResponse: JSON object (unknown keys ignored).""" @@ -365,6 +409,7 @@ class GenerateResponse(msgspec.Struct, kw_only=True, frozen=True): meta_info: Optional[GenerateMetaInfo] = None output_ids: Optional[TokenIds] = None index: Optional[int] = None + prompt_token_ids: Optional[TokenIds] = None @classmethod def from_json_value(cls, value: Any) -> "GenerateResponse": @@ -1428,6 +1473,22 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest: kw["routing_key"] = None if x is None else _expect_str(x, expected="a string") elif key == "max_thinking_tokens": kw["max_thinking_tokens"] = None if x is None else _expect_uint(x, expected="u64") + elif key == "session_id": + kw["session_id"] = None if x is None else _expect_str(x, expected="a string") + elif key == "custom_logit_processor": + kw["custom_logit_processor"] = None if x is None else decode_OptionalStringOrList(x) + elif key == "custom_labels": + kw["custom_labels"] = {} if x is None else {k: _expect_str(e, expected="a string") for k, e in _expect_map(x, expected="a map").items()} + elif key == "images_config": + kw["images_config"] = None if x is None else decode_ImagesConfig(x) + elif key == "return_prompt_token_ids": + kw["return_prompt_token_ids"] = _expect_bool(x, expected="a boolean") + elif key == "routed_experts_start_len": + kw["routed_experts_start_len"] = _expect_int(x, expected="i64") + elif key == "return_sampling_mask": + kw["return_sampling_mask"] = None if x is None else decode_BoolOrList(x) + elif key == "modalities": + kw["modalities"] = None if x is None else decode_OptionalStringList(x) return GenerateRequest(**kw) @@ -1495,6 +1556,39 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]: d["routing_key"] = v.routing_key if v.max_thinking_tokens is not None: d["max_thinking_tokens"] = v.max_thinking_tokens + if v.session_id is not None: + d["session_id"] = v.session_id + if v.custom_logit_processor is not None: + d["custom_logit_processor"] = encode_OptionalStringOrList(v.custom_logit_processor) + if v.custom_labels: + d["custom_labels"] = dict(v.custom_labels) + if v.images_config is not None: + d["images_config"] = encode_ImagesConfig(v.images_config) + d["return_prompt_token_ids"] = v.return_prompt_token_ids + d["routed_experts_start_len"] = v.routed_experts_start_len + if v.return_sampling_mask is not None: + d["return_sampling_mask"] = encode_BoolOrList(v.return_sampling_mask) + if v.modalities is not None: + d["modalities"] = encode_OptionalStringList(v.modalities) + return d + + +def decode_ImagesConfig(value: Any) -> ImagesConfig: + m = _expect_map(value, expected="struct ImagesConfig") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "image_mode": + kw["image_mode"] = None if x is None else _expect_str(x, expected="a string") + elif key == "prefix": + kw["prefix"] = _expect_str(x, expected="a string") + return ImagesConfig(**kw) + + +def encode_ImagesConfig(v: ImagesConfig) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.image_mode is not None: + d["image_mode"] = v.image_mode + d["prefix"] = v.prefix return d @@ -1676,6 +1770,12 @@ def decode_GenerateMetaInfo(value: Any) -> GenerateMetaInfo: kw["cached_tokens"] = _expect_uint(x, expected="u32") elif key == "reasoning_tokens": kw["reasoning_tokens"] = _expect_uint(x, expected="u64") + elif key == "output_token_logprobs_length": + kw["output_token_logprobs_length"] = None if x is None else _expect_uint(x, expected="u64") + elif key == "weight_version": + kw["weight_version"] = None if x is None else _expect_str(x, expected="a string") + elif key == "weight_versions": + kw["weight_versions"] = [] if x is None else [decode_WeightVersionSpan(e) for e in _expect_seq(x, expected="a sequence")] return GenerateMetaInfo(**kw) @@ -1703,6 +1803,33 @@ def encode_GenerateMetaInfo(v: GenerateMetaInfo) -> Dict[str, Any]: d["e2e_latency"] = float(v.e2e_latency) d["cached_tokens"] = v.cached_tokens d["reasoning_tokens"] = v.reasoning_tokens + if v.output_token_logprobs_length is not None: + d["output_token_logprobs_length"] = v.output_token_logprobs_length + if v.weight_version is not None: + d["weight_version"] = v.weight_version + if v.weight_versions: + d["weight_versions"] = [encode_WeightVersionSpan(e) for e in v.weight_versions] + return d + + +def decode_WeightVersionSpan(value: Any) -> WeightVersionSpan: + m = _expect_map(value, expected="struct WeightVersionSpan") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "version": + kw["version"] = _expect_str(x, expected="a string") + elif key == "start": + kw["start"] = _expect_int(x, expected="i64") + elif key == "end": + kw["end"] = _expect_int(x, expected="i64") + return WeightVersionSpan(**kw) + + +def encode_WeightVersionSpan(v: WeightVersionSpan) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["version"] = v.version + d["start"] = v.start + d["end"] = v.end return d @@ -1718,6 +1845,8 @@ def decode_GenerateResponse(value: Any) -> GenerateResponse: kw["output_ids"] = None if x is None else decode_TokenIds(x) elif key == "index": kw["index"] = None if x is None else _expect_uint(x, expected="u32") + elif key == "prompt_token_ids": + kw["prompt_token_ids"] = None if x is None else decode_TokenIds(x) return GenerateResponse(**kw) @@ -1730,6 +1859,8 @@ def encode_GenerateResponse(v: GenerateResponse) -> Dict[str, Any]: d["output_ids"] = encode_TokenIds(v.output_ids) if v.index is not None: d["index"] = v.index + if v.prompt_token_ids is not None: + d["prompt_token_ids"] = encode_TokenIds(v.prompt_token_ids) return d diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index e301fed478..397182d6f1 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -411,6 +411,34 @@ pub struct GenerateRequest { pub max_thinking_tokens: ::core::option::Option, #[prost(message, optional, tag = "29")] pub kv_hints: ::core::option::Option, + /// Python serving extensions forwarded by the shared OpenAI processor. + /// Their presence in the wire contract does not imply native Rust support. + #[prost(string, optional, tag = "33")] + pub session_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, optional, tag = "34")] + pub custom_logit_processor: ::core::option::Option, + #[prost(map = "string, string", tag = "35")] + pub custom_labels: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + #[prost(message, optional, tag = "36")] + pub images_config: ::core::option::Option, + #[prost(bool, tag = "37")] + pub return_prompt_token_ids: bool, + #[prost(int64, tag = "38")] + pub routed_experts_start_len: i64, + #[prost(message, optional, tag = "39")] + pub return_sampling_mask: ::core::option::Option, + #[prost(message, optional, tag = "40")] + pub modalities: ::core::option::Option, +} +/// Per-request Unlimited-OCR options. The processor owns mode validation and +/// defaults: an absent image_mode uses "gundam", an empty prefix adds no tokens. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ImagesConfig { + #[prost(string, optional, tag = "1")] + pub image_mode: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, tag = "2")] + pub prefix: ::prost::alloc::string::String, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct KvHintsEnvelopeOrList { @@ -554,6 +582,23 @@ pub struct GenerateMetaInfo { pub cached_tokens: u32, #[prost(uint64, tag = "14")] pub reasoning_tokens: u64, + /// Python reports the cumulative count even for incremental logprob frames. + #[prost(uint64, optional, tag = "15")] + pub output_token_logprobs_length: ::core::option::Option, + #[prost(string, optional, tag = "16")] + pub weight_version: ::core::option::Option<::prost::alloc::string::String>, + /// Omitted until terminal output; reported spans are always nonempty. + #[prost(message, repeated, tag = "17")] + pub weight_versions: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct WeightVersionSpan { + #[prost(string, tag = "1")] + pub version: ::prost::alloc::string::String, + #[prost(int64, tag = "2")] + pub start: i64, + #[prost(int64, tag = "3")] + pub end: i64, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct GenerateResponse { @@ -570,6 +615,8 @@ pub struct GenerateResponse { /// Batch position; only on batch responses/frames. #[prost(uint32, optional, tag = "4")] pub index: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub prompt_token_ids: ::core::option::Option, } /// One request's failure inside a Generate stream; carries its batch position /// (the HTTP SSE shape puts `index` beside the error object the same way). diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index 762c9da8d5..534639763d 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -1637,6 +1637,38 @@ impl ::serde::Serialize for GenerateRequest { if let Some(v) = &self.max_thinking_tokens { ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_thinking_tokens", v)?; } + if let Some(v) = &self.session_id { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "session_id", v)?; + } + if let Some(v) = &self.custom_logit_processor { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "custom_logit_processor", v)?; + } + if !self.custom_labels.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "custom_labels", + &self.custom_labels, + )?; + } + if let Some(v) = &self.images_config { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "images_config", v)?; + } + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "return_prompt_token_ids", + &self.return_prompt_token_ids, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "routed_experts_start_len", + &self.routed_experts_start_len, + )?; + if let Some(v) = &self.return_sampling_mask { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "return_sampling_mask", v)?; + } + if let Some(v) = &self.modalities { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "modalities", v)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -1684,6 +1716,14 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { let mut f_kv_hints = None; let mut f_routing_key = None; let mut f_max_thinking_tokens = None; + let mut f_session_id = None; + let mut f_custom_logit_processor = None; + let mut f_custom_labels = None; + let mut f_images_config = None; + let mut f_return_prompt_token_ids = None; + let mut f_routed_experts_start_len = None; + let mut f_return_sampling_mask = None; + let mut f_modalities = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "rid" => { @@ -1902,6 +1942,72 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { } f_max_thinking_tokens = Some(map.next_value::>()?); } + "session_id" => { + if f_session_id.is_some() { + return Err(::serde::de::Error::duplicate_field("session_id")); + } + f_session_id = + Some(map.next_value::>()?); + } + "custom_logit_processor" => { + if f_custom_logit_processor.is_some() { + return Err(::serde::de::Error::duplicate_field( + "custom_logit_processor", + )); + } + f_custom_logit_processor = + Some(map.next_value::>()?); + } + "custom_labels" => { + if f_custom_labels.is_some() { + return Err(::serde::de::Error::duplicate_field("custom_labels")); + } + f_custom_labels = Some( + map.next_value::, + >>()? + .unwrap_or_default(), + ); + } + "images_config" => { + if f_images_config.is_some() { + return Err(::serde::de::Error::duplicate_field("images_config")); + } + f_images_config = Some(map.next_value::>()?); + } + "return_prompt_token_ids" => { + if f_return_prompt_token_ids.is_some() { + return Err(::serde::de::Error::duplicate_field( + "return_prompt_token_ids", + )); + } + f_return_prompt_token_ids = Some(map.next_value::()?); + } + "routed_experts_start_len" => { + if f_routed_experts_start_len.is_some() { + return Err(::serde::de::Error::duplicate_field( + "routed_experts_start_len", + )); + } + f_routed_experts_start_len = Some(map.next_value::()?); + } + "return_sampling_mask" => { + if f_return_sampling_mask.is_some() { + return Err(::serde::de::Error::duplicate_field( + "return_sampling_mask", + )); + } + f_return_sampling_mask = Some(map.next_value::>()?); + } + "modalities" => { + if f_modalities.is_some() { + return Err(::serde::de::Error::duplicate_field("modalities")); + } + f_modalities = Some(map.next_value::>()?); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -1939,6 +2045,69 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { kv_hints: f_kv_hints.flatten(), routing_key: f_routing_key.flatten(), max_thinking_tokens: f_max_thinking_tokens.flatten(), + session_id: f_session_id.flatten(), + custom_logit_processor: f_custom_logit_processor.flatten(), + custom_labels: f_custom_labels.unwrap_or_default(), + images_config: f_images_config.flatten(), + return_prompt_token_ids: f_return_prompt_token_ids.unwrap_or_default(), + routed_experts_start_len: f_routed_experts_start_len.unwrap_or_default(), + return_sampling_mask: f_return_sampling_mask.flatten(), + modalities: f_modalities.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for ImagesConfig { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if let Some(v) = &self.image_mode { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "image_mode", v)?; + } + ::serde::ser::SerializeMap::serialize_entry(&mut map, "prefix", &self.prefix)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for ImagesConfig { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = ImagesConfig; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct ImagesConfig") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_image_mode = None; + let mut f_prefix = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "image_mode" => { + if f_image_mode.is_some() { + return Err(::serde::de::Error::duplicate_field("image_mode")); + } + f_image_mode = + Some(map.next_value::>()?); + } + "prefix" => { + if f_prefix.is_some() { + return Err(::serde::de::Error::duplicate_field("prefix")); + } + f_prefix = Some(map.next_value::<::prost::alloc::string::String>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(ImagesConfig { + image_mode: f_image_mode.flatten(), + prefix: f_prefix.unwrap_or_default(), }) } } @@ -2432,6 +2601,23 @@ impl ::serde::Serialize for GenerateMetaInfo { "reasoning_tokens", &self.reasoning_tokens, )?; + if let Some(v) = &self.output_token_logprobs_length { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "output_token_logprobs_length", + v, + )?; + } + if let Some(v) = &self.weight_version { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "weight_version", v)?; + } + if !self.weight_versions.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "weight_versions", + &self.weight_versions, + )?; + } ::serde::ser::SerializeMap::end(map) } } @@ -2462,6 +2648,9 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo { let mut f_e2e_latency = None; let mut f_cached_tokens = None; let mut f_reasoning_tokens = None; + let mut f_output_token_logprobs_length = None; + let mut f_weight_version = None; + let mut f_weight_versions = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "id" => { @@ -2581,6 +2770,30 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo { } f_reasoning_tokens = Some(map.next_value::()?); } + "output_token_logprobs_length" => { + if f_output_token_logprobs_length.is_some() { + return Err(::serde::de::Error::duplicate_field( + "output_token_logprobs_length", + )); + } + f_output_token_logprobs_length = Some(map.next_value::>()?); + } + "weight_version" => { + if f_weight_version.is_some() { + return Err(::serde::de::Error::duplicate_field("weight_version")); + } + f_weight_version = + Some(map.next_value::>()?); + } + "weight_versions" => { + if f_weight_versions.is_some() { + return Err(::serde::de::Error::duplicate_field("weight_versions")); + } + f_weight_versions = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -2601,6 +2814,70 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo { e2e_latency: f_e2e_latency.flatten(), cached_tokens: f_cached_tokens.unwrap_or_default(), reasoning_tokens: f_reasoning_tokens.unwrap_or_default(), + output_token_logprobs_length: f_output_token_logprobs_length.flatten(), + weight_version: f_weight_version.flatten(), + weight_versions: f_weight_versions.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for WeightVersionSpan { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "version", &self.version)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "start", &self.start)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "end", &self.end)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for WeightVersionSpan { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = WeightVersionSpan; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct WeightVersionSpan") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_version = None; + let mut f_start = None; + let mut f_end = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "version" => { + if f_version.is_some() { + return Err(::serde::de::Error::duplicate_field("version")); + } + f_version = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "start" => { + if f_start.is_some() { + return Err(::serde::de::Error::duplicate_field("start")); + } + f_start = Some(map.next_value::()?); + } + "end" => { + if f_end.is_some() { + return Err(::serde::de::Error::duplicate_field("end")); + } + f_end = Some(map.next_value::()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(WeightVersionSpan { + version: f_version.unwrap_or_default(), + start: f_start.unwrap_or_default(), + end: f_end.unwrap_or_default(), }) } } @@ -2621,6 +2898,9 @@ impl ::serde::Serialize for GenerateResponse { if let Some(v) = &self.index { ::serde::ser::SerializeMap::serialize_entry(&mut map, "index", v)?; } + if let Some(v) = &self.prompt_token_ids { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "prompt_token_ids", v)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -2641,6 +2921,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateResponse { let mut f_meta_info = None; let mut f_output_ids = None; let mut f_index = None; + let mut f_prompt_token_ids = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "text" => { @@ -2667,6 +2948,14 @@ impl<'de> ::serde::Deserialize<'de> for GenerateResponse { } f_index = Some(map.next_value::>()?); } + "prompt_token_ids" => { + if f_prompt_token_ids.is_some() { + return Err(::serde::de::Error::duplicate_field( + "prompt_token_ids", + )); + } + f_prompt_token_ids = Some(map.next_value::>()?); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -2677,6 +2966,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateResponse { meta_info: f_meta_info.flatten(), output_ids: f_output_ids.flatten(), index: f_index.flatten(), + prompt_token_ids: f_prompt_token_ids.flatten(), }) } } diff --git a/rust/sglang-api-types/testdata/json_parity.json b/rust/sglang-api-types/testdata/json_parity.json index 078fdaa738..8086095a13 100644 --- a/rust/sglang-api-types/testdata/json_parity.json +++ b/rust/sglang-api-types/testdata/json_parity.json @@ -1,6 +1,61 @@ { "_": "Cross-language JSON parity fixtures for sglang.api.v1. Each case: decoding `input` and re-encoding yields `canonical` byte for byte, and `canonical` re-encodes to itself. Checked by rust/sglang-api-types/tests/json_parity.rs and test/registered/unit/api/v1/test_parity.py.", "cases": [ + { + "type": "GenerateRequest", + "name": "empty labels are omitted while image options default to an empty prefix", + "input": { + "text": "hi", + "custom_labels": {}, + "images_config": {} + }, + "canonical": "{\"text\":\"hi\",\"stream\":false,\"images_config\":{\"prefix\":\"\"},\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}" + }, + { + "type": "GenerateRequest", + "name": "custom labels and typed image options round-trip", + "input": { + "text": "hi", + "custom_labels": { + "tenant": "team" + }, + "images_config": { + "image_mode": "base", + "prefix": "OCR: " + } + }, + "canonical": "{\"text\":\"hi\",\"stream\":false,\"custom_labels\":{\"tenant\":\"team\"},\"images_config\":{\"image_mode\":\"base\",\"prefix\":\"OCR: \"},\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}" + }, + { + "type": "GenerateResponse", + "name": "empty spans are omitted on nonterminal output", + "input": { + "text": "", + "meta_info": { + "id": "r", + "weight_versions": [] + } + }, + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":0,\"completion_tokens\":0,\"finish_reason\":null,\"cached_tokens\":0,\"reasoning_tokens\":0}}" + }, + { + "type": "GenerateResponse", + "name": "zero-length weight spans survive serialization", + "input": { + "text": "", + "meta_info": { + "id": "r", + "weight_versions": [ + { + "version": "v1", + "start": 0, + "end": 0 + } + ] + } + }, + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":0,\"completion_tokens\":0,\"finish_reason\":null,\"cached_tokens\":0,\"reasoning_tokens\":0,\"weight_versions\":[{\"version\":\"v1\",\"start\":0,\"end\":0}]}}" + }, { "type": "ListModelsResponse", "name": "model cards preserve a uint64 context limit and omit absent parent", @@ -47,7 +102,7 @@ "max_new_tokens": 8 } }, - "canonical": "{\"text\":\"hi\",\"stream\":false,\"sampling_params\":{\"max_new_tokens\":8,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}}" + "canonical": "{\"text\":\"hi\",\"stream\":false,\"sampling_params\":{\"max_new_tokens\":8,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}" }, { "type": "GenerateRequest", @@ -76,7 +131,7 @@ null ] }, - "canonical": "{\"text\":[\"a\",\"b\"],\"stream\":true,\"sampling_params\":[{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":0.9,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},{\"max_new_tokens\":128,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}],\"return_logprob\":true,\"bootstrap_host\":[null,\"h\"],\"bootstrap_port\":[8998,null]}" + "canonical": "{\"text\":[\"a\",\"b\"],\"stream\":true,\"sampling_params\":[{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":0.9,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},{\"max_new_tokens\":128,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}],\"return_logprob\":true,\"bootstrap_host\":[null,\"h\"],\"bootstrap_port\":[8998,null],\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}" }, { "type": "GenerateRequest", @@ -100,7 +155,7 @@ "return_hidden_states": true, "routed_dp_rank": 1 }, - "canonical": "{\"rid\":\"r-1\",\"input_ids\":[1,2,3],\"stream\":false,\"return_logprob\":true,\"logprob_start_len\":0,\"top_logprobs_num\":2,\"token_ids_logprob\":[5,6],\"return_hidden_states\":true,\"return_text_in_logprobs\":true,\"routed_dp_rank\":1}" + "canonical": "{\"rid\":\"r-1\",\"input_ids\":[1,2,3],\"stream\":false,\"return_logprob\":true,\"logprob_start_len\":0,\"top_logprobs_num\":2,\"token_ids_logprob\":[5,6],\"return_hidden_states\":true,\"return_text_in_logprobs\":true,\"routed_dp_rank\":1,\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}" }, { "type": "GenerateRequest", @@ -125,9 +180,9 @@ "h1", "h2" ], - "session_id": "ignored" + "conversation_id": "ignored" }, - "canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\"}" + "canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\",\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}" }, { "type": "SamplingParams", diff --git a/rust/sglang-api-types/tests/json_contract.rs b/rust/sglang-api-types/tests/json_contract.rs index 8d0c283e8c..e6c45e043f 100644 --- a/rust/sglang-api-types/tests/json_contract.rs +++ b/rust/sglang-api-types/tests/json_contract.rs @@ -84,10 +84,9 @@ fn sampling_denies_unknown_keys() { /// pinned in message/request.rs). #[test] fn generate_request_ignores_unknown_keys() { - let req: GenerateRequest = serde_json::from_str( - r#"{"text": "hi", "priority": 3, "session_id": "s", "custom_logit_processor": "x"}"#, - ) - .unwrap(); + let req: GenerateRequest = + serde_json::from_str(r#"{"text": "hi", "priority": 3, "conversation_id": "ignored"}"#) + .unwrap(); assert!(matches!( req.text, Some(StringOrList { @@ -251,6 +250,7 @@ fn generate_response_matches_frame_value_shape() { ..Default::default() }), index: None, + prompt_token_ids: None, }; assert_eq!( serde_json::to_string(&frame).unwrap(), diff --git a/rust/sglang-bench/src/request.rs b/rust/sglang-bench/src/request.rs index 05adea9cc7..ff1f6bf285 100644 --- a/rust/sglang-bench/src/request.rs +++ b/rust/sglang-bench/src/request.rs @@ -869,6 +869,7 @@ mod tests { }), output_ids: Some(api::TokenIds { ids: vec![7, 8] }), index: None, + prompt_token_ids: None, }; let data = serde_json::to_value(&frame).unwrap(); assert_eq!(data["text"], "ab"); diff --git a/rust/sglang-bench/tests/grpc.rs b/rust/sglang-bench/tests/grpc.rs index 6eaf920333..1396f6bbb1 100644 --- a/rust/sglang-bench/tests/grpc.rs +++ b/rust/sglang-bench/tests/grpc.rs @@ -63,6 +63,7 @@ impl SglangService for Mock { }), output_ids: Some(api::TokenIds { ids: ids.clone() }), index: None, + prompt_token_ids: None, }, )), })); diff --git a/rust/sglang-server/src/api_server/core/frame.rs b/rust/sglang-server/src/api_server/core/frame.rs index 6c8995fd8a..2f7bc284b2 100644 --- a/rust/sglang-server/src/api_server/core/frame.rs +++ b/rust/sglang-server/src/api_server/core/frame.rs @@ -207,6 +207,7 @@ impl CoreOutput { ids: self.token_ids, }), index, + prompt_token_ids: None, } } diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index f154329bfb..9c36dce2f1 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -107,6 +107,7 @@ pub fn into_requests( extra_key, cache_salt, session_params, + .. } = req; if session_params.is_some() { return Err(Error::Validation( diff --git a/test/registered/unit/api/v1/test_json_contract.py b/test/registered/unit/api/v1/test_json_contract.py index bde00d2757..76098fe170 100644 --- a/test/registered/unit/api/v1/test_json_contract.py +++ b/test/registered/unit/api/v1/test_json_contract.py @@ -131,8 +131,7 @@ class TestGenerateRequest(CustomTestCase): { "text": "hi", "priority": 3, - "session_id": "s", - "custom_logit_processor": "x", + "conversation_id": "ignored", } ) self.assertEqual(req.text, "hi") diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 6aa4acbe8c..bac5d13e66 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -51,14 +51,11 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( { "background", "conversation_id", - "custom_labels", - "custom_logit_processor", "data_parallel_rank", "encoder_urls", "external_trace_header", "http_worker_ipc", "image_max_dynamic_patch", - "images_config", "input_embeds", "log_metrics", "lora_id", @@ -66,7 +63,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "min_dynamic_patch", "mm_content_hashes", "mm_processor_kwargs", - "modalities", "multi_item_delimiter_indices", "need_wait_for_mm_inputs", "no_logs", @@ -79,12 +75,8 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "return_flat_raw_top_logprobs", "return_flat_raw_top_logprobs_b64", "return_indexer_topk", - "return_prompt_token_ids", "return_routed_experts", - "return_sampling_mask", - "routed_experts_start_len", "sampling_logprobs_mode", - "session_id", "token_indices_to_pool", "use_audio_in_video", "video_config", @@ -231,7 +223,7 @@ class TestRequestFieldParity(CustomTestCase): class TestResponseParity(CustomTestCase): def test_python_server_meta_info_decodes_to_the_schema_subset(self): """The Python server's `meta_info` carries keys the schema does not - model (weight_version, num_retractions, spec stats, ...). The + model (num_retractions, spec stats, ...). The generated types ignore them and re-emit the schema subset, so a Rust-server client and a Python-server client see one shape.""" python_frame = { @@ -260,6 +252,7 @@ class TestResponseParity(CustomTestCase): "e2e_latency", "cached_tokens", "reasoning_tokens", + "weight_version", ], ) self.assertEqual( @@ -273,15 +266,29 @@ class TestResponseParity(CustomTestCase): serializers must keep matching it.""" self.assertEqual( list(GenerateResponse.__struct_fields__), - ["text", "meta_info", "output_ids", "index"], + ["text", "meta_info", "output_ids", "index", "prompt_token_ids"], ) self.assertEqual( - list(GenerateMetaInfo.__struct_fields__)[:4], - ["id", "prompt_tokens", "completion_tokens", "finish_reason"], - ) - self.assertEqual( - list(GenerateMetaInfo.__struct_fields__)[-3:], - ["e2e_latency", "cached_tokens", "reasoning_tokens"], + list(GenerateMetaInfo.__struct_fields__), + [ + "id", + "prompt_tokens", + "completion_tokens", + "finish_reason", + "output_token_logprobs", + "input_token_logprobs", + "output_top_logprobs", + "input_top_logprobs", + "output_token_ids_logprobs", + "input_token_ids_logprobs", + "hidden_states", + "e2e_latency", + "cached_tokens", + "reasoning_tokens", + "output_token_logprobs_length", + "weight_version", + "weight_versions", + ], )