[Chore] api: add fields to generate request (#45164)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
@@ -1,9 +1,7 @@
|
|||||||
// The native `/generate` surface — the ground truth for
|
// Shared `/generate` wire contract for HTTP, gRPC, and OpenAI lowering.
|
||||||
// rust/sglang-server/src/message/request.rs's GenerateBody (request) and
|
// Native Rust serving supports a subset of Python's fields; its normalization
|
||||||
// api_core/frame.rs's frame shape (response). Unknown JSON keys on the
|
// explicitly ignores the Python-only extensions. Unknown JSON keys remain
|
||||||
// request are IGNORED: `deny_unknown_fields` broke real clients that send
|
// ignored for compatibility with clients sending additional Python fields.
|
||||||
// Python-only `GenerateReqInput` fields (session_id,
|
|
||||||
// ...), pinned by request.rs's unported-fields test.
|
|
||||||
|
|
||||||
syntax = "proto3";
|
syntax = "proto3";
|
||||||
|
|
||||||
@@ -78,6 +76,24 @@ message GenerateRequest {
|
|||||||
optional string routing_key = 31;
|
optional string routing_key = 31;
|
||||||
optional uint64 max_thinking_tokens = 32;
|
optional uint64 max_thinking_tokens = 32;
|
||||||
KvHintsEnvelopeOrList kv_hints = 29;
|
KvHintsEnvelopeOrList kv_hints = 29;
|
||||||
|
|
||||||
|
// Python serving extensions forwarded by the shared OpenAI processor.
|
||||||
|
// Their presence in the wire contract does not imply native Rust support.
|
||||||
|
optional string session_id = 33;
|
||||||
|
OptionalStringOrList custom_logit_processor = 34;
|
||||||
|
map<string, string> custom_labels = 35;
|
||||||
|
ImagesConfig images_config = 36;
|
||||||
|
bool return_prompt_token_ids = 37;
|
||||||
|
int64 routed_experts_start_len = 38;
|
||||||
|
BoolOrList return_sampling_mask = 39;
|
||||||
|
OptionalStringList modalities = 40;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Per-request Unlimited-OCR options. The processor owns mode validation and
|
||||||
|
// defaults: an absent image_mode uses "gundam", an empty prefix adds no tokens.
|
||||||
|
message ImagesConfig {
|
||||||
|
optional string image_mode = 1;
|
||||||
|
string prefix = 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
message KvHintsEnvelopeOrList {
|
message KvHintsEnvelopeOrList {
|
||||||
@@ -184,6 +200,17 @@ message GenerateMetaInfo {
|
|||||||
// reasoning so far; reported on every frame.
|
// reasoning so far; reported on every frame.
|
||||||
uint32 cached_tokens = 13;
|
uint32 cached_tokens = 13;
|
||||||
uint64 reasoning_tokens = 14;
|
uint64 reasoning_tokens = 14;
|
||||||
|
// Python reports the cumulative count even for incremental logprob frames.
|
||||||
|
optional uint64 output_token_logprobs_length = 15;
|
||||||
|
optional string weight_version = 16;
|
||||||
|
// Omitted until terminal output; reported spans are always nonempty.
|
||||||
|
repeated WeightVersionSpan weight_versions = 17;
|
||||||
|
}
|
||||||
|
|
||||||
|
message WeightVersionSpan {
|
||||||
|
string version = 1;
|
||||||
|
int64 start = 2;
|
||||||
|
int64 end = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
message GenerateResponse {
|
message GenerateResponse {
|
||||||
@@ -196,6 +223,7 @@ message GenerateResponse {
|
|||||||
TokenIds output_ids = 3;
|
TokenIds output_ids = 3;
|
||||||
// Batch position; only on batch responses/frames.
|
// Batch position; only on batch responses/frames.
|
||||||
optional uint32 index = 4;
|
optional uint32 index = 4;
|
||||||
|
TokenIds prompt_token_ids = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
// One request's failure inside a Generate stream; carries its batch position
|
// One request's failure inside a Generate stream; carries its batch position
|
||||||
|
|||||||
@@ -267,6 +267,14 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
|
|||||||
kv_hints: Optional[KvHintsEnvelopeOrList] = None
|
kv_hints: Optional[KvHintsEnvelopeOrList] = None
|
||||||
routing_key: Optional[str] = None
|
routing_key: Optional[str] = None
|
||||||
max_thinking_tokens: Optional[int] = None
|
max_thinking_tokens: Optional[int] = None
|
||||||
|
session_id: Optional[str] = None
|
||||||
|
custom_logit_processor: Optional[OptionalStringOrList] = None
|
||||||
|
custom_labels: Dict[str, str] = {}
|
||||||
|
images_config: Optional[ImagesConfig] = None
|
||||||
|
return_prompt_token_ids: bool = False
|
||||||
|
routed_experts_start_len: int = 0
|
||||||
|
return_sampling_mask: Optional[BoolOrList] = None
|
||||||
|
modalities: Optional[OptionalStringList] = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_json_value(cls, value: Any) -> "GenerateRequest":
|
def from_json_value(cls, value: Any) -> "GenerateRequest":
|
||||||
@@ -278,6 +286,22 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
|
|||||||
return encode_GenerateRequest(self)
|
return encode_GenerateRequest(self)
|
||||||
|
|
||||||
|
|
||||||
|
class ImagesConfig(msgspec.Struct, kw_only=True, frozen=True):
|
||||||
|
"""sglang.api.v1.ImagesConfig: JSON object (unknown keys ignored)."""
|
||||||
|
|
||||||
|
image_mode: Optional[str] = None
|
||||||
|
prefix: str = ""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_json_value(cls, value: Any) -> "ImagesConfig":
|
||||||
|
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
|
||||||
|
return decode_ImagesConfig(value)
|
||||||
|
|
||||||
|
def to_json_value(self) -> Dict[str, Any]:
|
||||||
|
"""Encode to a JSON-ready value (json.dumps-compatible)."""
|
||||||
|
return encode_ImagesConfig(self)
|
||||||
|
|
||||||
|
|
||||||
class KvHintsEnvelope(msgspec.Struct, kw_only=True, frozen=True):
|
class KvHintsEnvelope(msgspec.Struct, kw_only=True, frozen=True):
|
||||||
"""sglang.api.v1.KvHintsEnvelope: JSON object (unknown keys ignored)."""
|
"""sglang.api.v1.KvHintsEnvelope: JSON object (unknown keys ignored)."""
|
||||||
|
|
||||||
@@ -347,6 +371,9 @@ class GenerateMetaInfo(msgspec.Struct, kw_only=True, frozen=True):
|
|||||||
e2e_latency: Optional[float] = None
|
e2e_latency: Optional[float] = None
|
||||||
cached_tokens: int = 0
|
cached_tokens: int = 0
|
||||||
reasoning_tokens: int = 0
|
reasoning_tokens: int = 0
|
||||||
|
output_token_logprobs_length: Optional[int] = None
|
||||||
|
weight_version: Optional[str] = None
|
||||||
|
weight_versions: List[WeightVersionSpan] = []
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_json_value(cls, value: Any) -> "GenerateMetaInfo":
|
def from_json_value(cls, value: Any) -> "GenerateMetaInfo":
|
||||||
@@ -358,6 +385,23 @@ class GenerateMetaInfo(msgspec.Struct, kw_only=True, frozen=True):
|
|||||||
return encode_GenerateMetaInfo(self)
|
return encode_GenerateMetaInfo(self)
|
||||||
|
|
||||||
|
|
||||||
|
class WeightVersionSpan(msgspec.Struct, kw_only=True, frozen=True):
|
||||||
|
"""sglang.api.v1.WeightVersionSpan: JSON object (unknown keys ignored)."""
|
||||||
|
|
||||||
|
version: str = ""
|
||||||
|
start: int = 0
|
||||||
|
end: int = 0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_json_value(cls, value: Any) -> "WeightVersionSpan":
|
||||||
|
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
|
||||||
|
return decode_WeightVersionSpan(value)
|
||||||
|
|
||||||
|
def to_json_value(self) -> Dict[str, Any]:
|
||||||
|
"""Encode to a JSON-ready value (json.dumps-compatible)."""
|
||||||
|
return encode_WeightVersionSpan(self)
|
||||||
|
|
||||||
|
|
||||||
class GenerateResponse(msgspec.Struct, kw_only=True, frozen=True):
|
class GenerateResponse(msgspec.Struct, kw_only=True, frozen=True):
|
||||||
"""sglang.api.v1.GenerateResponse: JSON object (unknown keys ignored)."""
|
"""sglang.api.v1.GenerateResponse: JSON object (unknown keys ignored)."""
|
||||||
|
|
||||||
@@ -365,6 +409,7 @@ class GenerateResponse(msgspec.Struct, kw_only=True, frozen=True):
|
|||||||
meta_info: Optional[GenerateMetaInfo] = None
|
meta_info: Optional[GenerateMetaInfo] = None
|
||||||
output_ids: Optional[TokenIds] = None
|
output_ids: Optional[TokenIds] = None
|
||||||
index: Optional[int] = None
|
index: Optional[int] = None
|
||||||
|
prompt_token_ids: Optional[TokenIds] = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_json_value(cls, value: Any) -> "GenerateResponse":
|
def from_json_value(cls, value: Any) -> "GenerateResponse":
|
||||||
@@ -1428,6 +1473,22 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest:
|
|||||||
kw["routing_key"] = None if x is None else _expect_str(x, expected="a string")
|
kw["routing_key"] = None if x is None else _expect_str(x, expected="a string")
|
||||||
elif key == "max_thinking_tokens":
|
elif key == "max_thinking_tokens":
|
||||||
kw["max_thinking_tokens"] = None if x is None else _expect_uint(x, expected="u64")
|
kw["max_thinking_tokens"] = None if x is None else _expect_uint(x, expected="u64")
|
||||||
|
elif key == "session_id":
|
||||||
|
kw["session_id"] = None if x is None else _expect_str(x, expected="a string")
|
||||||
|
elif key == "custom_logit_processor":
|
||||||
|
kw["custom_logit_processor"] = None if x is None else decode_OptionalStringOrList(x)
|
||||||
|
elif key == "custom_labels":
|
||||||
|
kw["custom_labels"] = {} if x is None else {k: _expect_str(e, expected="a string") for k, e in _expect_map(x, expected="a map").items()}
|
||||||
|
elif key == "images_config":
|
||||||
|
kw["images_config"] = None if x is None else decode_ImagesConfig(x)
|
||||||
|
elif key == "return_prompt_token_ids":
|
||||||
|
kw["return_prompt_token_ids"] = _expect_bool(x, expected="a boolean")
|
||||||
|
elif key == "routed_experts_start_len":
|
||||||
|
kw["routed_experts_start_len"] = _expect_int(x, expected="i64")
|
||||||
|
elif key == "return_sampling_mask":
|
||||||
|
kw["return_sampling_mask"] = None if x is None else decode_BoolOrList(x)
|
||||||
|
elif key == "modalities":
|
||||||
|
kw["modalities"] = None if x is None else decode_OptionalStringList(x)
|
||||||
return GenerateRequest(**kw)
|
return GenerateRequest(**kw)
|
||||||
|
|
||||||
|
|
||||||
@@ -1495,6 +1556,39 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]:
|
|||||||
d["routing_key"] = v.routing_key
|
d["routing_key"] = v.routing_key
|
||||||
if v.max_thinking_tokens is not None:
|
if v.max_thinking_tokens is not None:
|
||||||
d["max_thinking_tokens"] = v.max_thinking_tokens
|
d["max_thinking_tokens"] = v.max_thinking_tokens
|
||||||
|
if v.session_id is not None:
|
||||||
|
d["session_id"] = v.session_id
|
||||||
|
if v.custom_logit_processor is not None:
|
||||||
|
d["custom_logit_processor"] = encode_OptionalStringOrList(v.custom_logit_processor)
|
||||||
|
if v.custom_labels:
|
||||||
|
d["custom_labels"] = dict(v.custom_labels)
|
||||||
|
if v.images_config is not None:
|
||||||
|
d["images_config"] = encode_ImagesConfig(v.images_config)
|
||||||
|
d["return_prompt_token_ids"] = v.return_prompt_token_ids
|
||||||
|
d["routed_experts_start_len"] = v.routed_experts_start_len
|
||||||
|
if v.return_sampling_mask is not None:
|
||||||
|
d["return_sampling_mask"] = encode_BoolOrList(v.return_sampling_mask)
|
||||||
|
if v.modalities is not None:
|
||||||
|
d["modalities"] = encode_OptionalStringList(v.modalities)
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def decode_ImagesConfig(value: Any) -> ImagesConfig:
|
||||||
|
m = _expect_map(value, expected="struct ImagesConfig")
|
||||||
|
kw: Dict[str, Any] = {}
|
||||||
|
for key, x in m.items():
|
||||||
|
if key == "image_mode":
|
||||||
|
kw["image_mode"] = None if x is None else _expect_str(x, expected="a string")
|
||||||
|
elif key == "prefix":
|
||||||
|
kw["prefix"] = _expect_str(x, expected="a string")
|
||||||
|
return ImagesConfig(**kw)
|
||||||
|
|
||||||
|
|
||||||
|
def encode_ImagesConfig(v: ImagesConfig) -> Dict[str, Any]:
|
||||||
|
d: Dict[str, Any] = {}
|
||||||
|
if v.image_mode is not None:
|
||||||
|
d["image_mode"] = v.image_mode
|
||||||
|
d["prefix"] = v.prefix
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
@@ -1676,6 +1770,12 @@ def decode_GenerateMetaInfo(value: Any) -> GenerateMetaInfo:
|
|||||||
kw["cached_tokens"] = _expect_uint(x, expected="u32")
|
kw["cached_tokens"] = _expect_uint(x, expected="u32")
|
||||||
elif key == "reasoning_tokens":
|
elif key == "reasoning_tokens":
|
||||||
kw["reasoning_tokens"] = _expect_uint(x, expected="u64")
|
kw["reasoning_tokens"] = _expect_uint(x, expected="u64")
|
||||||
|
elif key == "output_token_logprobs_length":
|
||||||
|
kw["output_token_logprobs_length"] = None if x is None else _expect_uint(x, expected="u64")
|
||||||
|
elif key == "weight_version":
|
||||||
|
kw["weight_version"] = None if x is None else _expect_str(x, expected="a string")
|
||||||
|
elif key == "weight_versions":
|
||||||
|
kw["weight_versions"] = [] if x is None else [decode_WeightVersionSpan(e) for e in _expect_seq(x, expected="a sequence")]
|
||||||
return GenerateMetaInfo(**kw)
|
return GenerateMetaInfo(**kw)
|
||||||
|
|
||||||
|
|
||||||
@@ -1703,6 +1803,33 @@ def encode_GenerateMetaInfo(v: GenerateMetaInfo) -> Dict[str, Any]:
|
|||||||
d["e2e_latency"] = float(v.e2e_latency)
|
d["e2e_latency"] = float(v.e2e_latency)
|
||||||
d["cached_tokens"] = v.cached_tokens
|
d["cached_tokens"] = v.cached_tokens
|
||||||
d["reasoning_tokens"] = v.reasoning_tokens
|
d["reasoning_tokens"] = v.reasoning_tokens
|
||||||
|
if v.output_token_logprobs_length is not None:
|
||||||
|
d["output_token_logprobs_length"] = v.output_token_logprobs_length
|
||||||
|
if v.weight_version is not None:
|
||||||
|
d["weight_version"] = v.weight_version
|
||||||
|
if v.weight_versions:
|
||||||
|
d["weight_versions"] = [encode_WeightVersionSpan(e) for e in v.weight_versions]
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def decode_WeightVersionSpan(value: Any) -> WeightVersionSpan:
|
||||||
|
m = _expect_map(value, expected="struct WeightVersionSpan")
|
||||||
|
kw: Dict[str, Any] = {}
|
||||||
|
for key, x in m.items():
|
||||||
|
if key == "version":
|
||||||
|
kw["version"] = _expect_str(x, expected="a string")
|
||||||
|
elif key == "start":
|
||||||
|
kw["start"] = _expect_int(x, expected="i64")
|
||||||
|
elif key == "end":
|
||||||
|
kw["end"] = _expect_int(x, expected="i64")
|
||||||
|
return WeightVersionSpan(**kw)
|
||||||
|
|
||||||
|
|
||||||
|
def encode_WeightVersionSpan(v: WeightVersionSpan) -> Dict[str, Any]:
|
||||||
|
d: Dict[str, Any] = {}
|
||||||
|
d["version"] = v.version
|
||||||
|
d["start"] = v.start
|
||||||
|
d["end"] = v.end
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
@@ -1718,6 +1845,8 @@ def decode_GenerateResponse(value: Any) -> GenerateResponse:
|
|||||||
kw["output_ids"] = None if x is None else decode_TokenIds(x)
|
kw["output_ids"] = None if x is None else decode_TokenIds(x)
|
||||||
elif key == "index":
|
elif key == "index":
|
||||||
kw["index"] = None if x is None else _expect_uint(x, expected="u32")
|
kw["index"] = None if x is None else _expect_uint(x, expected="u32")
|
||||||
|
elif key == "prompt_token_ids":
|
||||||
|
kw["prompt_token_ids"] = None if x is None else decode_TokenIds(x)
|
||||||
return GenerateResponse(**kw)
|
return GenerateResponse(**kw)
|
||||||
|
|
||||||
|
|
||||||
@@ -1730,6 +1859,8 @@ def encode_GenerateResponse(v: GenerateResponse) -> Dict[str, Any]:
|
|||||||
d["output_ids"] = encode_TokenIds(v.output_ids)
|
d["output_ids"] = encode_TokenIds(v.output_ids)
|
||||||
if v.index is not None:
|
if v.index is not None:
|
||||||
d["index"] = v.index
|
d["index"] = v.index
|
||||||
|
if v.prompt_token_ids is not None:
|
||||||
|
d["prompt_token_ids"] = encode_TokenIds(v.prompt_token_ids)
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -411,6 +411,34 @@ pub struct GenerateRequest {
|
|||||||
pub max_thinking_tokens: ::core::option::Option<u64>,
|
pub max_thinking_tokens: ::core::option::Option<u64>,
|
||||||
#[prost(message, optional, tag = "29")]
|
#[prost(message, optional, tag = "29")]
|
||||||
pub kv_hints: ::core::option::Option<KvHintsEnvelopeOrList>,
|
pub kv_hints: ::core::option::Option<KvHintsEnvelopeOrList>,
|
||||||
|
/// Python serving extensions forwarded by the shared OpenAI processor.
|
||||||
|
/// Their presence in the wire contract does not imply native Rust support.
|
||||||
|
#[prost(string, optional, tag = "33")]
|
||||||
|
pub session_id: ::core::option::Option<::prost::alloc::string::String>,
|
||||||
|
#[prost(message, optional, tag = "34")]
|
||||||
|
pub custom_logit_processor: ::core::option::Option<OptionalStringOrList>,
|
||||||
|
#[prost(map = "string, string", tag = "35")]
|
||||||
|
pub custom_labels:
|
||||||
|
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||||
|
#[prost(message, optional, tag = "36")]
|
||||||
|
pub images_config: ::core::option::Option<ImagesConfig>,
|
||||||
|
#[prost(bool, tag = "37")]
|
||||||
|
pub return_prompt_token_ids: bool,
|
||||||
|
#[prost(int64, tag = "38")]
|
||||||
|
pub routed_experts_start_len: i64,
|
||||||
|
#[prost(message, optional, tag = "39")]
|
||||||
|
pub return_sampling_mask: ::core::option::Option<BoolOrList>,
|
||||||
|
#[prost(message, optional, tag = "40")]
|
||||||
|
pub modalities: ::core::option::Option<OptionalStringList>,
|
||||||
|
}
|
||||||
|
/// Per-request Unlimited-OCR options. The processor owns mode validation and
|
||||||
|
/// defaults: an absent image_mode uses "gundam", an empty prefix adds no tokens.
|
||||||
|
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||||
|
pub struct ImagesConfig {
|
||||||
|
#[prost(string, optional, tag = "1")]
|
||||||
|
pub image_mode: ::core::option::Option<::prost::alloc::string::String>,
|
||||||
|
#[prost(string, tag = "2")]
|
||||||
|
pub prefix: ::prost::alloc::string::String,
|
||||||
}
|
}
|
||||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||||
pub struct KvHintsEnvelopeOrList {
|
pub struct KvHintsEnvelopeOrList {
|
||||||
@@ -554,6 +582,23 @@ pub struct GenerateMetaInfo {
|
|||||||
pub cached_tokens: u32,
|
pub cached_tokens: u32,
|
||||||
#[prost(uint64, tag = "14")]
|
#[prost(uint64, tag = "14")]
|
||||||
pub reasoning_tokens: u64,
|
pub reasoning_tokens: u64,
|
||||||
|
/// Python reports the cumulative count even for incremental logprob frames.
|
||||||
|
#[prost(uint64, optional, tag = "15")]
|
||||||
|
pub output_token_logprobs_length: ::core::option::Option<u64>,
|
||||||
|
#[prost(string, optional, tag = "16")]
|
||||||
|
pub weight_version: ::core::option::Option<::prost::alloc::string::String>,
|
||||||
|
/// Omitted until terminal output; reported spans are always nonempty.
|
||||||
|
#[prost(message, repeated, tag = "17")]
|
||||||
|
pub weight_versions: ::prost::alloc::vec::Vec<WeightVersionSpan>,
|
||||||
|
}
|
||||||
|
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||||
|
pub struct WeightVersionSpan {
|
||||||
|
#[prost(string, tag = "1")]
|
||||||
|
pub version: ::prost::alloc::string::String,
|
||||||
|
#[prost(int64, tag = "2")]
|
||||||
|
pub start: i64,
|
||||||
|
#[prost(int64, tag = "3")]
|
||||||
|
pub end: i64,
|
||||||
}
|
}
|
||||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||||
pub struct GenerateResponse {
|
pub struct GenerateResponse {
|
||||||
@@ -570,6 +615,8 @@ pub struct GenerateResponse {
|
|||||||
/// Batch position; only on batch responses/frames.
|
/// Batch position; only on batch responses/frames.
|
||||||
#[prost(uint32, optional, tag = "4")]
|
#[prost(uint32, optional, tag = "4")]
|
||||||
pub index: ::core::option::Option<u32>,
|
pub index: ::core::option::Option<u32>,
|
||||||
|
#[prost(message, optional, tag = "5")]
|
||||||
|
pub prompt_token_ids: ::core::option::Option<TokenIds>,
|
||||||
}
|
}
|
||||||
/// One request's failure inside a Generate stream; carries its batch position
|
/// One request's failure inside a Generate stream; carries its batch position
|
||||||
/// (the HTTP SSE shape puts `index` beside the error object the same way).
|
/// (the HTTP SSE shape puts `index` beside the error object the same way).
|
||||||
|
|||||||
@@ -1637,6 +1637,38 @@ impl ::serde::Serialize for GenerateRequest {
|
|||||||
if let Some(v) = &self.max_thinking_tokens {
|
if let Some(v) = &self.max_thinking_tokens {
|
||||||
::serde::ser::SerializeMap::serialize_entry(&mut map, "max_thinking_tokens", v)?;
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "max_thinking_tokens", v)?;
|
||||||
}
|
}
|
||||||
|
if let Some(v) = &self.session_id {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "session_id", v)?;
|
||||||
|
}
|
||||||
|
if let Some(v) = &self.custom_logit_processor {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "custom_logit_processor", v)?;
|
||||||
|
}
|
||||||
|
if !self.custom_labels.is_empty() {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(
|
||||||
|
&mut map,
|
||||||
|
"custom_labels",
|
||||||
|
&self.custom_labels,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
if let Some(v) = &self.images_config {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "images_config", v)?;
|
||||||
|
}
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(
|
||||||
|
&mut map,
|
||||||
|
"return_prompt_token_ids",
|
||||||
|
&self.return_prompt_token_ids,
|
||||||
|
)?;
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(
|
||||||
|
&mut map,
|
||||||
|
"routed_experts_start_len",
|
||||||
|
&self.routed_experts_start_len,
|
||||||
|
)?;
|
||||||
|
if let Some(v) = &self.return_sampling_mask {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "return_sampling_mask", v)?;
|
||||||
|
}
|
||||||
|
if let Some(v) = &self.modalities {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "modalities", v)?;
|
||||||
|
}
|
||||||
::serde::ser::SerializeMap::end(map)
|
::serde::ser::SerializeMap::end(map)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1684,6 +1716,14 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
|||||||
let mut f_kv_hints = None;
|
let mut f_kv_hints = None;
|
||||||
let mut f_routing_key = None;
|
let mut f_routing_key = None;
|
||||||
let mut f_max_thinking_tokens = None;
|
let mut f_max_thinking_tokens = None;
|
||||||
|
let mut f_session_id = None;
|
||||||
|
let mut f_custom_logit_processor = None;
|
||||||
|
let mut f_custom_labels = None;
|
||||||
|
let mut f_images_config = None;
|
||||||
|
let mut f_return_prompt_token_ids = None;
|
||||||
|
let mut f_routed_experts_start_len = None;
|
||||||
|
let mut f_return_sampling_mask = None;
|
||||||
|
let mut f_modalities = None;
|
||||||
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
||||||
match key.as_ref() {
|
match key.as_ref() {
|
||||||
"rid" => {
|
"rid" => {
|
||||||
@@ -1902,6 +1942,72 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
|||||||
}
|
}
|
||||||
f_max_thinking_tokens = Some(map.next_value::<Option<u64>>()?);
|
f_max_thinking_tokens = Some(map.next_value::<Option<u64>>()?);
|
||||||
}
|
}
|
||||||
|
"session_id" => {
|
||||||
|
if f_session_id.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("session_id"));
|
||||||
|
}
|
||||||
|
f_session_id =
|
||||||
|
Some(map.next_value::<Option<::prost::alloc::string::String>>()?);
|
||||||
|
}
|
||||||
|
"custom_logit_processor" => {
|
||||||
|
if f_custom_logit_processor.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field(
|
||||||
|
"custom_logit_processor",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
f_custom_logit_processor =
|
||||||
|
Some(map.next_value::<Option<OptionalStringOrList>>()?);
|
||||||
|
}
|
||||||
|
"custom_labels" => {
|
||||||
|
if f_custom_labels.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("custom_labels"));
|
||||||
|
}
|
||||||
|
f_custom_labels = Some(
|
||||||
|
map.next_value::<Option<
|
||||||
|
::std::collections::HashMap<
|
||||||
|
String,
|
||||||
|
::prost::alloc::string::String,
|
||||||
|
>,
|
||||||
|
>>()?
|
||||||
|
.unwrap_or_default(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
"images_config" => {
|
||||||
|
if f_images_config.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("images_config"));
|
||||||
|
}
|
||||||
|
f_images_config = Some(map.next_value::<Option<ImagesConfig>>()?);
|
||||||
|
}
|
||||||
|
"return_prompt_token_ids" => {
|
||||||
|
if f_return_prompt_token_ids.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field(
|
||||||
|
"return_prompt_token_ids",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
f_return_prompt_token_ids = Some(map.next_value::<bool>()?);
|
||||||
|
}
|
||||||
|
"routed_experts_start_len" => {
|
||||||
|
if f_routed_experts_start_len.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field(
|
||||||
|
"routed_experts_start_len",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
f_routed_experts_start_len = Some(map.next_value::<i64>()?);
|
||||||
|
}
|
||||||
|
"return_sampling_mask" => {
|
||||||
|
if f_return_sampling_mask.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field(
|
||||||
|
"return_sampling_mask",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
f_return_sampling_mask = Some(map.next_value::<Option<BoolOrList>>()?);
|
||||||
|
}
|
||||||
|
"modalities" => {
|
||||||
|
if f_modalities.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("modalities"));
|
||||||
|
}
|
||||||
|
f_modalities = Some(map.next_value::<Option<OptionalStringList>>()?);
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
map.next_value::<::serde::de::IgnoredAny>()?;
|
map.next_value::<::serde::de::IgnoredAny>()?;
|
||||||
}
|
}
|
||||||
@@ -1939,6 +2045,69 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
|||||||
kv_hints: f_kv_hints.flatten(),
|
kv_hints: f_kv_hints.flatten(),
|
||||||
routing_key: f_routing_key.flatten(),
|
routing_key: f_routing_key.flatten(),
|
||||||
max_thinking_tokens: f_max_thinking_tokens.flatten(),
|
max_thinking_tokens: f_max_thinking_tokens.flatten(),
|
||||||
|
session_id: f_session_id.flatten(),
|
||||||
|
custom_logit_processor: f_custom_logit_processor.flatten(),
|
||||||
|
custom_labels: f_custom_labels.unwrap_or_default(),
|
||||||
|
images_config: f_images_config.flatten(),
|
||||||
|
return_prompt_token_ids: f_return_prompt_token_ids.unwrap_or_default(),
|
||||||
|
routed_experts_start_len: f_routed_experts_start_len.unwrap_or_default(),
|
||||||
|
return_sampling_mask: f_return_sampling_mask.flatten(),
|
||||||
|
modalities: f_modalities.flatten(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deserializer.deserialize_map(V)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ::serde::Serialize for ImagesConfig {
|
||||||
|
fn serialize<S: ::serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||||
|
let mut map = serializer.serialize_map(None)?;
|
||||||
|
if let Some(v) = &self.image_mode {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "image_mode", v)?;
|
||||||
|
}
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "prefix", &self.prefix)?;
|
||||||
|
::serde::ser::SerializeMap::end(map)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de> ::serde::Deserialize<'de> for ImagesConfig {
|
||||||
|
fn deserialize<D: ::serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
struct V;
|
||||||
|
impl<'de> ::serde::de::Visitor<'de> for V {
|
||||||
|
type Value = ImagesConfig;
|
||||||
|
fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
|
||||||
|
f.write_str("struct ImagesConfig")
|
||||||
|
}
|
||||||
|
fn visit_map<A: ::serde::de::MapAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut map: A,
|
||||||
|
) -> Result<Self::Value, A::Error> {
|
||||||
|
let mut f_image_mode = None;
|
||||||
|
let mut f_prefix = None;
|
||||||
|
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
||||||
|
match key.as_ref() {
|
||||||
|
"image_mode" => {
|
||||||
|
if f_image_mode.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("image_mode"));
|
||||||
|
}
|
||||||
|
f_image_mode =
|
||||||
|
Some(map.next_value::<Option<::prost::alloc::string::String>>()?);
|
||||||
|
}
|
||||||
|
"prefix" => {
|
||||||
|
if f_prefix.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("prefix"));
|
||||||
|
}
|
||||||
|
f_prefix = Some(map.next_value::<::prost::alloc::string::String>()?);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
map.next_value::<::serde::de::IgnoredAny>()?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(ImagesConfig {
|
||||||
|
image_mode: f_image_mode.flatten(),
|
||||||
|
prefix: f_prefix.unwrap_or_default(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2432,6 +2601,23 @@ impl ::serde::Serialize for GenerateMetaInfo {
|
|||||||
"reasoning_tokens",
|
"reasoning_tokens",
|
||||||
&self.reasoning_tokens,
|
&self.reasoning_tokens,
|
||||||
)?;
|
)?;
|
||||||
|
if let Some(v) = &self.output_token_logprobs_length {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(
|
||||||
|
&mut map,
|
||||||
|
"output_token_logprobs_length",
|
||||||
|
v,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
if let Some(v) = &self.weight_version {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "weight_version", v)?;
|
||||||
|
}
|
||||||
|
if !self.weight_versions.is_empty() {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(
|
||||||
|
&mut map,
|
||||||
|
"weight_versions",
|
||||||
|
&self.weight_versions,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
::serde::ser::SerializeMap::end(map)
|
::serde::ser::SerializeMap::end(map)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2462,6 +2648,9 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo {
|
|||||||
let mut f_e2e_latency = None;
|
let mut f_e2e_latency = None;
|
||||||
let mut f_cached_tokens = None;
|
let mut f_cached_tokens = None;
|
||||||
let mut f_reasoning_tokens = None;
|
let mut f_reasoning_tokens = None;
|
||||||
|
let mut f_output_token_logprobs_length = None;
|
||||||
|
let mut f_weight_version = None;
|
||||||
|
let mut f_weight_versions = None;
|
||||||
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
||||||
match key.as_ref() {
|
match key.as_ref() {
|
||||||
"id" => {
|
"id" => {
|
||||||
@@ -2581,6 +2770,30 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo {
|
|||||||
}
|
}
|
||||||
f_reasoning_tokens = Some(map.next_value::<u64>()?);
|
f_reasoning_tokens = Some(map.next_value::<u64>()?);
|
||||||
}
|
}
|
||||||
|
"output_token_logprobs_length" => {
|
||||||
|
if f_output_token_logprobs_length.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field(
|
||||||
|
"output_token_logprobs_length",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
f_output_token_logprobs_length = Some(map.next_value::<Option<u64>>()?);
|
||||||
|
}
|
||||||
|
"weight_version" => {
|
||||||
|
if f_weight_version.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("weight_version"));
|
||||||
|
}
|
||||||
|
f_weight_version =
|
||||||
|
Some(map.next_value::<Option<::prost::alloc::string::String>>()?);
|
||||||
|
}
|
||||||
|
"weight_versions" => {
|
||||||
|
if f_weight_versions.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("weight_versions"));
|
||||||
|
}
|
||||||
|
f_weight_versions = Some(
|
||||||
|
map.next_value::<Option<Vec<WeightVersionSpan>>>()?
|
||||||
|
.unwrap_or_default(),
|
||||||
|
);
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
map.next_value::<::serde::de::IgnoredAny>()?;
|
map.next_value::<::serde::de::IgnoredAny>()?;
|
||||||
}
|
}
|
||||||
@@ -2601,6 +2814,70 @@ impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo {
|
|||||||
e2e_latency: f_e2e_latency.flatten(),
|
e2e_latency: f_e2e_latency.flatten(),
|
||||||
cached_tokens: f_cached_tokens.unwrap_or_default(),
|
cached_tokens: f_cached_tokens.unwrap_or_default(),
|
||||||
reasoning_tokens: f_reasoning_tokens.unwrap_or_default(),
|
reasoning_tokens: f_reasoning_tokens.unwrap_or_default(),
|
||||||
|
output_token_logprobs_length: f_output_token_logprobs_length.flatten(),
|
||||||
|
weight_version: f_weight_version.flatten(),
|
||||||
|
weight_versions: f_weight_versions.unwrap_or_default(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deserializer.deserialize_map(V)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ::serde::Serialize for WeightVersionSpan {
|
||||||
|
fn serialize<S: ::serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||||
|
let mut map = serializer.serialize_map(None)?;
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "version", &self.version)?;
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "start", &self.start)?;
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "end", &self.end)?;
|
||||||
|
::serde::ser::SerializeMap::end(map)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de> ::serde::Deserialize<'de> for WeightVersionSpan {
|
||||||
|
fn deserialize<D: ::serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
struct V;
|
||||||
|
impl<'de> ::serde::de::Visitor<'de> for V {
|
||||||
|
type Value = WeightVersionSpan;
|
||||||
|
fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
|
||||||
|
f.write_str("struct WeightVersionSpan")
|
||||||
|
}
|
||||||
|
fn visit_map<A: ::serde::de::MapAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut map: A,
|
||||||
|
) -> Result<Self::Value, A::Error> {
|
||||||
|
let mut f_version = None;
|
||||||
|
let mut f_start = None;
|
||||||
|
let mut f_end = None;
|
||||||
|
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
||||||
|
match key.as_ref() {
|
||||||
|
"version" => {
|
||||||
|
if f_version.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("version"));
|
||||||
|
}
|
||||||
|
f_version = Some(map.next_value::<::prost::alloc::string::String>()?);
|
||||||
|
}
|
||||||
|
"start" => {
|
||||||
|
if f_start.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("start"));
|
||||||
|
}
|
||||||
|
f_start = Some(map.next_value::<i64>()?);
|
||||||
|
}
|
||||||
|
"end" => {
|
||||||
|
if f_end.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field("end"));
|
||||||
|
}
|
||||||
|
f_end = Some(map.next_value::<i64>()?);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
map.next_value::<::serde::de::IgnoredAny>()?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(WeightVersionSpan {
|
||||||
|
version: f_version.unwrap_or_default(),
|
||||||
|
start: f_start.unwrap_or_default(),
|
||||||
|
end: f_end.unwrap_or_default(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2621,6 +2898,9 @@ impl ::serde::Serialize for GenerateResponse {
|
|||||||
if let Some(v) = &self.index {
|
if let Some(v) = &self.index {
|
||||||
::serde::ser::SerializeMap::serialize_entry(&mut map, "index", v)?;
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "index", v)?;
|
||||||
}
|
}
|
||||||
|
if let Some(v) = &self.prompt_token_ids {
|
||||||
|
::serde::ser::SerializeMap::serialize_entry(&mut map, "prompt_token_ids", v)?;
|
||||||
|
}
|
||||||
::serde::ser::SerializeMap::end(map)
|
::serde::ser::SerializeMap::end(map)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2641,6 +2921,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateResponse {
|
|||||||
let mut f_meta_info = None;
|
let mut f_meta_info = None;
|
||||||
let mut f_output_ids = None;
|
let mut f_output_ids = None;
|
||||||
let mut f_index = None;
|
let mut f_index = None;
|
||||||
|
let mut f_prompt_token_ids = None;
|
||||||
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
||||||
match key.as_ref() {
|
match key.as_ref() {
|
||||||
"text" => {
|
"text" => {
|
||||||
@@ -2667,6 +2948,14 @@ impl<'de> ::serde::Deserialize<'de> for GenerateResponse {
|
|||||||
}
|
}
|
||||||
f_index = Some(map.next_value::<Option<u32>>()?);
|
f_index = Some(map.next_value::<Option<u32>>()?);
|
||||||
}
|
}
|
||||||
|
"prompt_token_ids" => {
|
||||||
|
if f_prompt_token_ids.is_some() {
|
||||||
|
return Err(::serde::de::Error::duplicate_field(
|
||||||
|
"prompt_token_ids",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
f_prompt_token_ids = Some(map.next_value::<Option<TokenIds>>()?);
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
map.next_value::<::serde::de::IgnoredAny>()?;
|
map.next_value::<::serde::de::IgnoredAny>()?;
|
||||||
}
|
}
|
||||||
@@ -2677,6 +2966,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateResponse {
|
|||||||
meta_info: f_meta_info.flatten(),
|
meta_info: f_meta_info.flatten(),
|
||||||
output_ids: f_output_ids.flatten(),
|
output_ids: f_output_ids.flatten(),
|
||||||
index: f_index.flatten(),
|
index: f_index.flatten(),
|
||||||
|
prompt_token_ids: f_prompt_token_ids.flatten(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+60
-5
@@ -1,6 +1,61 @@
|
|||||||
{
|
{
|
||||||
"_": "Cross-language JSON parity fixtures for sglang.api.v1. Each case: decoding `input` and re-encoding yields `canonical` byte for byte, and `canonical` re-encodes to itself. Checked by rust/sglang-api-types/tests/json_parity.rs and test/registered/unit/api/v1/test_parity.py.",
|
"_": "Cross-language JSON parity fixtures for sglang.api.v1. Each case: decoding `input` and re-encoding yields `canonical` byte for byte, and `canonical` re-encodes to itself. Checked by rust/sglang-api-types/tests/json_parity.rs and test/registered/unit/api/v1/test_parity.py.",
|
||||||
"cases": [
|
"cases": [
|
||||||
|
{
|
||||||
|
"type": "GenerateRequest",
|
||||||
|
"name": "empty labels are omitted while image options default to an empty prefix",
|
||||||
|
"input": {
|
||||||
|
"text": "hi",
|
||||||
|
"custom_labels": {},
|
||||||
|
"images_config": {}
|
||||||
|
},
|
||||||
|
"canonical": "{\"text\":\"hi\",\"stream\":false,\"images_config\":{\"prefix\":\"\"},\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "GenerateRequest",
|
||||||
|
"name": "custom labels and typed image options round-trip",
|
||||||
|
"input": {
|
||||||
|
"text": "hi",
|
||||||
|
"custom_labels": {
|
||||||
|
"tenant": "team"
|
||||||
|
},
|
||||||
|
"images_config": {
|
||||||
|
"image_mode": "base",
|
||||||
|
"prefix": "OCR: "
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"canonical": "{\"text\":\"hi\",\"stream\":false,\"custom_labels\":{\"tenant\":\"team\"},\"images_config\":{\"image_mode\":\"base\",\"prefix\":\"OCR: \"},\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "GenerateResponse",
|
||||||
|
"name": "empty spans are omitted on nonterminal output",
|
||||||
|
"input": {
|
||||||
|
"text": "",
|
||||||
|
"meta_info": {
|
||||||
|
"id": "r",
|
||||||
|
"weight_versions": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":0,\"completion_tokens\":0,\"finish_reason\":null,\"cached_tokens\":0,\"reasoning_tokens\":0}}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "GenerateResponse",
|
||||||
|
"name": "zero-length weight spans survive serialization",
|
||||||
|
"input": {
|
||||||
|
"text": "",
|
||||||
|
"meta_info": {
|
||||||
|
"id": "r",
|
||||||
|
"weight_versions": [
|
||||||
|
{
|
||||||
|
"version": "v1",
|
||||||
|
"start": 0,
|
||||||
|
"end": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":0,\"completion_tokens\":0,\"finish_reason\":null,\"cached_tokens\":0,\"reasoning_tokens\":0,\"weight_versions\":[{\"version\":\"v1\",\"start\":0,\"end\":0}]}}"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"type": "ListModelsResponse",
|
"type": "ListModelsResponse",
|
||||||
"name": "model cards preserve a uint64 context limit and omit absent parent",
|
"name": "model cards preserve a uint64 context limit and omit absent parent",
|
||||||
@@ -47,7 +102,7 @@
|
|||||||
"max_new_tokens": 8
|
"max_new_tokens": 8
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"canonical": "{\"text\":\"hi\",\"stream\":false,\"sampling_params\":{\"max_new_tokens\":8,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}}"
|
"canonical": "{\"text\":\"hi\",\"stream\":false,\"sampling_params\":{\"max_new_tokens\":8,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"type": "GenerateRequest",
|
"type": "GenerateRequest",
|
||||||
@@ -76,7 +131,7 @@
|
|||||||
null
|
null
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"canonical": "{\"text\":[\"a\",\"b\"],\"stream\":true,\"sampling_params\":[{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":0.9,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},{\"max_new_tokens\":128,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}],\"return_logprob\":true,\"bootstrap_host\":[null,\"h\"],\"bootstrap_port\":[8998,null]}"
|
"canonical": "{\"text\":[\"a\",\"b\"],\"stream\":true,\"sampling_params\":[{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":0.9,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},{\"max_new_tokens\":128,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}],\"return_logprob\":true,\"bootstrap_host\":[null,\"h\"],\"bootstrap_port\":[8998,null],\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"type": "GenerateRequest",
|
"type": "GenerateRequest",
|
||||||
@@ -100,7 +155,7 @@
|
|||||||
"return_hidden_states": true,
|
"return_hidden_states": true,
|
||||||
"routed_dp_rank": 1
|
"routed_dp_rank": 1
|
||||||
},
|
},
|
||||||
"canonical": "{\"rid\":\"r-1\",\"input_ids\":[1,2,3],\"stream\":false,\"return_logprob\":true,\"logprob_start_len\":0,\"top_logprobs_num\":2,\"token_ids_logprob\":[5,6],\"return_hidden_states\":true,\"return_text_in_logprobs\":true,\"routed_dp_rank\":1}"
|
"canonical": "{\"rid\":\"r-1\",\"input_ids\":[1,2,3],\"stream\":false,\"return_logprob\":true,\"logprob_start_len\":0,\"top_logprobs_num\":2,\"token_ids_logprob\":[5,6],\"return_hidden_states\":true,\"return_text_in_logprobs\":true,\"routed_dp_rank\":1,\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"type": "GenerateRequest",
|
"type": "GenerateRequest",
|
||||||
@@ -125,9 +180,9 @@
|
|||||||
"h1",
|
"h1",
|
||||||
"h2"
|
"h2"
|
||||||
],
|
],
|
||||||
"session_id": "ignored"
|
"conversation_id": "ignored"
|
||||||
},
|
},
|
||||||
"canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\"}"
|
"canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\",\"return_prompt_token_ids\":false,\"routed_experts_start_len\":0}"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"type": "SamplingParams",
|
"type": "SamplingParams",
|
||||||
|
|||||||
@@ -84,10 +84,9 @@ fn sampling_denies_unknown_keys() {
|
|||||||
/// pinned in message/request.rs).
|
/// pinned in message/request.rs).
|
||||||
#[test]
|
#[test]
|
||||||
fn generate_request_ignores_unknown_keys() {
|
fn generate_request_ignores_unknown_keys() {
|
||||||
let req: GenerateRequest = serde_json::from_str(
|
let req: GenerateRequest =
|
||||||
r#"{"text": "hi", "priority": 3, "session_id": "s", "custom_logit_processor": "x"}"#,
|
serde_json::from_str(r#"{"text": "hi", "priority": 3, "conversation_id": "ignored"}"#)
|
||||||
)
|
.unwrap();
|
||||||
.unwrap();
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
req.text,
|
req.text,
|
||||||
Some(StringOrList {
|
Some(StringOrList {
|
||||||
@@ -251,6 +250,7 @@ fn generate_response_matches_frame_value_shape() {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
index: None,
|
index: None,
|
||||||
|
prompt_token_ids: None,
|
||||||
};
|
};
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
serde_json::to_string(&frame).unwrap(),
|
serde_json::to_string(&frame).unwrap(),
|
||||||
|
|||||||
@@ -869,6 +869,7 @@ mod tests {
|
|||||||
}),
|
}),
|
||||||
output_ids: Some(api::TokenIds { ids: vec![7, 8] }),
|
output_ids: Some(api::TokenIds { ids: vec![7, 8] }),
|
||||||
index: None,
|
index: None,
|
||||||
|
prompt_token_ids: None,
|
||||||
};
|
};
|
||||||
let data = serde_json::to_value(&frame).unwrap();
|
let data = serde_json::to_value(&frame).unwrap();
|
||||||
assert_eq!(data["text"], "ab");
|
assert_eq!(data["text"], "ab");
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ impl SglangService for Mock {
|
|||||||
}),
|
}),
|
||||||
output_ids: Some(api::TokenIds { ids: ids.clone() }),
|
output_ids: Some(api::TokenIds { ids: ids.clone() }),
|
||||||
index: None,
|
index: None,
|
||||||
|
prompt_token_ids: None,
|
||||||
},
|
},
|
||||||
)),
|
)),
|
||||||
}));
|
}));
|
||||||
|
|||||||
@@ -207,6 +207,7 @@ impl CoreOutput {
|
|||||||
ids: self.token_ids,
|
ids: self.token_ids,
|
||||||
}),
|
}),
|
||||||
index,
|
index,
|
||||||
|
prompt_token_ids: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -107,6 +107,7 @@ pub fn into_requests(
|
|||||||
extra_key,
|
extra_key,
|
||||||
cache_salt,
|
cache_salt,
|
||||||
session_params,
|
session_params,
|
||||||
|
..
|
||||||
} = req;
|
} = req;
|
||||||
if session_params.is_some() {
|
if session_params.is_some() {
|
||||||
return Err(Error::Validation(
|
return Err(Error::Validation(
|
||||||
|
|||||||
@@ -131,8 +131,7 @@ class TestGenerateRequest(CustomTestCase):
|
|||||||
{
|
{
|
||||||
"text": "hi",
|
"text": "hi",
|
||||||
"priority": 3,
|
"priority": 3,
|
||||||
"session_id": "s",
|
"conversation_id": "ignored",
|
||||||
"custom_logit_processor": "x",
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
self.assertEqual(req.text, "hi")
|
self.assertEqual(req.text, "hi")
|
||||||
|
|||||||
@@ -51,14 +51,11 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
|||||||
{
|
{
|
||||||
"background",
|
"background",
|
||||||
"conversation_id",
|
"conversation_id",
|
||||||
"custom_labels",
|
|
||||||
"custom_logit_processor",
|
|
||||||
"data_parallel_rank",
|
"data_parallel_rank",
|
||||||
"encoder_urls",
|
"encoder_urls",
|
||||||
"external_trace_header",
|
"external_trace_header",
|
||||||
"http_worker_ipc",
|
"http_worker_ipc",
|
||||||
"image_max_dynamic_patch",
|
"image_max_dynamic_patch",
|
||||||
"images_config",
|
|
||||||
"input_embeds",
|
"input_embeds",
|
||||||
"log_metrics",
|
"log_metrics",
|
||||||
"lora_id",
|
"lora_id",
|
||||||
@@ -66,7 +63,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
|||||||
"min_dynamic_patch",
|
"min_dynamic_patch",
|
||||||
"mm_content_hashes",
|
"mm_content_hashes",
|
||||||
"mm_processor_kwargs",
|
"mm_processor_kwargs",
|
||||||
"modalities",
|
|
||||||
"multi_item_delimiter_indices",
|
"multi_item_delimiter_indices",
|
||||||
"need_wait_for_mm_inputs",
|
"need_wait_for_mm_inputs",
|
||||||
"no_logs",
|
"no_logs",
|
||||||
@@ -79,12 +75,8 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
|||||||
"return_flat_raw_top_logprobs",
|
"return_flat_raw_top_logprobs",
|
||||||
"return_flat_raw_top_logprobs_b64",
|
"return_flat_raw_top_logprobs_b64",
|
||||||
"return_indexer_topk",
|
"return_indexer_topk",
|
||||||
"return_prompt_token_ids",
|
|
||||||
"return_routed_experts",
|
"return_routed_experts",
|
||||||
"return_sampling_mask",
|
|
||||||
"routed_experts_start_len",
|
|
||||||
"sampling_logprobs_mode",
|
"sampling_logprobs_mode",
|
||||||
"session_id",
|
|
||||||
"token_indices_to_pool",
|
"token_indices_to_pool",
|
||||||
"use_audio_in_video",
|
"use_audio_in_video",
|
||||||
"video_config",
|
"video_config",
|
||||||
@@ -231,7 +223,7 @@ class TestRequestFieldParity(CustomTestCase):
|
|||||||
class TestResponseParity(CustomTestCase):
|
class TestResponseParity(CustomTestCase):
|
||||||
def test_python_server_meta_info_decodes_to_the_schema_subset(self):
|
def test_python_server_meta_info_decodes_to_the_schema_subset(self):
|
||||||
"""The Python server's `meta_info` carries keys the schema does not
|
"""The Python server's `meta_info` carries keys the schema does not
|
||||||
model (weight_version, num_retractions, spec stats, ...). The
|
model (num_retractions, spec stats, ...). The
|
||||||
generated types ignore them and re-emit the schema subset, so a
|
generated types ignore them and re-emit the schema subset, so a
|
||||||
Rust-server client and a Python-server client see one shape."""
|
Rust-server client and a Python-server client see one shape."""
|
||||||
python_frame = {
|
python_frame = {
|
||||||
@@ -260,6 +252,7 @@ class TestResponseParity(CustomTestCase):
|
|||||||
"e2e_latency",
|
"e2e_latency",
|
||||||
"cached_tokens",
|
"cached_tokens",
|
||||||
"reasoning_tokens",
|
"reasoning_tokens",
|
||||||
|
"weight_version",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -273,15 +266,29 @@ class TestResponseParity(CustomTestCase):
|
|||||||
serializers must keep matching it."""
|
serializers must keep matching it."""
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
list(GenerateResponse.__struct_fields__),
|
list(GenerateResponse.__struct_fields__),
|
||||||
["text", "meta_info", "output_ids", "index"],
|
["text", "meta_info", "output_ids", "index", "prompt_token_ids"],
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
list(GenerateMetaInfo.__struct_fields__)[:4],
|
list(GenerateMetaInfo.__struct_fields__),
|
||||||
["id", "prompt_tokens", "completion_tokens", "finish_reason"],
|
[
|
||||||
)
|
"id",
|
||||||
self.assertEqual(
|
"prompt_tokens",
|
||||||
list(GenerateMetaInfo.__struct_fields__)[-3:],
|
"completion_tokens",
|
||||||
["e2e_latency", "cached_tokens", "reasoning_tokens"],
|
"finish_reason",
|
||||||
|
"output_token_logprobs",
|
||||||
|
"input_token_logprobs",
|
||||||
|
"output_top_logprobs",
|
||||||
|
"input_top_logprobs",
|
||||||
|
"output_token_ids_logprobs",
|
||||||
|
"input_token_ids_logprobs",
|
||||||
|
"hidden_states",
|
||||||
|
"e2e_latency",
|
||||||
|
"cached_tokens",
|
||||||
|
"reasoning_tokens",
|
||||||
|
"output_token_logprobs_length",
|
||||||
|
"weight_version",
|
||||||
|
"weight_versions",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user