[Fix] sglang-server: Honour extra_key on the Rust frontend's /generate (#44207)
Co-authored-by: fei-xx <fei-xx@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
fei-xx
shodoco
parent
e7d6a88bfc
commit
209f8220d6
@@ -2,7 +2,7 @@
|
||||
// rust/sglang-server/src/message/request.rs's GenerateBody (request) and
|
||||
// api_core/frame.rs's frame shape (response). Unknown JSON keys on the
|
||||
// request are IGNORED: `deny_unknown_fields` broke real clients that send
|
||||
// Python-only `GenerateReqInput` fields (priority, extra_key, session_id,
|
||||
// Python-only `GenerateReqInput` fields (priority, session_id,
|
||||
// ...), pinned by request.rs's unported-fields test.
|
||||
|
||||
syntax = "proto3";
|
||||
@@ -53,6 +53,10 @@ message GenerateRequest {
|
||||
optional StringListOrList mm_hashes = 20;
|
||||
optional MediaInput video_data = 21;
|
||||
optional MediaInput audio_data = 22;
|
||||
// Prefix-cache namespace: requests share cached prefixes only under the same
|
||||
// key. One key for every batch item, or one per item; an empty key means
|
||||
// none.
|
||||
StringOrList extra_key = 23;
|
||||
}
|
||||
|
||||
message SamplingParamsOrList {
|
||||
|
||||
@@ -257,6 +257,7 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
|
||||
mm_hashes: Optional[StringListOrList] = None
|
||||
video_data: Optional[MediaInput] = None
|
||||
audio_data: Optional[MediaInput] = None
|
||||
extra_key: Optional[StringOrList] = None
|
||||
|
||||
@classmethod
|
||||
def from_json_value(cls, value: Any) -> "GenerateRequest":
|
||||
@@ -1096,6 +1097,8 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest:
|
||||
kw["video_data"] = None if x is None else decode_MediaInput(x)
|
||||
elif key == "audio_data":
|
||||
kw["audio_data"] = None if x is None else decode_MediaInput(x)
|
||||
elif key == "extra_key":
|
||||
kw["extra_key"] = None if x is None else decode_StringOrList(x)
|
||||
return GenerateRequest(**kw)
|
||||
|
||||
|
||||
@@ -1145,6 +1148,8 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]:
|
||||
d["video_data"] = encode_MediaInput(v.video_data)
|
||||
if v.audio_data is not None:
|
||||
d["audio_data"] = encode_MediaInput(v.audio_data)
|
||||
if v.extra_key is not None:
|
||||
d["extra_key"] = encode_StringOrList(v.extra_key)
|
||||
return d
|
||||
|
||||
|
||||
|
||||
@@ -374,6 +374,11 @@ pub struct GenerateRequest {
|
||||
pub video_data: ::core::option::Option<MediaInput>,
|
||||
#[prost(message, optional, tag = "22")]
|
||||
pub audio_data: ::core::option::Option<MediaInput>,
|
||||
/// Prefix-cache namespace: requests share cached prefixes only under the same
|
||||
/// key. One key for every batch item, or one per item; an empty key means
|
||||
/// none.
|
||||
#[prost(message, optional, tag = "23")]
|
||||
pub extra_key: ::core::option::Option<StringOrList>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct SamplingParamsOrList {
|
||||
|
||||
@@ -1590,6 +1590,9 @@ impl ::serde::Serialize for GenerateRequest {
|
||||
if let Some(v) = &self.audio_data {
|
||||
::serde::ser::SerializeMap::serialize_entry(&mut map, "audio_data", v)?;
|
||||
}
|
||||
if let Some(v) = &self.extra_key {
|
||||
::serde::ser::SerializeMap::serialize_entry(&mut map, "extra_key", v)?;
|
||||
}
|
||||
::serde::ser::SerializeMap::end(map)
|
||||
}
|
||||
}
|
||||
@@ -1628,6 +1631,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
||||
let mut f_mm_hashes = None;
|
||||
let mut f_video_data = None;
|
||||
let mut f_audio_data = None;
|
||||
let mut f_extra_key = None;
|
||||
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
||||
match key.as_ref() {
|
||||
"rid" => {
|
||||
@@ -1784,6 +1788,12 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
||||
}
|
||||
f_audio_data = Some(map.next_value::<Option<MediaInput>>()?);
|
||||
}
|
||||
"extra_key" => {
|
||||
if f_extra_key.is_some() {
|
||||
return Err(::serde::de::Error::duplicate_field("extra_key"));
|
||||
}
|
||||
f_extra_key = Some(map.next_value::<Option<StringOrList>>()?);
|
||||
}
|
||||
_ => {
|
||||
map.next_value::<::serde::de::IgnoredAny>()?;
|
||||
}
|
||||
@@ -1812,6 +1822,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
||||
mm_hashes: f_mm_hashes.flatten(),
|
||||
video_data: f_video_data.flatten(),
|
||||
audio_data: f_audio_data.flatten(),
|
||||
extra_key: f_extra_key.flatten(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,9 +43,9 @@ wire_struct! {
|
||||
lora_id: (),
|
||||
custom_logit_processor: (),
|
||||
positional_embed_overrides: (),
|
||||
/// PD-disaggregation block — the last fields emitted; everything after
|
||||
/// `disagg_prefill_dp_rank` in Python has a msgspec default and is
|
||||
/// omitted (short arrays decode with defaulted tails).
|
||||
/// PD-disaggregation block. Everything after `extra_key` (the last
|
||||
/// field emitted) has a msgspec default in Python and is omitted
|
||||
/// (short arrays decode with defaulted tails).
|
||||
bootstrap_host: Option<&'a str>,
|
||||
bootstrap_port: Option<i64>,
|
||||
bootstrap_room: Option<i64>,
|
||||
@@ -53,6 +53,14 @@ wire_struct! {
|
||||
decode_tp_size: Option<i64>,
|
||||
routed_dp_rank: Option<i64>,
|
||||
disagg_prefill_dp_rank: Option<i64>,
|
||||
/// The scheduler decodes this header by position (Python's
|
||||
/// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so the
|
||||
/// fields between `disagg_prefill_dp_rank` and `extra_key` must be sent,
|
||||
/// at their defaults, for `extra_key` to land in its slot.
|
||||
routing_key: (),
|
||||
require_reasoning: bool,
|
||||
priority: (),
|
||||
extra_key: Option<&'a str>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,6 +131,10 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> {
|
||||
decode_tp_size: req.decode_tp_size,
|
||||
routed_dp_rank: req.routed_dp_rank,
|
||||
disagg_prefill_dp_rank: req.disagg_prefill_dp_rank,
|
||||
routing_key: (),
|
||||
require_reasoning: false,
|
||||
priority: (),
|
||||
extra_key: req.extra_key.as_deref(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -235,9 +247,9 @@ mod tests {
|
||||
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
|
||||
let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap();
|
||||
let arr = val.as_array().expect("array");
|
||||
// msgspec requires >= 14 (through `stream`); we emit 32 (through
|
||||
// `disagg_prefill_dp_rank`). Trailing defaulted fields are omitted.
|
||||
assert_eq!(arr.len(), 32, "header ends at disagg_prefill_dp_rank");
|
||||
// msgspec requires >= 14 (through `stream`); we emit 36 (through
|
||||
// `extra_key`). Trailing defaulted fields are omitted.
|
||||
assert_eq!(arr.len(), 36, "header ends at extra_key");
|
||||
assert_eq!(arr[0].as_str(), Some("TokenizedGenerateReqInput"));
|
||||
assert_eq!(arr[1].as_str(), Some("r1"));
|
||||
assert!(arr[5].is_nil(), "idx 5 must be input_embeds (nil)");
|
||||
@@ -271,7 +283,7 @@ mod tests {
|
||||
/// block (17–24) holding its defaults — a shift here silently routes KV
|
||||
/// transfers to the wrong host/room.
|
||||
#[test]
|
||||
fn header_bootstrap_block_is_positionally_aligned() {
|
||||
fn header_bootstrap_block_and_extra_key_are_positionally_aligned() {
|
||||
let req = GenerateRequest {
|
||||
rid: "r1".into(),
|
||||
text: Some("hi".into()),
|
||||
@@ -282,6 +294,7 @@ mod tests {
|
||||
decode_tp_size: Some(2),
|
||||
routed_dp_rank: Some(3),
|
||||
disagg_prefill_dp_rank: Some(4),
|
||||
extra_key: Some("salt".into()),
|
||||
..Default::default()
|
||||
};
|
||||
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
|
||||
@@ -300,5 +313,9 @@ mod tests {
|
||||
assert_eq!(arr[29].as_i64(), Some(2), "decode_tp_size at 29");
|
||||
assert_eq!(arr[30].as_i64(), Some(3), "routed_dp_rank at 30");
|
||||
assert_eq!(arr[31].as_i64(), Some(4), "disagg_prefill_dp_rank at 31");
|
||||
assert!(arr[32].is_nil(), "routing_key at 32");
|
||||
assert_eq!(arr[33].as_bool(), Some(false), "require_reasoning at 33");
|
||||
assert!(arr[34].is_nil(), "priority at 34");
|
||||
assert_eq!(arr[35].as_str(), Some("salt"), "extra_key at 35");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,6 +79,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
mm_hashes,
|
||||
video_data,
|
||||
audio_data,
|
||||
extra_key,
|
||||
} = req;
|
||||
let rid = rid.and_then(wire::string_or_list);
|
||||
let text = text.and_then(wire::string_or_list);
|
||||
@@ -103,6 +104,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
let video_data = video_data.map(wire::media_input).transpose()?;
|
||||
let audio_data = audio_data.map(wire::media_input).transpose()?;
|
||||
let mm_hashes = mm_hashes.and_then(wire::string_list_or_list);
|
||||
let extra_key = extra_key.and_then(wire::string_or_list);
|
||||
|
||||
// Cap the batch BEFORE the columns below allocate anything. Reading the
|
||||
// declared length off the input costs nothing; the previous placement (after
|
||||
@@ -270,6 +272,17 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
} = normalize_bootstrap_columns(bootstrap_host, bootstrap_port, bootstrap_room, n)?;
|
||||
let bootstrap_pair_keys = flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?);
|
||||
let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?);
|
||||
// Python `_normalize_extra_key`: one key broadcasts, a list is per item
|
||||
// (batch requests only), and an empty key means none.
|
||||
if !is_batch && matches!(extra_key, Some(OneOrMany::Many(_))) {
|
||||
return Err(Error::Validation(
|
||||
"extra_key should be a string for a single request.".into(),
|
||||
));
|
||||
}
|
||||
let extra_keys: Vec<Option<String>> = fan_out(extra_key, n, "extra_key")?
|
||||
.into_iter()
|
||||
.map(|key| key.filter(|key| !key.is_empty()))
|
||||
.collect();
|
||||
// `mm_hashes` has no batch form: honoring it only here would give the two
|
||||
// servers different prefix-cache keys for the same body. Reject instead of
|
||||
// dropping it silently as Python does — the field exists to align a
|
||||
@@ -311,6 +324,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
bootstrap_rooms,
|
||||
bootstrap_pair_keys,
|
||||
decode_tp_sizes,
|
||||
extra_keys,
|
||||
images,
|
||||
videos,
|
||||
audios,
|
||||
@@ -331,6 +345,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
bootstrap_room,
|
||||
bootstrap_pair_key,
|
||||
decode_tp_size,
|
||||
extra_key,
|
||||
image_data,
|
||||
video_data,
|
||||
audio_data,
|
||||
@@ -360,6 +375,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
decode_tp_size,
|
||||
routed_dp_rank,
|
||||
disagg_prefill_dp_rank,
|
||||
extra_key,
|
||||
mm: pack_mm(image_data, video_data, audio_data),
|
||||
mm_buffers: Vec::new(),
|
||||
},
|
||||
@@ -502,6 +518,9 @@ pub struct GenerateRequest {
|
||||
/// so these are pure passthrough for the scheduler/LB protocol.
|
||||
pub routed_dp_rank: Option<i64>,
|
||||
pub disagg_prefill_dp_rank: Option<i64>,
|
||||
/// Prefix-cache namespace: requests share cached prefixes only under the
|
||||
/// same key.
|
||||
pub extra_key: Option<String>,
|
||||
/// Multimodal inputs. Consumed by the Encoding stage, which ships them to
|
||||
/// the MM worker pool; never read by the tokenizer or serialized onto the
|
||||
/// scheduler header. Boxed so the common text-only request doesn't grow
|
||||
@@ -807,7 +826,6 @@ mod tests {
|
||||
fn unported_generate_req_input_fields_are_ignored() {
|
||||
for field in [
|
||||
r#""priority": 3"#,
|
||||
r#""extra_key": "k""#,
|
||||
r#""session_id": "s""#,
|
||||
r#""session_params": {"a": 1}"#,
|
||||
r#""return_sampling_mask": true"#,
|
||||
@@ -1189,6 +1207,26 @@ mod tests {
|
||||
assert!(err.to_string().contains("bootstrap_room"), "{err}");
|
||||
}
|
||||
|
||||
/// `extra_key` fans out like Python `_normalize_extra_key`: one key
|
||||
/// broadcasts, a list is per item and must match the batch (and needs a
|
||||
/// batch), and an empty key means none.
|
||||
#[test]
|
||||
fn extra_key_fans_out() {
|
||||
let (ps, _) = requests(r#"{"text": "a", "extra_key": "salt"}"#).unwrap();
|
||||
assert_eq!(ps[0].extra_key.as_deref(), Some("salt"));
|
||||
let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": "salt"}"#).unwrap();
|
||||
assert!(ps.iter().all(|p| p.extra_key.as_deref() == Some("salt")));
|
||||
let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": ["k", ""]}"#).unwrap();
|
||||
assert_eq!(ps[0].extra_key.as_deref(), Some("k"));
|
||||
assert_eq!(ps[1].extra_key, None);
|
||||
let (ps, _) = requests(r#"{"text": "a", "extra_key": ""}"#).unwrap();
|
||||
assert_eq!(ps[0].extra_key, None);
|
||||
assert!(requests(r#"{"text": "a", "extra_key": ["k"]}"#).is_err());
|
||||
|
||||
let err = requests(r#"{"text": ["a", "b"], "extra_key": ["k"]}"#).unwrap_err();
|
||||
assert!(err.to_string().contains("extra_key"), "{err}");
|
||||
}
|
||||
|
||||
/// The PD router (mini_lb) and PD-warmup payload shapes must parse. The
|
||||
/// router sends `bootstrap_port: [null, …]` when no port was configured
|
||||
/// (the scheduler fills its default) — null list elements must parse.
|
||||
|
||||
@@ -50,7 +50,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
||||
"data_parallel_rank",
|
||||
"encoder_urls",
|
||||
"external_trace_header",
|
||||
"extra_key",
|
||||
"http_worker_ipc",
|
||||
"image_max_dynamic_patch",
|
||||
"images_config",
|
||||
|
||||
Reference in New Issue
Block a user