[Fix] sglang-server: Honour extra_key on the Rust frontend's /generate (#44207)

Co-authored-by: fei-xx <fei-xx@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-14 16:44:31 -07:00
committed by GitHub
co-authored by fei-xx shodoco
parent e7d6a88bfc
commit 209f8220d6
7 changed files with 89 additions and 10 deletions
+5 -1
View File
@@ -2,7 +2,7 @@
// rust/sglang-server/src/message/request.rs's GenerateBody (request) and
// api_core/frame.rs's frame shape (response). Unknown JSON keys on the
// request are IGNORED: `deny_unknown_fields` broke real clients that send
// Python-only `GenerateReqInput` fields (priority, extra_key, session_id,
// Python-only `GenerateReqInput` fields (priority, session_id,
// ...), pinned by request.rs's unported-fields test.
syntax = "proto3";
@@ -53,6 +53,10 @@ message GenerateRequest {
optional StringListOrList mm_hashes = 20;
optional MediaInput video_data = 21;
optional MediaInput audio_data = 22;
// Prefix-cache namespace: requests share cached prefixes only under the same
// key. One key for every batch item, or one per item; an empty key means
// none.
StringOrList extra_key = 23;
}
message SamplingParamsOrList {
+5
View File
@@ -257,6 +257,7 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
mm_hashes: Optional[StringListOrList] = None
video_data: Optional[MediaInput] = None
audio_data: Optional[MediaInput] = None
extra_key: Optional[StringOrList] = None
@classmethod
def from_json_value(cls, value: Any) -> "GenerateRequest":
@@ -1096,6 +1097,8 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest:
kw["video_data"] = None if x is None else decode_MediaInput(x)
elif key == "audio_data":
kw["audio_data"] = None if x is None else decode_MediaInput(x)
elif key == "extra_key":
kw["extra_key"] = None if x is None else decode_StringOrList(x)
return GenerateRequest(**kw)
@@ -1145,6 +1148,8 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]:
d["video_data"] = encode_MediaInput(v.video_data)
if v.audio_data is not None:
d["audio_data"] = encode_MediaInput(v.audio_data)
if v.extra_key is not None:
d["extra_key"] = encode_StringOrList(v.extra_key)
return d
@@ -374,6 +374,11 @@ pub struct GenerateRequest {
pub video_data: ::core::option::Option<MediaInput>,
#[prost(message, optional, tag = "22")]
pub audio_data: ::core::option::Option<MediaInput>,
/// Prefix-cache namespace: requests share cached prefixes only under the same
/// key. One key for every batch item, or one per item; an empty key means
/// none.
#[prost(message, optional, tag = "23")]
pub extra_key: ::core::option::Option<StringOrList>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SamplingParamsOrList {
@@ -1590,6 +1590,9 @@ impl ::serde::Serialize for GenerateRequest {
if let Some(v) = &self.audio_data {
::serde::ser::SerializeMap::serialize_entry(&mut map, "audio_data", v)?;
}
if let Some(v) = &self.extra_key {
::serde::ser::SerializeMap::serialize_entry(&mut map, "extra_key", v)?;
}
::serde::ser::SerializeMap::end(map)
}
}
@@ -1628,6 +1631,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
let mut f_mm_hashes = None;
let mut f_video_data = None;
let mut f_audio_data = None;
let mut f_extra_key = None;
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
match key.as_ref() {
"rid" => {
@@ -1784,6 +1788,12 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
}
f_audio_data = Some(map.next_value::<Option<MediaInput>>()?);
}
"extra_key" => {
if f_extra_key.is_some() {
return Err(::serde::de::Error::duplicate_field("extra_key"));
}
f_extra_key = Some(map.next_value::<Option<StringOrList>>()?);
}
_ => {
map.next_value::<::serde::de::IgnoredAny>()?;
}
@@ -1812,6 +1822,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
mm_hashes: f_mm_hashes.flatten(),
video_data: f_video_data.flatten(),
audio_data: f_audio_data.flatten(),
extra_key: f_extra_key.flatten(),
})
}
}
+24 -7
View File
@@ -43,9 +43,9 @@ wire_struct! {
lora_id: (),
custom_logit_processor: (),
positional_embed_overrides: (),
/// PD-disaggregation block — the last fields emitted; everything after
/// `disagg_prefill_dp_rank` in Python has a msgspec default and is
/// omitted (short arrays decode with defaulted tails).
/// PD-disaggregation block. Everything after `extra_key` (the last
/// field emitted) has a msgspec default in Python and is omitted
/// (short arrays decode with defaulted tails).
bootstrap_host: Option<&'a str>,
bootstrap_port: Option<i64>,
bootstrap_room: Option<i64>,
@@ -53,6 +53,14 @@ wire_struct! {
decode_tp_size: Option<i64>,
routed_dp_rank: Option<i64>,
disagg_prefill_dp_rank: Option<i64>,
/// The scheduler decodes this header by position (Python's
/// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so the
/// fields between `disagg_prefill_dp_rank` and `extra_key` must be sent,
/// at their defaults, for `extra_key` to land in its slot.
routing_key: (),
require_reasoning: bool,
priority: (),
extra_key: Option<&'a str>,
}
}
@@ -123,6 +131,10 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> {
decode_tp_size: req.decode_tp_size,
routed_dp_rank: req.routed_dp_rank,
disagg_prefill_dp_rank: req.disagg_prefill_dp_rank,
routing_key: (),
require_reasoning: false,
priority: (),
extra_key: req.extra_key.as_deref(),
}
}
}
@@ -235,9 +247,9 @@ mod tests {
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap();
let arr = val.as_array().expect("array");
// msgspec requires >= 14 (through `stream`); we emit 32 (through
// `disagg_prefill_dp_rank`). Trailing defaulted fields are omitted.
assert_eq!(arr.len(), 32, "header ends at disagg_prefill_dp_rank");
// msgspec requires >= 14 (through `stream`); we emit 36 (through
// `extra_key`). Trailing defaulted fields are omitted.
assert_eq!(arr.len(), 36, "header ends at extra_key");
assert_eq!(arr[0].as_str(), Some("TokenizedGenerateReqInput"));
assert_eq!(arr[1].as_str(), Some("r1"));
assert!(arr[5].is_nil(), "idx 5 must be input_embeds (nil)");
@@ -271,7 +283,7 @@ mod tests {
/// block (17–24) holding its defaults — a shift here silently routes KV
/// transfers to the wrong host/room.
#[test]
fn header_bootstrap_block_is_positionally_aligned() {
fn header_bootstrap_block_and_extra_key_are_positionally_aligned() {
let req = GenerateRequest {
rid: "r1".into(),
text: Some("hi".into()),
@@ -282,6 +294,7 @@ mod tests {
decode_tp_size: Some(2),
routed_dp_rank: Some(3),
disagg_prefill_dp_rank: Some(4),
extra_key: Some("salt".into()),
..Default::default()
};
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
@@ -300,5 +313,9 @@ mod tests {
assert_eq!(arr[29].as_i64(), Some(2), "decode_tp_size at 29");
assert_eq!(arr[30].as_i64(), Some(3), "routed_dp_rank at 30");
assert_eq!(arr[31].as_i64(), Some(4), "disagg_prefill_dp_rank at 31");
assert!(arr[32].is_nil(), "routing_key at 32");
assert_eq!(arr[33].as_bool(), Some(false), "require_reasoning at 33");
assert!(arr[34].is_nil(), "priority at 34");
assert_eq!(arr[35].as_str(), Some("salt"), "extra_key at 35");
}
}
+39 -1
View File
@@ -79,6 +79,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
mm_hashes,
video_data,
audio_data,
extra_key,
} = req;
let rid = rid.and_then(wire::string_or_list);
let text = text.and_then(wire::string_or_list);
@@ -103,6 +104,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
let video_data = video_data.map(wire::media_input).transpose()?;
let audio_data = audio_data.map(wire::media_input).transpose()?;
let mm_hashes = mm_hashes.and_then(wire::string_list_or_list);
let extra_key = extra_key.and_then(wire::string_or_list);
// Cap the batch BEFORE the columns below allocate anything. Reading the
// declared length off the input costs nothing; the previous placement (after
@@ -270,6 +272,17 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
} = normalize_bootstrap_columns(bootstrap_host, bootstrap_port, bootstrap_room, n)?;
let bootstrap_pair_keys = flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?);
let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?);
// Python `_normalize_extra_key`: one key broadcasts, a list is per item
// (batch requests only), and an empty key means none.
if !is_batch && matches!(extra_key, Some(OneOrMany::Many(_))) {
return Err(Error::Validation(
"extra_key should be a string for a single request.".into(),
));
}
let extra_keys: Vec<Option<String>> = fan_out(extra_key, n, "extra_key")?
.into_iter()
.map(|key| key.filter(|key| !key.is_empty()))
.collect();
// `mm_hashes` has no batch form: honoring it only here would give the two
// servers different prefix-cache keys for the same body. Reject instead of
// dropping it silently as Python does — the field exists to align a
@@ -311,6 +324,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
bootstrap_rooms,
bootstrap_pair_keys,
decode_tp_sizes,
extra_keys,
images,
videos,
audios,
@@ -331,6 +345,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
bootstrap_room,
bootstrap_pair_key,
decode_tp_size,
extra_key,
image_data,
video_data,
audio_data,
@@ -360,6 +375,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
decode_tp_size,
routed_dp_rank,
disagg_prefill_dp_rank,
extra_key,
mm: pack_mm(image_data, video_data, audio_data),
mm_buffers: Vec::new(),
},
@@ -502,6 +518,9 @@ pub struct GenerateRequest {
/// so these are pure passthrough for the scheduler/LB protocol.
pub routed_dp_rank: Option<i64>,
pub disagg_prefill_dp_rank: Option<i64>,
/// Prefix-cache namespace: requests share cached prefixes only under the
/// same key.
pub extra_key: Option<String>,
/// Multimodal inputs. Consumed by the Encoding stage, which ships them to
/// the MM worker pool; never read by the tokenizer or serialized onto the
/// scheduler header. Boxed so the common text-only request doesn't grow
@@ -807,7 +826,6 @@ mod tests {
fn unported_generate_req_input_fields_are_ignored() {
for field in [
r#""priority": 3"#,
r#""extra_key": "k""#,
r#""session_id": "s""#,
r#""session_params": {"a": 1}"#,
r#""return_sampling_mask": true"#,
@@ -1189,6 +1207,26 @@ mod tests {
assert!(err.to_string().contains("bootstrap_room"), "{err}");
}
/// `extra_key` fans out like Python `_normalize_extra_key`: one key
/// broadcasts, a list is per item and must match the batch (and needs a
/// batch), and an empty key means none.
#[test]
fn extra_key_fans_out() {
let (ps, _) = requests(r#"{"text": "a", "extra_key": "salt"}"#).unwrap();
assert_eq!(ps[0].extra_key.as_deref(), Some("salt"));
let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": "salt"}"#).unwrap();
assert!(ps.iter().all(|p| p.extra_key.as_deref() == Some("salt")));
let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": ["k", ""]}"#).unwrap();
assert_eq!(ps[0].extra_key.as_deref(), Some("k"));
assert_eq!(ps[1].extra_key, None);
let (ps, _) = requests(r#"{"text": "a", "extra_key": ""}"#).unwrap();
assert_eq!(ps[0].extra_key, None);
assert!(requests(r#"{"text": "a", "extra_key": ["k"]}"#).is_err());
let err = requests(r#"{"text": ["a", "b"], "extra_key": ["k"]}"#).unwrap_err();
assert!(err.to_string().contains("extra_key"), "{err}");
}
/// The PD router (mini_lb) and PD-warmup payload shapes must parse. The
/// router sends `bootstrap_port: [null, …]` when no port was configured
/// (the scheduler fills its default) — null list elements must parse.
@@ -50,7 +50,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
"data_parallel_rank",
"encoder_urls",
"external_trace_header",
"extra_key",
"http_worker_ipc",
"image_max_dynamic_patch",
"images_config",