diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto index bdc9a9096c..f183bef8f0 100644 --- a/proto/sglang/api/v1/generate.proto +++ b/proto/sglang/api/v1/generate.proto @@ -2,7 +2,7 @@ // rust/sglang-server/src/message/request.rs's GenerateBody (request) and // api_core/frame.rs's frame shape (response). Unknown JSON keys on the // request are IGNORED: `deny_unknown_fields` broke real clients that send -// Python-only `GenerateReqInput` fields (priority, extra_key, session_id, +// Python-only `GenerateReqInput` fields (priority, session_id, // ...), pinned by request.rs's unported-fields test. syntax = "proto3"; @@ -53,6 +53,10 @@ message GenerateRequest { optional StringListOrList mm_hashes = 20; optional MediaInput video_data = 21; optional MediaInput audio_data = 22; + // Prefix-cache namespace: requests share cached prefixes only under the same + // key. One key for every batch item, or one per item; an empty key means + // none. + StringOrList extra_key = 23; } message SamplingParamsOrList { diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index 3861803b83..57c8c07d07 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -257,6 +257,7 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): mm_hashes: Optional[StringListOrList] = None video_data: Optional[MediaInput] = None audio_data: Optional[MediaInput] = None + extra_key: Optional[StringOrList] = None @classmethod def from_json_value(cls, value: Any) -> "GenerateRequest": @@ -1096,6 +1097,8 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest: kw["video_data"] = None if x is None else decode_MediaInput(x) elif key == "audio_data": kw["audio_data"] = None if x is None else decode_MediaInput(x) + elif key == "extra_key": + kw["extra_key"] = None if x is None else decode_StringOrList(x) return GenerateRequest(**kw) @@ -1145,6 +1148,8 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]: d["video_data"] = encode_MediaInput(v.video_data) if v.audio_data is not None: d["audio_data"] = encode_MediaInput(v.audio_data) + if v.extra_key is not None: + d["extra_key"] = encode_StringOrList(v.extra_key) return d diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index b3426f3718..eb1df914a7 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -374,6 +374,11 @@ pub struct GenerateRequest { pub video_data: ::core::option::Option, #[prost(message, optional, tag = "22")] pub audio_data: ::core::option::Option, + /// Prefix-cache namespace: requests share cached prefixes only under the same + /// key. One key for every batch item, or one per item; an empty key means + /// none. + #[prost(message, optional, tag = "23")] + pub extra_key: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct SamplingParamsOrList { diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index e356ea3dc8..ff73d65b31 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -1590,6 +1590,9 @@ impl ::serde::Serialize for GenerateRequest { if let Some(v) = &self.audio_data { ::serde::ser::SerializeMap::serialize_entry(&mut map, "audio_data", v)?; } + if let Some(v) = &self.extra_key { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "extra_key", v)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -1628,6 +1631,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { let mut f_mm_hashes = None; let mut f_video_data = None; let mut f_audio_data = None; + let mut f_extra_key = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "rid" => { @@ -1784,6 +1788,12 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { } f_audio_data = Some(map.next_value::>()?); } + "extra_key" => { + if f_extra_key.is_some() { + return Err(::serde::de::Error::duplicate_field("extra_key")); + } + f_extra_key = Some(map.next_value::>()?); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -1812,6 +1822,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { mm_hashes: f_mm_hashes.flatten(), video_data: f_video_data.flatten(), audio_data: f_audio_data.flatten(), + extra_key: f_extra_key.flatten(), }) } } diff --git a/rust/sglang-server/src/message/io_struct.rs b/rust/sglang-server/src/message/io_struct.rs index 2032fe8ad3..0c6499afca 100644 --- a/rust/sglang-server/src/message/io_struct.rs +++ b/rust/sglang-server/src/message/io_struct.rs @@ -43,9 +43,9 @@ wire_struct! { lora_id: (), custom_logit_processor: (), positional_embed_overrides: (), - /// PD-disaggregation block — the last fields emitted; everything after - /// `disagg_prefill_dp_rank` in Python has a msgspec default and is - /// omitted (short arrays decode with defaulted tails). + /// PD-disaggregation block. Everything after `extra_key` (the last + /// field emitted) has a msgspec default in Python and is omitted + /// (short arrays decode with defaulted tails). bootstrap_host: Option<&'a str>, bootstrap_port: Option, bootstrap_room: Option, @@ -53,6 +53,14 @@ wire_struct! { decode_tp_size: Option, routed_dp_rank: Option, disagg_prefill_dp_rank: Option, + /// The scheduler decodes this header by position (Python's + /// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so the + /// fields between `disagg_prefill_dp_rank` and `extra_key` must be sent, + /// at their defaults, for `extra_key` to land in its slot. + routing_key: (), + require_reasoning: bool, + priority: (), + extra_key: Option<&'a str>, } } @@ -123,6 +131,10 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> { decode_tp_size: req.decode_tp_size, routed_dp_rank: req.routed_dp_rank, disagg_prefill_dp_rank: req.disagg_prefill_dp_rank, + routing_key: (), + require_reasoning: false, + priority: (), + extra_key: req.extra_key.as_deref(), } } } @@ -235,9 +247,9 @@ mod tests { let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap(); let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap(); let arr = val.as_array().expect("array"); - // msgspec requires >= 14 (through `stream`); we emit 32 (through - // `disagg_prefill_dp_rank`). Trailing defaulted fields are omitted. - assert_eq!(arr.len(), 32, "header ends at disagg_prefill_dp_rank"); + // msgspec requires >= 14 (through `stream`); we emit 36 (through + // `extra_key`). Trailing defaulted fields are omitted. + assert_eq!(arr.len(), 36, "header ends at extra_key"); assert_eq!(arr[0].as_str(), Some("TokenizedGenerateReqInput")); assert_eq!(arr[1].as_str(), Some("r1")); assert!(arr[5].is_nil(), "idx 5 must be input_embeds (nil)"); @@ -271,7 +283,7 @@ mod tests { /// block (17–24) holding its defaults — a shift here silently routes KV /// transfers to the wrong host/room. #[test] - fn header_bootstrap_block_is_positionally_aligned() { + fn header_bootstrap_block_and_extra_key_are_positionally_aligned() { let req = GenerateRequest { rid: "r1".into(), text: Some("hi".into()), @@ -282,6 +294,7 @@ mod tests { decode_tp_size: Some(2), routed_dp_rank: Some(3), disagg_prefill_dp_rank: Some(4), + extra_key: Some("salt".into()), ..Default::default() }; let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap(); @@ -300,5 +313,9 @@ mod tests { assert_eq!(arr[29].as_i64(), Some(2), "decode_tp_size at 29"); assert_eq!(arr[30].as_i64(), Some(3), "routed_dp_rank at 30"); assert_eq!(arr[31].as_i64(), Some(4), "disagg_prefill_dp_rank at 31"); + assert!(arr[32].is_nil(), "routing_key at 32"); + assert_eq!(arr[33].as_bool(), Some(false), "require_reasoning at 33"); + assert!(arr[34].is_nil(), "priority at 34"); + assert_eq!(arr[35].as_str(), Some("salt"), "extra_key at 35"); } } diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index 3afd9e6dc2..deb11231b4 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -79,6 +79,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec Result<(Vec Result<(Vec> = fan_out(extra_key, n, "extra_key")? + .into_iter() + .map(|key| key.filter(|key| !key.is_empty())) + .collect(); // `mm_hashes` has no batch form: honoring it only here would give the two // servers different prefix-cache keys for the same body. Reject instead of // dropping it silently as Python does — the field exists to align a @@ -311,6 +324,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec Result<(Vec Result<(Vec, pub disagg_prefill_dp_rank: Option, + /// Prefix-cache namespace: requests share cached prefixes only under the + /// same key. + pub extra_key: Option, /// Multimodal inputs. Consumed by the Encoding stage, which ships them to /// the MM worker pool; never read by the tokenizer or serialized onto the /// scheduler header. Boxed so the common text-only request doesn't grow @@ -807,7 +826,6 @@ mod tests { fn unported_generate_req_input_fields_are_ignored() { for field in [ r#""priority": 3"#, - r#""extra_key": "k""#, r#""session_id": "s""#, r#""session_params": {"a": 1}"#, r#""return_sampling_mask": true"#, @@ -1189,6 +1207,26 @@ mod tests { assert!(err.to_string().contains("bootstrap_room"), "{err}"); } + /// `extra_key` fans out like Python `_normalize_extra_key`: one key + /// broadcasts, a list is per item and must match the batch (and needs a + /// batch), and an empty key means none. + #[test] + fn extra_key_fans_out() { + let (ps, _) = requests(r#"{"text": "a", "extra_key": "salt"}"#).unwrap(); + assert_eq!(ps[0].extra_key.as_deref(), Some("salt")); + let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": "salt"}"#).unwrap(); + assert!(ps.iter().all(|p| p.extra_key.as_deref() == Some("salt"))); + let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": ["k", ""]}"#).unwrap(); + assert_eq!(ps[0].extra_key.as_deref(), Some("k")); + assert_eq!(ps[1].extra_key, None); + let (ps, _) = requests(r#"{"text": "a", "extra_key": ""}"#).unwrap(); + assert_eq!(ps[0].extra_key, None); + assert!(requests(r#"{"text": "a", "extra_key": ["k"]}"#).is_err()); + + let err = requests(r#"{"text": ["a", "b"], "extra_key": ["k"]}"#).unwrap_err(); + assert!(err.to_string().contains("extra_key"), "{err}"); + } + /// The PD router (mini_lb) and PD-warmup payload shapes must parse. The /// router sends `bootstrap_port: [null, …]` when no port was configured /// (the scheduler fills its default) — null list elements must parse. diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 1b2b7a8ad2..888a3eb453 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -50,7 +50,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "data_parallel_rank", "encoder_urls", "external_trace_header", - "extra_key", "http_worker_ipc", "image_max_dynamic_patch", "images_config",