From 553068af25b040b291ae863c8aebfbda9e3d3392 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 16 Oct 2026 10:38:09 -0700 Subject: [PATCH] [Fix] rust-server: Honour cache_salt and require_reasoning on /generate (#44533) Co-authored-by: metamergebot Co-authored-by: shodoco --- proto/sglang/api/v1/generate.proto | 12 +++ python/sglang/api/v1/api_types.py | 20 +++++ .../src/generated/sglang.api.v1.rs | 16 ++++ .../src/generated/sglang.api.v1.serde.rs | 62 ++++++++++++++ rust/sglang-server/src/message/io_struct.rs | 52 +++++++++--- rust/sglang-server/src/message/request.rs | 82 +++++++++++++++---- test/registered/unit/api/v1/test_parity.py | 4 - 7 files changed, 219 insertions(+), 29 deletions(-) diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto index 0c71613c20..5a3734e102 100644 --- a/proto/sglang/api/v1/generate.proto +++ b/proto/sglang/api/v1/generate.proto @@ -60,6 +60,18 @@ message GenerateRequest { // Scheduling priority, one value for every batch item; the scheduler // validates it against its priority-scheduling mode. optional int64 priority = 24; + // Prefix-cache salt: a second namespace beside `extra_key`, normalized the + // same way. + StringOrList cache_salt = 25; + // Apply grammar constraints only after the reasoning section, for every + // batch item. + optional bool require_reasoning = 26; + // Fields this server cannot honour. Unknown keys are ignored, but + // dropping these would silently change the result (base-model output for a + // LoRA request, a stateless answer for a session request), so a non-null + // value is rejected instead. + optional string lora_path = 27 [(sglang.json.v1.field_json).raw_json = true]; + optional string session_params = 28 [(sglang.json.v1.field_json).raw_json = true]; } message SamplingParamsOrList { diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index afc90096c5..562b79b323 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -259,6 +259,10 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): audio_data: Optional[MediaInput] = None extra_key: Optional[StringOrList] = None priority: Optional[int] = None + cache_salt: Optional[StringOrList] = None + require_reasoning: Optional[bool] = None + lora_path: Optional[Any] = None + session_params: Optional[Any] = None @classmethod def from_json_value(cls, value: Any) -> "GenerateRequest": @@ -1104,6 +1108,14 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest: kw["extra_key"] = None if x is None else decode_StringOrList(x) elif key == "priority": kw["priority"] = None if x is None else _expect_int(x, expected="i64") + elif key == "cache_salt": + kw["cache_salt"] = None if x is None else decode_StringOrList(x) + elif key == "require_reasoning": + kw["require_reasoning"] = None if x is None else _expect_bool(x, expected="a boolean") + elif key == "lora_path": + kw["lora_path"] = x + elif key == "session_params": + kw["session_params"] = x return GenerateRequest(**kw) @@ -1157,6 +1169,14 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]: d["extra_key"] = encode_StringOrList(v.extra_key) if v.priority is not None: d["priority"] = v.priority + if v.cache_salt is not None: + d["cache_salt"] = encode_StringOrList(v.cache_salt) + if v.require_reasoning is not None: + d["require_reasoning"] = v.require_reasoning + if v.lora_path is not None: + d["lora_path"] = v.lora_path + if v.session_params is not None: + d["session_params"] = v.session_params return d diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index 9670bb0609..a926654030 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -383,6 +383,22 @@ pub struct GenerateRequest { /// validates it against its priority-scheduling mode. #[prost(int64, optional, tag = "24")] pub priority: ::core::option::Option, + /// Prefix-cache salt: a second namespace beside `extra_key`, normalized the + /// same way. + #[prost(message, optional, tag = "25")] + pub cache_salt: ::core::option::Option, + /// Apply grammar constraints only after the reasoning section, for every + /// batch item. + #[prost(bool, optional, tag = "26")] + pub require_reasoning: ::core::option::Option, + /// Fields this server cannot honour. Unknown keys are ignored, but + /// dropping these would silently change the result (base-model output for a + /// LoRA request, a stateless answer for a session request), so a non-null + /// value is rejected instead. + #[prost(string, optional, tag = "27")] + pub lora_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "28")] + pub session_params: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct SamplingParamsOrList { diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index 39faf1fb5c..1b804b54b9 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -1596,6 +1596,28 @@ impl ::serde::Serialize for GenerateRequest { if let Some(v) = &self.priority { ::serde::ser::SerializeMap::serialize_entry(&mut map, "priority", v)?; } + if let Some(v) = &self.cache_salt { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "cache_salt", v)?; + } + if let Some(v) = &self.require_reasoning { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "require_reasoning", v)?; + } + if let Some(v) = &self.lora_path { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "lora_path", + &::serde_json::from_str::<::serde_json::Value>(v) + .unwrap_or(::serde_json::Value::Null), + )?; + } + if let Some(v) = &self.session_params { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "session_params", + &::serde_json::from_str::<::serde_json::Value>(v) + .unwrap_or(::serde_json::Value::Null), + )?; + } ::serde::ser::SerializeMap::end(map) } } @@ -1636,6 +1658,10 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { let mut f_audio_data = None; let mut f_extra_key = None; let mut f_priority = None; + let mut f_cache_salt = None; + let mut f_require_reasoning = None; + let mut f_lora_path = None; + let mut f_session_params = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "rid" => { @@ -1804,6 +1830,38 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { } f_priority = Some(map.next_value::>()?); } + "cache_salt" => { + if f_cache_salt.is_some() { + return Err(::serde::de::Error::duplicate_field("cache_salt")); + } + f_cache_salt = Some(map.next_value::>()?); + } + "require_reasoning" => { + if f_require_reasoning.is_some() { + return Err(::serde::de::Error::duplicate_field( + "require_reasoning", + )); + } + f_require_reasoning = Some(map.next_value::>()?); + } + "lora_path" => { + if f_lora_path.is_some() { + return Err(::serde::de::Error::duplicate_field("lora_path")); + } + f_lora_path = Some( + map.next_value::>()? + .map(|v| v.to_string()), + ); + } + "session_params" => { + if f_session_params.is_some() { + return Err(::serde::de::Error::duplicate_field("session_params")); + } + f_session_params = Some( + map.next_value::>()? + .map(|v| v.to_string()), + ); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -1834,6 +1892,10 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { audio_data: f_audio_data.flatten(), extra_key: f_extra_key.flatten(), priority: f_priority.flatten(), + cache_salt: f_cache_salt.flatten(), + require_reasoning: f_require_reasoning.flatten(), + lora_path: f_lora_path.flatten(), + session_params: f_session_params.flatten(), }) } } diff --git a/rust/sglang-server/src/message/io_struct.rs b/rust/sglang-server/src/message/io_struct.rs index 92b65e0749..2a185b5c8c 100644 --- a/rust/sglang-server/src/message/io_struct.rs +++ b/rust/sglang-server/src/message/io_struct.rs @@ -43,7 +43,7 @@ wire_struct! { lora_id: (), custom_logit_processor: (), positional_embed_overrides: (), - /// PD-disaggregation block. Everything after `extra_key` (the last + /// PD-disaggregation block. Everything after `cache_salt` (the last /// field emitted) has a msgspec default in Python and is omitted /// (short arrays decode with defaulted tails). bootstrap_host: Option<&'a str>, @@ -55,12 +55,24 @@ wire_struct! { disagg_prefill_dp_rank: Option, /// The scheduler decodes this header by position (Python's /// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so - /// `routing_key` and `require_reasoning` are sent at their defaults for - /// `priority` and `extra_key` to land in their slots. + /// `routing_key` is sent at its default for the fields after it to land + /// in their slots. routing_key: (), require_reasoning: bool, priority: Option, extra_key: Option<&'a str>, + /// Defaults, so `cache_salt`, the prefix cache's second key, lands in + /// its slot. + no_logs: bool, + return_bytes: bool, + return_entropy: bool, + need_wait_for_mm_inputs: (), + num_items_assigned: (), + encoder_urls: (), + multi_item_delimiter_indices: (), + token_indices_to_pool: (), + time_stats: (), + cache_salt: Option<&'a str>, } } @@ -132,9 +144,19 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> { routed_dp_rank: req.routed_dp_rank, disagg_prefill_dp_rank: req.disagg_prefill_dp_rank, routing_key: (), - require_reasoning: false, + require_reasoning: req.require_reasoning, priority: req.priority, extra_key: req.extra_key.as_deref(), + no_logs: false, + return_bytes: false, + return_entropy: false, + need_wait_for_mm_inputs: (), + num_items_assigned: (), + encoder_urls: (), + multi_item_delimiter_indices: (), + token_indices_to_pool: (), + time_stats: (), + cache_salt: req.cache_salt.as_deref(), } } } @@ -247,9 +269,9 @@ mod tests { let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap(); let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap(); let arr = val.as_array().expect("array"); - // msgspec requires >= 14 (through `stream`); we emit 36 (through - // `extra_key`). Trailing defaulted fields are omitted. - assert_eq!(arr.len(), 36, "header ends at extra_key"); + // msgspec requires >= 14 (through `stream`); we emit 46 (through + // `cache_salt`). Trailing defaulted fields are omitted. + assert_eq!(arr.len(), 46, "header ends at cache_salt"); assert_eq!(arr[0].as_str(), Some("TokenizedGenerateReqInput")); assert_eq!(arr[1].as_str(), Some("r1")); assert!(arr[5].is_nil(), "idx 5 must be input_embeds (nil)"); @@ -281,9 +303,10 @@ mod tests { /// The PD block must land on Python's wire indices 25–31, with the filler /// block (17–24) holding its defaults — a shift here silently routes KV - /// transfers to the wrong host/room. + /// transfers to the wrong host/room. The cache keys and `require_reasoning` + /// follow at 33–45. #[test] - fn header_bootstrap_block_and_extra_key_are_positionally_aligned() { + fn header_bootstrap_block_and_cache_keys_are_positionally_aligned() { let req = GenerateRequest { rid: "r1".into(), text: Some("hi".into()), @@ -295,7 +318,9 @@ mod tests { routed_dp_rank: Some(3), disagg_prefill_dp_rank: Some(4), extra_key: Some("salt".into()), + cache_salt: Some("pepper".into()), priority: Some(-2), + require_reasoning: true, ..Default::default() }; let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap(); @@ -315,8 +340,15 @@ mod tests { assert_eq!(arr[30].as_i64(), Some(3), "routed_dp_rank at 30"); assert_eq!(arr[31].as_i64(), Some(4), "disagg_prefill_dp_rank at 31"); assert!(arr[32].is_nil(), "routing_key at 32"); - assert_eq!(arr[33].as_bool(), Some(false), "require_reasoning at 33"); + assert_eq!(arr[33].as_bool(), Some(true), "require_reasoning at 33"); assert_eq!(arr[34].as_i64(), Some(-2), "priority at 34"); assert_eq!(arr[35].as_str(), Some("salt"), "extra_key at 35"); + for (i, slot) in arr.iter().enumerate().take(39).skip(36) { + assert_eq!(slot.as_bool(), Some(false), "idx {i} must be false"); + } + for (i, slot) in arr.iter().enumerate().take(45).skip(39) { + assert!(slot.is_nil(), "idx {i} must be a nil default"); + } + assert_eq!(arr[45].as_str(), Some("pepper"), "cache_salt at 45"); } } diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index 5fd96969f9..d1c78eb3d9 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -81,7 +81,21 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec Result<(Vec Result<(Vec> = fan_out(extra_key, n, "extra_key")? - .into_iter() - .map(|key| key.filter(|key| !key.is_empty())) - .collect(); + // Python `_normalize_extra_key` / `_normalize_cache_salt`: one key + // broadcasts, a list is per item (batch requests only), and an empty key + // means none. + let cache_key_column = |value: Option>, name: &str| { + if !is_batch && matches!(value, Some(OneOrMany::Many(_))) { + return Err(Error::Validation(format!( + "{name} should be a string for a single request." + ))); + } + Ok::>, Error>( + fan_out(value, n, name)? + .into_iter() + .map(|key| key.filter(|key| !key.is_empty())) + .collect(), + ) + }; + let extra_keys = cache_key_column(extra_key, "extra_key")?; + let cache_salts = cache_key_column(cache_salt, "cache_salt")?; // `mm_hashes` has no batch form: honoring it only here would give the two // servers different prefix-cache keys for the same body. Reject instead of // dropping it silently as Python does — the field exists to align a @@ -326,6 +348,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec Result<(Vec Result<(Vec, + /// The prefix cache's second namespace key. + pub cache_salt: Option, /// Scheduling priority, shared by every item of a batch. pub priority: Option, + /// Grammar constraints start after the reasoning section. + pub require_reasoning: bool, /// Multimodal inputs. Consumed by the Encoding stage, which ships them to /// the MM worker pool; never read by the tokenizer or serialized onto the /// scheduler header. Boxed so the common text-only request doesn't grow @@ -826,14 +856,17 @@ mod tests { /// to be declared and dropped by hand just to let `bench_serving` through. /// FastAPI's pydantic dataclass drops extras, so ignoring them is the parity /// behavior; a typo being silently ignored is the same trade Python makes. + /// The exceptions are `lora_path` and `session_params`: dropping them would + /// silently change the answer, so a non-null value is a 400 (as Python's is + /// on a server without LoRA or without that session), and `null` passes. #[test] fn unported_generate_req_input_fields_are_ignored() { for field in [ r#""session_id": "s""#, - r#""session_params": {"a": 1}"#, + r#""session_params": null"#, r#""return_sampling_mask": true"#, r#""custom_logit_processor": "cls""#, - r#""lora_path": "adapter""#, + r#""lora_path": null"#, r#""image_data": "base64""#, r#""return_routed_experts": true"#, r#""bootstrap_host": "h""#, @@ -847,6 +880,14 @@ mod tests { assert_eq!(ps.len(), 1, "{field}"); assert_eq!(ps[0].text.as_deref(), Some("hi"), "{field}"); } + for field in [ + r#""lora_path": "adapter""#, + r#""session_params": {"id": "s"}"#, + ] { + let body = format!(r#"{{"text": "hi", {field}}}"#); + let err = requests(&body).unwrap_err().to_string(); + assert!(err.contains("not supported"), "{field}: {err}"); + } } /// Client-supplied rid semantics mirror Python's `_normalize_batch`: a @@ -1210,11 +1251,12 @@ mod tests { assert!(err.to_string().contains("bootstrap_room"), "{err}"); } - /// `extra_key` fans out like Python `_normalize_extra_key`: one key + /// `extra_key` fans out like Python `_normalize_extra_key` (and + /// `cache_salt` like `_normalize_cache_salt`, the same rules): one key /// broadcasts, a list is per item and must match the batch (and needs a /// batch), and an empty key means none. #[test] - fn extra_key_fans_out() { + fn cache_keys_fan_out() { let (ps, _) = requests(r#"{"text": "a", "extra_key": "salt"}"#).unwrap(); assert_eq!(ps[0].extra_key.as_deref(), Some("salt")); let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": "salt"}"#).unwrap(); @@ -1228,6 +1270,16 @@ mod tests { let err = requests(r#"{"text": ["a", "b"], "extra_key": ["k"]}"#).unwrap_err(); assert!(err.to_string().contains("extra_key"), "{err}"); + + let (ps, _) = requests(r#"{"text": ["a", "b"], "cache_salt": ["s", ""]}"#).unwrap(); + assert_eq!(ps[0].cache_salt.as_deref(), Some("s")); + assert_eq!(ps[1].cache_salt, None); + assert!(requests(r#"{"text": "a", "cache_salt": ["s"]}"#).is_err()); + let (ps, _) = + requests(r#"{"text": ["a", "b"], "cache_salt": "s", "require_reasoning": true}"#) + .unwrap(); + assert!(ps.iter().all(|p| p.cache_salt.as_deref() == Some("s"))); + assert!(ps.iter().all(|p| p.require_reasoning)); } /// The PD router (mini_lb) and PD-warmup payload shapes must parse. The diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 98bbc82c10..30ae036d5b 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -43,7 +43,6 @@ FIXTURES = REPO_ROOT / "rust" / "sglang-api-types" / "testdata" / "json_parity.j PYTHON_ONLY_REQUEST_FIELDS = frozenset( { "background", - "cache_salt", "conversation_id", "custom_labels", "custom_logit_processor", @@ -57,7 +56,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "kv_hints", "log_metrics", "lora_id", - "lora_path", "max_dynamic_patch", "max_thinking_tokens", "min_dynamic_patch", @@ -70,7 +68,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "num_items_assigned", "positional_embed_overrides", "received_time", - "require_reasoning", "return_bytes", "return_entropy", "return_flat_raw_output_top_logprobs", @@ -84,7 +81,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "routing_key", "sampling_logprobs_mode", "session_id", - "session_params", "token_indices_to_pool", "use_audio_in_video", "video_config",