[Fix] rust-server: Honour cache_salt and require_reasoning on /generate (#44533)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 10:38:09 -07:00
committed by GitHub
co-authored by metamergebot shodoco
parent 453b251dbd
commit 553068af25
7 changed files with 219 additions and 29 deletions
+12
View File
@@ -60,6 +60,18 @@ message GenerateRequest {
// Scheduling priority, one value for every batch item; the scheduler
// validates it against its priority-scheduling mode.
optional int64 priority = 24;
// Prefix-cache salt: a second namespace beside `extra_key`, normalized the
// same way.
StringOrList cache_salt = 25;
// Apply grammar constraints only after the reasoning section, for every
// batch item.
optional bool require_reasoning = 26;
// Fields this server cannot honour. Unknown keys are ignored, but
// dropping these would silently change the result (base-model output for a
// LoRA request, a stateless answer for a session request), so a non-null
// value is rejected instead.
optional string lora_path = 27 [(sglang.json.v1.field_json).raw_json = true];
optional string session_params = 28 [(sglang.json.v1.field_json).raw_json = true];
}
message SamplingParamsOrList {
+20
View File
@@ -259,6 +259,10 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
audio_data: Optional[MediaInput] = None
extra_key: Optional[StringOrList] = None
priority: Optional[int] = None
cache_salt: Optional[StringOrList] = None
require_reasoning: Optional[bool] = None
lora_path: Optional[Any] = None
session_params: Optional[Any] = None
@classmethod
def from_json_value(cls, value: Any) -> "GenerateRequest":
@@ -1104,6 +1108,14 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest:
kw["extra_key"] = None if x is None else decode_StringOrList(x)
elif key == "priority":
kw["priority"] = None if x is None else _expect_int(x, expected="i64")
elif key == "cache_salt":
kw["cache_salt"] = None if x is None else decode_StringOrList(x)
elif key == "require_reasoning":
kw["require_reasoning"] = None if x is None else _expect_bool(x, expected="a boolean")
elif key == "lora_path":
kw["lora_path"] = x
elif key == "session_params":
kw["session_params"] = x
return GenerateRequest(**kw)
@@ -1157,6 +1169,14 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]:
d["extra_key"] = encode_StringOrList(v.extra_key)
if v.priority is not None:
d["priority"] = v.priority
if v.cache_salt is not None:
d["cache_salt"] = encode_StringOrList(v.cache_salt)
if v.require_reasoning is not None:
d["require_reasoning"] = v.require_reasoning
if v.lora_path is not None:
d["lora_path"] = v.lora_path
if v.session_params is not None:
d["session_params"] = v.session_params
return d
@@ -383,6 +383,22 @@ pub struct GenerateRequest {
/// validates it against its priority-scheduling mode.
#[prost(int64, optional, tag = "24")]
pub priority: ::core::option::Option<i64>,
/// Prefix-cache salt: a second namespace beside `extra_key`, normalized the
/// same way.
#[prost(message, optional, tag = "25")]
pub cache_salt: ::core::option::Option<StringOrList>,
/// Apply grammar constraints only after the reasoning section, for every
/// batch item.
#[prost(bool, optional, tag = "26")]
pub require_reasoning: ::core::option::Option<bool>,
/// Fields this server cannot honour. Unknown keys are ignored, but
/// dropping these would silently change the result (base-model output for a
/// LoRA request, a stateless answer for a session request), so a non-null
/// value is rejected instead.
#[prost(string, optional, tag = "27")]
pub lora_path: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "28")]
pub session_params: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SamplingParamsOrList {
@@ -1596,6 +1596,28 @@ impl ::serde::Serialize for GenerateRequest {
if let Some(v) = &self.priority {
::serde::ser::SerializeMap::serialize_entry(&mut map, "priority", v)?;
}
if let Some(v) = &self.cache_salt {
::serde::ser::SerializeMap::serialize_entry(&mut map, "cache_salt", v)?;
}
if let Some(v) = &self.require_reasoning {
::serde::ser::SerializeMap::serialize_entry(&mut map, "require_reasoning", v)?;
}
if let Some(v) = &self.lora_path {
::serde::ser::SerializeMap::serialize_entry(
&mut map,
"lora_path",
&::serde_json::from_str::<::serde_json::Value>(v)
.unwrap_or(::serde_json::Value::Null),
)?;
}
if let Some(v) = &self.session_params {
::serde::ser::SerializeMap::serialize_entry(
&mut map,
"session_params",
&::serde_json::from_str::<::serde_json::Value>(v)
.unwrap_or(::serde_json::Value::Null),
)?;
}
::serde::ser::SerializeMap::end(map)
}
}
@@ -1636,6 +1658,10 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
let mut f_audio_data = None;
let mut f_extra_key = None;
let mut f_priority = None;
let mut f_cache_salt = None;
let mut f_require_reasoning = None;
let mut f_lora_path = None;
let mut f_session_params = None;
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
match key.as_ref() {
"rid" => {
@@ -1804,6 +1830,38 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
}
f_priority = Some(map.next_value::<Option<i64>>()?);
}
"cache_salt" => {
if f_cache_salt.is_some() {
return Err(::serde::de::Error::duplicate_field("cache_salt"));
}
f_cache_salt = Some(map.next_value::<Option<StringOrList>>()?);
}
"require_reasoning" => {
if f_require_reasoning.is_some() {
return Err(::serde::de::Error::duplicate_field(
"require_reasoning",
));
}
f_require_reasoning = Some(map.next_value::<Option<bool>>()?);
}
"lora_path" => {
if f_lora_path.is_some() {
return Err(::serde::de::Error::duplicate_field("lora_path"));
}
f_lora_path = Some(
map.next_value::<Option<::serde_json::Value>>()?
.map(|v| v.to_string()),
);
}
"session_params" => {
if f_session_params.is_some() {
return Err(::serde::de::Error::duplicate_field("session_params"));
}
f_session_params = Some(
map.next_value::<Option<::serde_json::Value>>()?
.map(|v| v.to_string()),
);
}
_ => {
map.next_value::<::serde::de::IgnoredAny>()?;
}
@@ -1834,6 +1892,10 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
audio_data: f_audio_data.flatten(),
extra_key: f_extra_key.flatten(),
priority: f_priority.flatten(),
cache_salt: f_cache_salt.flatten(),
require_reasoning: f_require_reasoning.flatten(),
lora_path: f_lora_path.flatten(),
session_params: f_session_params.flatten(),
})
}
}
+42 -10
View File
@@ -43,7 +43,7 @@ wire_struct! {
lora_id: (),
custom_logit_processor: (),
positional_embed_overrides: (),
/// PD-disaggregation block. Everything after `extra_key` (the last
/// PD-disaggregation block. Everything after `cache_salt` (the last
/// field emitted) has a msgspec default in Python and is omitted
/// (short arrays decode with defaulted tails).
bootstrap_host: Option<&'a str>,
@@ -55,12 +55,24 @@ wire_struct! {
disagg_prefill_dp_rank: Option<i64>,
/// The scheduler decodes this header by position (Python's
/// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so
/// `routing_key` and `require_reasoning` are sent at their defaults for
/// `priority` and `extra_key` to land in their slots.
/// `routing_key` is sent at its default for the fields after it to land
/// in their slots.
routing_key: (),
require_reasoning: bool,
priority: Option<i64>,
extra_key: Option<&'a str>,
/// Defaults, so `cache_salt`, the prefix cache's second key, lands in
/// its slot.
no_logs: bool,
return_bytes: bool,
return_entropy: bool,
need_wait_for_mm_inputs: (),
num_items_assigned: (),
encoder_urls: (),
multi_item_delimiter_indices: (),
token_indices_to_pool: (),
time_stats: (),
cache_salt: Option<&'a str>,
}
}
@@ -132,9 +144,19 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> {
routed_dp_rank: req.routed_dp_rank,
disagg_prefill_dp_rank: req.disagg_prefill_dp_rank,
routing_key: (),
require_reasoning: false,
require_reasoning: req.require_reasoning,
priority: req.priority,
extra_key: req.extra_key.as_deref(),
no_logs: false,
return_bytes: false,
return_entropy: false,
need_wait_for_mm_inputs: (),
num_items_assigned: (),
encoder_urls: (),
multi_item_delimiter_indices: (),
token_indices_to_pool: (),
time_stats: (),
cache_salt: req.cache_salt.as_deref(),
}
}
}
@@ -247,9 +269,9 @@ mod tests {
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap();
let arr = val.as_array().expect("array");
// msgspec requires >= 14 (through `stream`); we emit 36 (through
// `extra_key`). Trailing defaulted fields are omitted.
assert_eq!(arr.len(), 36, "header ends at extra_key");
// msgspec requires >= 14 (through `stream`); we emit 46 (through
// `cache_salt`). Trailing defaulted fields are omitted.
assert_eq!(arr.len(), 46, "header ends at cache_salt");
assert_eq!(arr[0].as_str(), Some("TokenizedGenerateReqInput"));
assert_eq!(arr[1].as_str(), Some("r1"));
assert!(arr[5].is_nil(), "idx 5 must be input_embeds (nil)");
@@ -281,9 +303,10 @@ mod tests {
/// The PD block must land on Python's wire indices 25–31, with the filler
/// block (17–24) holding its defaults — a shift here silently routes KV
/// transfers to the wrong host/room.
/// transfers to the wrong host/room. The cache keys and `require_reasoning`
/// follow at 33–45.
#[test]
fn header_bootstrap_block_and_extra_key_are_positionally_aligned() {
fn header_bootstrap_block_and_cache_keys_are_positionally_aligned() {
let req = GenerateRequest {
rid: "r1".into(),
text: Some("hi".into()),
@@ -295,7 +318,9 @@ mod tests {
routed_dp_rank: Some(3),
disagg_prefill_dp_rank: Some(4),
extra_key: Some("salt".into()),
cache_salt: Some("pepper".into()),
priority: Some(-2),
require_reasoning: true,
..Default::default()
};
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
@@ -315,8 +340,15 @@ mod tests {
assert_eq!(arr[30].as_i64(), Some(3), "routed_dp_rank at 30");
assert_eq!(arr[31].as_i64(), Some(4), "disagg_prefill_dp_rank at 31");
assert!(arr[32].is_nil(), "routing_key at 32");
assert_eq!(arr[33].as_bool(), Some(false), "require_reasoning at 33");
assert_eq!(arr[33].as_bool(), Some(true), "require_reasoning at 33");
assert_eq!(arr[34].as_i64(), Some(-2), "priority at 34");
assert_eq!(arr[35].as_str(), Some("salt"), "extra_key at 35");
for (i, slot) in arr.iter().enumerate().take(39).skip(36) {
assert_eq!(slot.as_bool(), Some(false), "idx {i} must be false");
}
for (i, slot) in arr.iter().enumerate().take(45).skip(39) {
assert!(slot.is_nil(), "idx {i} must be a nil default");
}
assert_eq!(arr[45].as_str(), Some("pepper"), "cache_salt at 45");
}
}
+67 -15
View File
@@ -81,7 +81,21 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
audio_data,
extra_key,
priority,
cache_salt,
require_reasoning,
lora_path,
session_params,
} = req;
for (name, value) in [
("lora_path", &lora_path),
("session_params", &session_params),
] {
if value.is_some() {
return Err(Error::Validation(format!(
"{name} is not supported by the Rust server"
)));
}
}
let rid = rid.and_then(wire::string_or_list);
let text = text.and_then(wire::string_or_list);
let input_ids = input_ids.and_then(wire::token_ids_or_list);
@@ -106,6 +120,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
let audio_data = audio_data.map(wire::media_input).transpose()?;
let mm_hashes = mm_hashes.and_then(wire::string_list_or_list);
let extra_key = extra_key.and_then(wire::string_or_list);
let cache_salt = cache_salt.and_then(wire::string_or_list);
// Cap the batch BEFORE the columns below allocate anything. Reading the
// declared length off the input costs nothing; the previous placement (after
@@ -273,17 +288,24 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
} = normalize_bootstrap_columns(bootstrap_host, bootstrap_port, bootstrap_room, n)?;
let bootstrap_pair_keys = flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?);
let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?);
// Python `_normalize_extra_key`: one key broadcasts, a list is per item
// (batch requests only), and an empty key means none.
if !is_batch && matches!(extra_key, Some(OneOrMany::Many(_))) {
return Err(Error::Validation(
"extra_key should be a string for a single request.".into(),
));
}
let extra_keys: Vec<Option<String>> = fan_out(extra_key, n, "extra_key")?
.into_iter()
.map(|key| key.filter(|key| !key.is_empty()))
.collect();
// Python `_normalize_extra_key` / `_normalize_cache_salt`: one key
// broadcasts, a list is per item (batch requests only), and an empty key
// means none.
let cache_key_column = |value: Option<OneOrMany<String>>, name: &str| {
if !is_batch && matches!(value, Some(OneOrMany::Many(_))) {
return Err(Error::Validation(format!(
"{name} should be a string for a single request."
)));
}
Ok::<Vec<Option<String>>, Error>(
fan_out(value, n, name)?
.into_iter()
.map(|key| key.filter(|key| !key.is_empty()))
.collect(),
)
};
let extra_keys = cache_key_column(extra_key, "extra_key")?;
let cache_salts = cache_key_column(cache_salt, "cache_salt")?;
// `mm_hashes` has no batch form: honoring it only here would give the two
// servers different prefix-cache keys for the same body. Reject instead of
// dropping it silently as Python does — the field exists to align a
@@ -326,6 +348,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
bootstrap_pair_keys,
decode_tp_sizes,
extra_keys,
cache_salts,
images,
videos,
audios,
@@ -347,6 +370,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
bootstrap_pair_key,
decode_tp_size,
extra_key,
cache_salt,
image_data,
video_data,
audio_data,
@@ -377,7 +401,9 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
routed_dp_rank,
disagg_prefill_dp_rank,
extra_key,
cache_salt,
priority,
require_reasoning: require_reasoning.unwrap_or(false),
mm: pack_mm(image_data, video_data, audio_data),
mm_buffers: Vec::new(),
},
@@ -523,8 +549,12 @@ pub struct GenerateRequest {
/// Prefix-cache namespace: requests share cached prefixes only under the
/// same key.
pub extra_key: Option<String>,
/// The prefix cache's second namespace key.
pub cache_salt: Option<String>,
/// Scheduling priority, shared by every item of a batch.
pub priority: Option<i64>,
/// Grammar constraints start after the reasoning section.
pub require_reasoning: bool,
/// Multimodal inputs. Consumed by the Encoding stage, which ships them to
/// the MM worker pool; never read by the tokenizer or serialized onto the
/// scheduler header. Boxed so the common text-only request doesn't grow
@@ -826,14 +856,17 @@ mod tests {
/// to be declared and dropped by hand just to let `bench_serving` through.
/// FastAPI's pydantic dataclass drops extras, so ignoring them is the parity
/// behavior; a typo being silently ignored is the same trade Python makes.
/// The exceptions are `lora_path` and `session_params`: dropping them would
/// silently change the answer, so a non-null value is a 400 (as Python's is
/// on a server without LoRA or without that session), and `null` passes.
#[test]
fn unported_generate_req_input_fields_are_ignored() {
for field in [
r#""session_id": "s""#,
r#""session_params": {"a": 1}"#,
r#""session_params": null"#,
r#""return_sampling_mask": true"#,
r#""custom_logit_processor": "cls""#,
r#""lora_path": "adapter""#,
r#""lora_path": null"#,
r#""image_data": "base64""#,
r#""return_routed_experts": true"#,
r#""bootstrap_host": "h""#,
@@ -847,6 +880,14 @@ mod tests {
assert_eq!(ps.len(), 1, "{field}");
assert_eq!(ps[0].text.as_deref(), Some("hi"), "{field}");
}
for field in [
r#""lora_path": "adapter""#,
r#""session_params": {"id": "s"}"#,
] {
let body = format!(r#"{{"text": "hi", {field}}}"#);
let err = requests(&body).unwrap_err().to_string();
assert!(err.contains("not supported"), "{field}: {err}");
}
}
/// Client-supplied rid semantics mirror Python's `_normalize_batch`: a
@@ -1210,11 +1251,12 @@ mod tests {
assert!(err.to_string().contains("bootstrap_room"), "{err}");
}
/// `extra_key` fans out like Python `_normalize_extra_key`: one key
/// `extra_key` fans out like Python `_normalize_extra_key` (and
/// `cache_salt` like `_normalize_cache_salt`, the same rules): one key
/// broadcasts, a list is per item and must match the batch (and needs a
/// batch), and an empty key means none.
#[test]
fn extra_key_fans_out() {
fn cache_keys_fan_out() {
let (ps, _) = requests(r#"{"text": "a", "extra_key": "salt"}"#).unwrap();
assert_eq!(ps[0].extra_key.as_deref(), Some("salt"));
let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": "salt"}"#).unwrap();
@@ -1228,6 +1270,16 @@ mod tests {
let err = requests(r#"{"text": ["a", "b"], "extra_key": ["k"]}"#).unwrap_err();
assert!(err.to_string().contains("extra_key"), "{err}");
let (ps, _) = requests(r#"{"text": ["a", "b"], "cache_salt": ["s", ""]}"#).unwrap();
assert_eq!(ps[0].cache_salt.as_deref(), Some("s"));
assert_eq!(ps[1].cache_salt, None);
assert!(requests(r#"{"text": "a", "cache_salt": ["s"]}"#).is_err());
let (ps, _) =
requests(r#"{"text": ["a", "b"], "cache_salt": "s", "require_reasoning": true}"#)
.unwrap();
assert!(ps.iter().all(|p| p.cache_salt.as_deref() == Some("s")));
assert!(ps.iter().all(|p| p.require_reasoning));
}
/// The PD router (mini_lb) and PD-warmup payload shapes must parse. The
@@ -43,7 +43,6 @@ FIXTURES = REPO_ROOT / "rust" / "sglang-api-types" / "testdata" / "json_parity.j
PYTHON_ONLY_REQUEST_FIELDS = frozenset(
{
"background",
"cache_salt",
"conversation_id",
"custom_labels",
"custom_logit_processor",
@@ -57,7 +56,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
"kv_hints",
"log_metrics",
"lora_id",
"lora_path",
"max_dynamic_patch",
"max_thinking_tokens",
"min_dynamic_patch",
@@ -70,7 +68,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
"num_items_assigned",
"positional_embed_overrides",
"received_time",
"require_reasoning",
"return_bytes",
"return_entropy",
"return_flat_raw_output_top_logprobs",
@@ -84,7 +81,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
"routing_key",
"sampling_logprobs_mode",
"session_id",
"session_params",
"token_indices_to_pool",
"use_audio_in_video",
"video_config",