[Fix] rust-server: Honour cache_salt and require_reasoning on /generate (#44533)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
shodoco
parent
453b251dbd
commit
553068af25
@@ -60,6 +60,18 @@ message GenerateRequest {
|
||||
// Scheduling priority, one value for every batch item; the scheduler
|
||||
// validates it against its priority-scheduling mode.
|
||||
optional int64 priority = 24;
|
||||
// Prefix-cache salt: a second namespace beside `extra_key`, normalized the
|
||||
// same way.
|
||||
StringOrList cache_salt = 25;
|
||||
// Apply grammar constraints only after the reasoning section, for every
|
||||
// batch item.
|
||||
optional bool require_reasoning = 26;
|
||||
// Fields this server cannot honour. Unknown keys are ignored, but
|
||||
// dropping these would silently change the result (base-model output for a
|
||||
// LoRA request, a stateless answer for a session request), so a non-null
|
||||
// value is rejected instead.
|
||||
optional string lora_path = 27 [(sglang.json.v1.field_json).raw_json = true];
|
||||
optional string session_params = 28 [(sglang.json.v1.field_json).raw_json = true];
|
||||
}
|
||||
|
||||
message SamplingParamsOrList {
|
||||
|
||||
@@ -259,6 +259,10 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
|
||||
audio_data: Optional[MediaInput] = None
|
||||
extra_key: Optional[StringOrList] = None
|
||||
priority: Optional[int] = None
|
||||
cache_salt: Optional[StringOrList] = None
|
||||
require_reasoning: Optional[bool] = None
|
||||
lora_path: Optional[Any] = None
|
||||
session_params: Optional[Any] = None
|
||||
|
||||
@classmethod
|
||||
def from_json_value(cls, value: Any) -> "GenerateRequest":
|
||||
@@ -1104,6 +1108,14 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest:
|
||||
kw["extra_key"] = None if x is None else decode_StringOrList(x)
|
||||
elif key == "priority":
|
||||
kw["priority"] = None if x is None else _expect_int(x, expected="i64")
|
||||
elif key == "cache_salt":
|
||||
kw["cache_salt"] = None if x is None else decode_StringOrList(x)
|
||||
elif key == "require_reasoning":
|
||||
kw["require_reasoning"] = None if x is None else _expect_bool(x, expected="a boolean")
|
||||
elif key == "lora_path":
|
||||
kw["lora_path"] = x
|
||||
elif key == "session_params":
|
||||
kw["session_params"] = x
|
||||
return GenerateRequest(**kw)
|
||||
|
||||
|
||||
@@ -1157,6 +1169,14 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]:
|
||||
d["extra_key"] = encode_StringOrList(v.extra_key)
|
||||
if v.priority is not None:
|
||||
d["priority"] = v.priority
|
||||
if v.cache_salt is not None:
|
||||
d["cache_salt"] = encode_StringOrList(v.cache_salt)
|
||||
if v.require_reasoning is not None:
|
||||
d["require_reasoning"] = v.require_reasoning
|
||||
if v.lora_path is not None:
|
||||
d["lora_path"] = v.lora_path
|
||||
if v.session_params is not None:
|
||||
d["session_params"] = v.session_params
|
||||
return d
|
||||
|
||||
|
||||
|
||||
@@ -383,6 +383,22 @@ pub struct GenerateRequest {
|
||||
/// validates it against its priority-scheduling mode.
|
||||
#[prost(int64, optional, tag = "24")]
|
||||
pub priority: ::core::option::Option<i64>,
|
||||
/// Prefix-cache salt: a second namespace beside `extra_key`, normalized the
|
||||
/// same way.
|
||||
#[prost(message, optional, tag = "25")]
|
||||
pub cache_salt: ::core::option::Option<StringOrList>,
|
||||
/// Apply grammar constraints only after the reasoning section, for every
|
||||
/// batch item.
|
||||
#[prost(bool, optional, tag = "26")]
|
||||
pub require_reasoning: ::core::option::Option<bool>,
|
||||
/// Fields this server cannot honour. Unknown keys are ignored, but
|
||||
/// dropping these would silently change the result (base-model output for a
|
||||
/// LoRA request, a stateless answer for a session request), so a non-null
|
||||
/// value is rejected instead.
|
||||
#[prost(string, optional, tag = "27")]
|
||||
pub lora_path: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "28")]
|
||||
pub session_params: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct SamplingParamsOrList {
|
||||
|
||||
@@ -1596,6 +1596,28 @@ impl ::serde::Serialize for GenerateRequest {
|
||||
if let Some(v) = &self.priority {
|
||||
::serde::ser::SerializeMap::serialize_entry(&mut map, "priority", v)?;
|
||||
}
|
||||
if let Some(v) = &self.cache_salt {
|
||||
::serde::ser::SerializeMap::serialize_entry(&mut map, "cache_salt", v)?;
|
||||
}
|
||||
if let Some(v) = &self.require_reasoning {
|
||||
::serde::ser::SerializeMap::serialize_entry(&mut map, "require_reasoning", v)?;
|
||||
}
|
||||
if let Some(v) = &self.lora_path {
|
||||
::serde::ser::SerializeMap::serialize_entry(
|
||||
&mut map,
|
||||
"lora_path",
|
||||
&::serde_json::from_str::<::serde_json::Value>(v)
|
||||
.unwrap_or(::serde_json::Value::Null),
|
||||
)?;
|
||||
}
|
||||
if let Some(v) = &self.session_params {
|
||||
::serde::ser::SerializeMap::serialize_entry(
|
||||
&mut map,
|
||||
"session_params",
|
||||
&::serde_json::from_str::<::serde_json::Value>(v)
|
||||
.unwrap_or(::serde_json::Value::Null),
|
||||
)?;
|
||||
}
|
||||
::serde::ser::SerializeMap::end(map)
|
||||
}
|
||||
}
|
||||
@@ -1636,6 +1658,10 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
||||
let mut f_audio_data = None;
|
||||
let mut f_extra_key = None;
|
||||
let mut f_priority = None;
|
||||
let mut f_cache_salt = None;
|
||||
let mut f_require_reasoning = None;
|
||||
let mut f_lora_path = None;
|
||||
let mut f_session_params = None;
|
||||
while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {
|
||||
match key.as_ref() {
|
||||
"rid" => {
|
||||
@@ -1804,6 +1830,38 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
||||
}
|
||||
f_priority = Some(map.next_value::<Option<i64>>()?);
|
||||
}
|
||||
"cache_salt" => {
|
||||
if f_cache_salt.is_some() {
|
||||
return Err(::serde::de::Error::duplicate_field("cache_salt"));
|
||||
}
|
||||
f_cache_salt = Some(map.next_value::<Option<StringOrList>>()?);
|
||||
}
|
||||
"require_reasoning" => {
|
||||
if f_require_reasoning.is_some() {
|
||||
return Err(::serde::de::Error::duplicate_field(
|
||||
"require_reasoning",
|
||||
));
|
||||
}
|
||||
f_require_reasoning = Some(map.next_value::<Option<bool>>()?);
|
||||
}
|
||||
"lora_path" => {
|
||||
if f_lora_path.is_some() {
|
||||
return Err(::serde::de::Error::duplicate_field("lora_path"));
|
||||
}
|
||||
f_lora_path = Some(
|
||||
map.next_value::<Option<::serde_json::Value>>()?
|
||||
.map(|v| v.to_string()),
|
||||
);
|
||||
}
|
||||
"session_params" => {
|
||||
if f_session_params.is_some() {
|
||||
return Err(::serde::de::Error::duplicate_field("session_params"));
|
||||
}
|
||||
f_session_params = Some(
|
||||
map.next_value::<Option<::serde_json::Value>>()?
|
||||
.map(|v| v.to_string()),
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
map.next_value::<::serde::de::IgnoredAny>()?;
|
||||
}
|
||||
@@ -1834,6 +1892,10 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest {
|
||||
audio_data: f_audio_data.flatten(),
|
||||
extra_key: f_extra_key.flatten(),
|
||||
priority: f_priority.flatten(),
|
||||
cache_salt: f_cache_salt.flatten(),
|
||||
require_reasoning: f_require_reasoning.flatten(),
|
||||
lora_path: f_lora_path.flatten(),
|
||||
session_params: f_session_params.flatten(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ wire_struct! {
|
||||
lora_id: (),
|
||||
custom_logit_processor: (),
|
||||
positional_embed_overrides: (),
|
||||
/// PD-disaggregation block. Everything after `extra_key` (the last
|
||||
/// PD-disaggregation block. Everything after `cache_salt` (the last
|
||||
/// field emitted) has a msgspec default in Python and is omitted
|
||||
/// (short arrays decode with defaulted tails).
|
||||
bootstrap_host: Option<&'a str>,
|
||||
@@ -55,12 +55,24 @@ wire_struct! {
|
||||
disagg_prefill_dp_rank: Option<i64>,
|
||||
/// The scheduler decodes this header by position (Python's
|
||||
/// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so
|
||||
/// `routing_key` and `require_reasoning` are sent at their defaults for
|
||||
/// `priority` and `extra_key` to land in their slots.
|
||||
/// `routing_key` is sent at its default for the fields after it to land
|
||||
/// in their slots.
|
||||
routing_key: (),
|
||||
require_reasoning: bool,
|
||||
priority: Option<i64>,
|
||||
extra_key: Option<&'a str>,
|
||||
/// Defaults, so `cache_salt`, the prefix cache's second key, lands in
|
||||
/// its slot.
|
||||
no_logs: bool,
|
||||
return_bytes: bool,
|
||||
return_entropy: bool,
|
||||
need_wait_for_mm_inputs: (),
|
||||
num_items_assigned: (),
|
||||
encoder_urls: (),
|
||||
multi_item_delimiter_indices: (),
|
||||
token_indices_to_pool: (),
|
||||
time_stats: (),
|
||||
cache_salt: Option<&'a str>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,9 +144,19 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> {
|
||||
routed_dp_rank: req.routed_dp_rank,
|
||||
disagg_prefill_dp_rank: req.disagg_prefill_dp_rank,
|
||||
routing_key: (),
|
||||
require_reasoning: false,
|
||||
require_reasoning: req.require_reasoning,
|
||||
priority: req.priority,
|
||||
extra_key: req.extra_key.as_deref(),
|
||||
no_logs: false,
|
||||
return_bytes: false,
|
||||
return_entropy: false,
|
||||
need_wait_for_mm_inputs: (),
|
||||
num_items_assigned: (),
|
||||
encoder_urls: (),
|
||||
multi_item_delimiter_indices: (),
|
||||
token_indices_to_pool: (),
|
||||
time_stats: (),
|
||||
cache_salt: req.cache_salt.as_deref(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -247,9 +269,9 @@ mod tests {
|
||||
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
|
||||
let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap();
|
||||
let arr = val.as_array().expect("array");
|
||||
// msgspec requires >= 14 (through `stream`); we emit 36 (through
|
||||
// `extra_key`). Trailing defaulted fields are omitted.
|
||||
assert_eq!(arr.len(), 36, "header ends at extra_key");
|
||||
// msgspec requires >= 14 (through `stream`); we emit 46 (through
|
||||
// `cache_salt`). Trailing defaulted fields are omitted.
|
||||
assert_eq!(arr.len(), 46, "header ends at cache_salt");
|
||||
assert_eq!(arr[0].as_str(), Some("TokenizedGenerateReqInput"));
|
||||
assert_eq!(arr[1].as_str(), Some("r1"));
|
||||
assert!(arr[5].is_nil(), "idx 5 must be input_embeds (nil)");
|
||||
@@ -281,9 +303,10 @@ mod tests {
|
||||
|
||||
/// The PD block must land on Python's wire indices 25–31, with the filler
|
||||
/// block (17–24) holding its defaults — a shift here silently routes KV
|
||||
/// transfers to the wrong host/room.
|
||||
/// transfers to the wrong host/room. The cache keys and `require_reasoning`
|
||||
/// follow at 33–45.
|
||||
#[test]
|
||||
fn header_bootstrap_block_and_extra_key_are_positionally_aligned() {
|
||||
fn header_bootstrap_block_and_cache_keys_are_positionally_aligned() {
|
||||
let req = GenerateRequest {
|
||||
rid: "r1".into(),
|
||||
text: Some("hi".into()),
|
||||
@@ -295,7 +318,9 @@ mod tests {
|
||||
routed_dp_rank: Some(3),
|
||||
disagg_prefill_dp_rank: Some(4),
|
||||
extra_key: Some("salt".into()),
|
||||
cache_salt: Some("pepper".into()),
|
||||
priority: Some(-2),
|
||||
require_reasoning: true,
|
||||
..Default::default()
|
||||
};
|
||||
let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap();
|
||||
@@ -315,8 +340,15 @@ mod tests {
|
||||
assert_eq!(arr[30].as_i64(), Some(3), "routed_dp_rank at 30");
|
||||
assert_eq!(arr[31].as_i64(), Some(4), "disagg_prefill_dp_rank at 31");
|
||||
assert!(arr[32].is_nil(), "routing_key at 32");
|
||||
assert_eq!(arr[33].as_bool(), Some(false), "require_reasoning at 33");
|
||||
assert_eq!(arr[33].as_bool(), Some(true), "require_reasoning at 33");
|
||||
assert_eq!(arr[34].as_i64(), Some(-2), "priority at 34");
|
||||
assert_eq!(arr[35].as_str(), Some("salt"), "extra_key at 35");
|
||||
for (i, slot) in arr.iter().enumerate().take(39).skip(36) {
|
||||
assert_eq!(slot.as_bool(), Some(false), "idx {i} must be false");
|
||||
}
|
||||
for (i, slot) in arr.iter().enumerate().take(45).skip(39) {
|
||||
assert!(slot.is_nil(), "idx {i} must be a nil default");
|
||||
}
|
||||
assert_eq!(arr[45].as_str(), Some("pepper"), "cache_salt at 45");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +81,21 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
audio_data,
|
||||
extra_key,
|
||||
priority,
|
||||
cache_salt,
|
||||
require_reasoning,
|
||||
lora_path,
|
||||
session_params,
|
||||
} = req;
|
||||
for (name, value) in [
|
||||
("lora_path", &lora_path),
|
||||
("session_params", &session_params),
|
||||
] {
|
||||
if value.is_some() {
|
||||
return Err(Error::Validation(format!(
|
||||
"{name} is not supported by the Rust server"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let rid = rid.and_then(wire::string_or_list);
|
||||
let text = text.and_then(wire::string_or_list);
|
||||
let input_ids = input_ids.and_then(wire::token_ids_or_list);
|
||||
@@ -106,6 +120,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
let audio_data = audio_data.map(wire::media_input).transpose()?;
|
||||
let mm_hashes = mm_hashes.and_then(wire::string_list_or_list);
|
||||
let extra_key = extra_key.and_then(wire::string_or_list);
|
||||
let cache_salt = cache_salt.and_then(wire::string_or_list);
|
||||
|
||||
// Cap the batch BEFORE the columns below allocate anything. Reading the
|
||||
// declared length off the input costs nothing; the previous placement (after
|
||||
@@ -273,17 +288,24 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
} = normalize_bootstrap_columns(bootstrap_host, bootstrap_port, bootstrap_room, n)?;
|
||||
let bootstrap_pair_keys = flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?);
|
||||
let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?);
|
||||
// Python `_normalize_extra_key`: one key broadcasts, a list is per item
|
||||
// (batch requests only), and an empty key means none.
|
||||
if !is_batch && matches!(extra_key, Some(OneOrMany::Many(_))) {
|
||||
return Err(Error::Validation(
|
||||
"extra_key should be a string for a single request.".into(),
|
||||
));
|
||||
}
|
||||
let extra_keys: Vec<Option<String>> = fan_out(extra_key, n, "extra_key")?
|
||||
.into_iter()
|
||||
.map(|key| key.filter(|key| !key.is_empty()))
|
||||
.collect();
|
||||
// Python `_normalize_extra_key` / `_normalize_cache_salt`: one key
|
||||
// broadcasts, a list is per item (batch requests only), and an empty key
|
||||
// means none.
|
||||
let cache_key_column = |value: Option<OneOrMany<String>>, name: &str| {
|
||||
if !is_batch && matches!(value, Some(OneOrMany::Many(_))) {
|
||||
return Err(Error::Validation(format!(
|
||||
"{name} should be a string for a single request."
|
||||
)));
|
||||
}
|
||||
Ok::<Vec<Option<String>>, Error>(
|
||||
fan_out(value, n, name)?
|
||||
.into_iter()
|
||||
.map(|key| key.filter(|key| !key.is_empty()))
|
||||
.collect(),
|
||||
)
|
||||
};
|
||||
let extra_keys = cache_key_column(extra_key, "extra_key")?;
|
||||
let cache_salts = cache_key_column(cache_salt, "cache_salt")?;
|
||||
// `mm_hashes` has no batch form: honoring it only here would give the two
|
||||
// servers different prefix-cache keys for the same body. Reject instead of
|
||||
// dropping it silently as Python does — the field exists to align a
|
||||
@@ -326,6 +348,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
bootstrap_pair_keys,
|
||||
decode_tp_sizes,
|
||||
extra_keys,
|
||||
cache_salts,
|
||||
images,
|
||||
videos,
|
||||
audios,
|
||||
@@ -347,6 +370,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
bootstrap_pair_key,
|
||||
decode_tp_size,
|
||||
extra_key,
|
||||
cache_salt,
|
||||
image_data,
|
||||
video_data,
|
||||
audio_data,
|
||||
@@ -377,7 +401,9 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
|
||||
routed_dp_rank,
|
||||
disagg_prefill_dp_rank,
|
||||
extra_key,
|
||||
cache_salt,
|
||||
priority,
|
||||
require_reasoning: require_reasoning.unwrap_or(false),
|
||||
mm: pack_mm(image_data, video_data, audio_data),
|
||||
mm_buffers: Vec::new(),
|
||||
},
|
||||
@@ -523,8 +549,12 @@ pub struct GenerateRequest {
|
||||
/// Prefix-cache namespace: requests share cached prefixes only under the
|
||||
/// same key.
|
||||
pub extra_key: Option<String>,
|
||||
/// The prefix cache's second namespace key.
|
||||
pub cache_salt: Option<String>,
|
||||
/// Scheduling priority, shared by every item of a batch.
|
||||
pub priority: Option<i64>,
|
||||
/// Grammar constraints start after the reasoning section.
|
||||
pub require_reasoning: bool,
|
||||
/// Multimodal inputs. Consumed by the Encoding stage, which ships them to
|
||||
/// the MM worker pool; never read by the tokenizer or serialized onto the
|
||||
/// scheduler header. Boxed so the common text-only request doesn't grow
|
||||
@@ -826,14 +856,17 @@ mod tests {
|
||||
/// to be declared and dropped by hand just to let `bench_serving` through.
|
||||
/// FastAPI's pydantic dataclass drops extras, so ignoring them is the parity
|
||||
/// behavior; a typo being silently ignored is the same trade Python makes.
|
||||
/// The exceptions are `lora_path` and `session_params`: dropping them would
|
||||
/// silently change the answer, so a non-null value is a 400 (as Python's is
|
||||
/// on a server without LoRA or without that session), and `null` passes.
|
||||
#[test]
|
||||
fn unported_generate_req_input_fields_are_ignored() {
|
||||
for field in [
|
||||
r#""session_id": "s""#,
|
||||
r#""session_params": {"a": 1}"#,
|
||||
r#""session_params": null"#,
|
||||
r#""return_sampling_mask": true"#,
|
||||
r#""custom_logit_processor": "cls""#,
|
||||
r#""lora_path": "adapter""#,
|
||||
r#""lora_path": null"#,
|
||||
r#""image_data": "base64""#,
|
||||
r#""return_routed_experts": true"#,
|
||||
r#""bootstrap_host": "h""#,
|
||||
@@ -847,6 +880,14 @@ mod tests {
|
||||
assert_eq!(ps.len(), 1, "{field}");
|
||||
assert_eq!(ps[0].text.as_deref(), Some("hi"), "{field}");
|
||||
}
|
||||
for field in [
|
||||
r#""lora_path": "adapter""#,
|
||||
r#""session_params": {"id": "s"}"#,
|
||||
] {
|
||||
let body = format!(r#"{{"text": "hi", {field}}}"#);
|
||||
let err = requests(&body).unwrap_err().to_string();
|
||||
assert!(err.contains("not supported"), "{field}: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Client-supplied rid semantics mirror Python's `_normalize_batch`: a
|
||||
@@ -1210,11 +1251,12 @@ mod tests {
|
||||
assert!(err.to_string().contains("bootstrap_room"), "{err}");
|
||||
}
|
||||
|
||||
/// `extra_key` fans out like Python `_normalize_extra_key`: one key
|
||||
/// `extra_key` fans out like Python `_normalize_extra_key` (and
|
||||
/// `cache_salt` like `_normalize_cache_salt`, the same rules): one key
|
||||
/// broadcasts, a list is per item and must match the batch (and needs a
|
||||
/// batch), and an empty key means none.
|
||||
#[test]
|
||||
fn extra_key_fans_out() {
|
||||
fn cache_keys_fan_out() {
|
||||
let (ps, _) = requests(r#"{"text": "a", "extra_key": "salt"}"#).unwrap();
|
||||
assert_eq!(ps[0].extra_key.as_deref(), Some("salt"));
|
||||
let (ps, _) = requests(r#"{"text": ["a", "b"], "extra_key": "salt"}"#).unwrap();
|
||||
@@ -1228,6 +1270,16 @@ mod tests {
|
||||
|
||||
let err = requests(r#"{"text": ["a", "b"], "extra_key": ["k"]}"#).unwrap_err();
|
||||
assert!(err.to_string().contains("extra_key"), "{err}");
|
||||
|
||||
let (ps, _) = requests(r#"{"text": ["a", "b"], "cache_salt": ["s", ""]}"#).unwrap();
|
||||
assert_eq!(ps[0].cache_salt.as_deref(), Some("s"));
|
||||
assert_eq!(ps[1].cache_salt, None);
|
||||
assert!(requests(r#"{"text": "a", "cache_salt": ["s"]}"#).is_err());
|
||||
let (ps, _) =
|
||||
requests(r#"{"text": ["a", "b"], "cache_salt": "s", "require_reasoning": true}"#)
|
||||
.unwrap();
|
||||
assert!(ps.iter().all(|p| p.cache_salt.as_deref() == Some("s")));
|
||||
assert!(ps.iter().all(|p| p.require_reasoning));
|
||||
}
|
||||
|
||||
/// The PD router (mini_lb) and PD-warmup payload shapes must parse. The
|
||||
|
||||
@@ -43,7 +43,6 @@ FIXTURES = REPO_ROOT / "rust" / "sglang-api-types" / "testdata" / "json_parity.j
|
||||
PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
||||
{
|
||||
"background",
|
||||
"cache_salt",
|
||||
"conversation_id",
|
||||
"custom_labels",
|
||||
"custom_logit_processor",
|
||||
@@ -57,7 +56,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
||||
"kv_hints",
|
||||
"log_metrics",
|
||||
"lora_id",
|
||||
"lora_path",
|
||||
"max_dynamic_patch",
|
||||
"max_thinking_tokens",
|
||||
"min_dynamic_patch",
|
||||
@@ -70,7 +68,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
||||
"num_items_assigned",
|
||||
"positional_embed_overrides",
|
||||
"received_time",
|
||||
"require_reasoning",
|
||||
"return_bytes",
|
||||
"return_entropy",
|
||||
"return_flat_raw_output_top_logprobs",
|
||||
@@ -84,7 +81,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
||||
"routing_key",
|
||||
"sampling_logprobs_mode",
|
||||
"session_id",
|
||||
"session_params",
|
||||
"token_indices_to_pool",
|
||||
"use_audio_in_video",
|
||||
"video_config",
|
||||
|
||||
Reference in New Issue
Block a user