diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto index f183bef8f0..549ae1460c 100644 --- a/proto/sglang/api/v1/generate.proto +++ b/proto/sglang/api/v1/generate.proto @@ -2,7 +2,7 @@ // rust/sglang-server/src/message/request.rs's GenerateBody (request) and // api_core/frame.rs's frame shape (response). Unknown JSON keys on the // request are IGNORED: `deny_unknown_fields` broke real clients that send -// Python-only `GenerateReqInput` fields (priority, session_id, +// Python-only `GenerateReqInput` fields (session_id, // ...), pinned by request.rs's unported-fields test. syntax = "proto3"; @@ -57,6 +57,9 @@ message GenerateRequest { // key. One key for every batch item, or one per item; an empty key means // none. StringOrList extra_key = 23; + // Scheduling priority, one value for every batch item; the scheduler + // validates it against its priority-scheduling mode. + optional int64 priority = 24; } message SamplingParamsOrList { diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index 57c8c07d07..fb5611159c 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -258,6 +258,7 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): video_data: Optional[MediaInput] = None audio_data: Optional[MediaInput] = None extra_key: Optional[StringOrList] = None + priority: Optional[int] = None @classmethod def from_json_value(cls, value: Any) -> "GenerateRequest": @@ -1099,6 +1100,8 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest: kw["audio_data"] = None if x is None else decode_MediaInput(x) elif key == "extra_key": kw["extra_key"] = None if x is None else decode_StringOrList(x) + elif key == "priority": + kw["priority"] = None if x is None else _expect_int(x, expected="i64") return GenerateRequest(**kw) @@ -1150,6 +1153,8 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]: d["audio_data"] = encode_MediaInput(v.audio_data) if v.extra_key is not None: d["extra_key"] = encode_StringOrList(v.extra_key) + if v.priority is not None: + d["priority"] = v.priority return d diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index e910eef213..dd383653bc 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -9,12 +9,14 @@ from types import ModuleType from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple from sglang.srt.arg_groups.overrides import resolving_view +from sglang.srt.environ import envs from sglang.srt.managers.utils import compute_num_reserved_tokens from sglang.srt.runtime_context import ( get_disagg, get_mm, get_model, get_observability, + get_schedule, get_serving, ) from sglang.version import __version__ @@ -78,6 +80,12 @@ def _build_server_args( skip_tokenizer_init=get_serving().skip_tokenizer_init, skip_server_warmup=get_serving().skip_server_warmup, health_generate_headers=dict(health_generate_headers or {}), + default_priority_value=( + get_schedule().default_priority_value + if get_schedule().enable_priority_scheduling + else None + ), + enable_request_header_overrides=envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.get(), incremental_streaming_output=get_serving().incremental_streaming_output, disaggregation_mode=disaggregation_mode, model_config=ext.ModelConfig( diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index eb1df914a7..f1293a80ba 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -379,6 +379,10 @@ pub struct GenerateRequest { /// none. #[prost(message, optional, tag = "23")] pub extra_key: ::core::option::Option, + /// Scheduling priority, one value for every batch item; the scheduler + /// validates it against its priority-scheduling mode. + #[prost(int64, optional, tag = "24")] + pub priority: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct SamplingParamsOrList { diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index ff73d65b31..7579d355ee 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -1593,6 +1593,9 @@ impl ::serde::Serialize for GenerateRequest { if let Some(v) = &self.extra_key { ::serde::ser::SerializeMap::serialize_entry(&mut map, "extra_key", v)?; } + if let Some(v) = &self.priority { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "priority", v)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -1632,6 +1635,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { let mut f_video_data = None; let mut f_audio_data = None; let mut f_extra_key = None; + let mut f_priority = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "rid" => { @@ -1794,6 +1798,12 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { } f_extra_key = Some(map.next_value::>()?); } + "priority" => { + if f_priority.is_some() { + return Err(::serde::de::Error::duplicate_field("priority")); + } + f_priority = Some(map.next_value::>()?); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -1823,6 +1833,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { video_data: f_video_data.flatten(), audio_data: f_audio_data.flatten(), extra_key: f_extra_key.flatten(), + priority: f_priority.flatten(), }) } } diff --git a/rust/sglang-api-types/testdata/json_parity.json b/rust/sglang-api-types/testdata/json_parity.json index 369799ec73..67ba4017ef 100644 --- a/rust/sglang-api-types/testdata/json_parity.json +++ b/rust/sglang-api-types/testdata/json_parity.json @@ -89,7 +89,6 @@ "h1", "h2" ], - "priority": 3, "session_id": "ignored" }, "canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\"}" diff --git a/rust/sglang-server/src/api_server/http.rs b/rust/sglang-server/src/api_server/http.rs index a0484dae04..9e294651cf 100644 --- a/rust/sglang-server/src/api_server/http.rs +++ b/rust/sglang-server/src/api_server/http.rs @@ -3,3 +3,4 @@ mod info; mod metrics; mod native_api; mod openai; +mod request_headers; diff --git a/rust/sglang-server/src/api_server/http/native_api.rs b/rust/sglang-server/src/api_server/http/native_api.rs index d48d7b2e13..e6e01e38f1 100644 --- a/rust/sglang-server/src/api_server/http/native_api.rs +++ b/rust/sglang-server/src/api_server/http/native_api.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use super::app::AppState; +use super::request_headers::apply_header_overrides; use crate::api_server::core::{ CoreCall, CoreError, CoreEvent, CoreOutput, HealthStatus, recv_indexed, }; @@ -26,7 +27,7 @@ use axum::{ Json, Router, extract::State, extract::rejection::JsonRejection, - http::{HeaderName, HeaderValue, StatusCode}, + http::{HeaderMap, HeaderName, HeaderValue, StatusCode}, response::{ IntoResponse, Response, sse::{Event, Sse}, @@ -176,6 +177,7 @@ async fn health_generate( /// message, instead of axum's default 422. async fn generate( State(state): State>, + headers: HeaderMap, body: Result, JsonRejection>, ) -> Response { let mut body = match body { @@ -187,6 +189,11 @@ async fn generate( return native_error(StatusCode::BAD_REQUEST, &rejection.body_text(), false); } }; + if state.server_args.enable_request_header_overrides + && let Err(error) = apply_header_overrides(&mut body, &headers) + { + return native_error(StatusCode::BAD_REQUEST, &error, false); + } // Launch-time preferred sampling params fill in beneath the request's own // keys before decoding, so an explicit request value (even a default or // null) wins: Python TokenizerManager's precedence. diff --git a/rust/sglang-server/src/api_server/http/request_headers.rs b/rust/sglang-server/src/api_server/http/request_headers.rs new file mode 100644 index 0000000000..dd596abc42 --- /dev/null +++ b/rust/sglang-server/src/api_server/http/request_headers.rs @@ -0,0 +1,124 @@ +//! Router-computed request fields sent as headers, so the caller can leave the +//! `/generate` body opaque: the port of Python `request_headers.py`. Each header +//! overrides the body field of the same name before the body is decoded. + +use axum::http::HeaderMap; +use serde_json::Value; + +#[derive(Clone, Copy)] +enum Kind { + Str, + Int, +} + +/// Header → body field, Python's `_HEADER_OVERRIDES`. `conversation_id` has no +/// consumer past Python's TokenizerManager, so the schema drops it here. +const HEADER_OVERRIDES: &[(&str, &str, Kind)] = &[ + ("x-override-rid", "rid", Kind::Str), + ("x-override-bootstrap-host", "bootstrap_host", Kind::Str), + ("x-override-bootstrap-port", "bootstrap_port", Kind::Int), + ("x-override-bootstrap-room", "bootstrap_room", Kind::Int), + ("x-override-conversation-id", "conversation_id", Kind::Str), + ("x-override-routed-dp-rank", "routed_dp_rank", Kind::Int), + ( + "x-override-disagg-prefill-dp-rank", + "disagg_prefill_dp_rank", + Kind::Int, + ), + ("x-override-priority", "priority", Kind::Int), +]; + +/// Write each present override header into `body`. An unparsable value fails +/// the request (Python answers 400). A non-object body is left alone; decoding +/// rejects it. +pub(super) fn apply_header_overrides(body: &mut Value, headers: &HeaderMap) -> Result<(), String> { + let Some(fields) = body.as_object_mut() else { + return Ok(()); + }; + for &(header, field, kind) in HEADER_OVERRIDES { + let Some(value) = headers.get(header) else { + continue; + }; + let text = value + .to_str() + .map_err(|_| format!("invalid {header} header: not visible ASCII"))?; + let value = match kind { + Kind::Str => Value::from(text), + Kind::Int => text + .parse::() + .map(Value::from) + .map_err(|e| format!("invalid {header} header {text:?}: {e}"))?, + }; + fields.insert(field.to_owned(), value); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::message::request::into_requests; + + fn headers(pairs: &[(&'static str, &str)]) -> HeaderMap { + pairs + .iter() + .map(|&(name, value)| (name.parse().unwrap(), value.parse().unwrap())) + .collect() + } + + /// Headers override the body's fields, typed as Python casts them; absent + /// headers leave the body alone. The result decodes with every field in its + /// request slot, but `conversation_id`, which has none and is dropped. + #[test] + fn headers_override_body_fields() { + let mut body = serde_json::json!({"text": "hi", "rid": "client", "routed_dp_rank": 0}); + apply_header_overrides( + &mut body, + &headers(&[ + ("x-override-rid", "router"), + ("x-override-bootstrap-host", "10.0.0.1"), + ("x-override-bootstrap-port", "8998"), + ("x-override-bootstrap-room", "9223372036854775807"), + ("x-override-conversation-id", "conv"), + ("x-override-routed-dp-rank", "3"), + ("x-override-disagg-prefill-dp-rank", "1"), + ("x-override-priority", "-2"), + ]), + ) + .unwrap(); + assert_eq!( + body, + serde_json::json!({ + "text": "hi", "rid": "router", "routed_dp_rank": 3, + "bootstrap_host": "10.0.0.1", "bootstrap_port": 8998, + "bootstrap_room": i64::MAX, "conversation_id": "conv", + "disagg_prefill_dp_rank": 1, "priority": -2, + }) + ); + let (requests, _) = into_requests(serde_json::from_value(body.clone()).unwrap()).unwrap(); + let req = &requests[0]; + assert_eq!(req.rid.client_facing(), "router"); + assert_eq!(req.bootstrap_host.as_deref(), Some("10.0.0.1")); + let ints = [ + req.bootstrap_port, + req.bootstrap_room, + req.routed_dp_rank, + req.disagg_prefill_dp_rank, + req.priority, + ]; + assert_eq!( + ints, + [Some(8998), Some(i64::MAX), Some(3), Some(1), Some(-2)] + ); + + let err = apply_header_overrides( + &mut body, + &headers(&[("x-override-bootstrap-port", "eighty")]), + ) + .unwrap_err(); + assert!( + err.contains("invalid x-override-bootstrap-port header \"eighty\""), + "{err}" + ); + } +} diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index 27fca165e6..95f9a779eb 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -138,6 +138,12 @@ pub struct ServerArgs { /// Extra headers on a successful `/health_generate` response, for a router /// that reads deployment metadata off the deep health probe. pub health_generate_headers: BTreeMap, + /// Priority given to a generation request that sets none; `None` unless + /// priority scheduling is on (Python `_set_default_priority`). + pub default_priority_value: Option, + /// Apply the router's `x-override-*` headers to `/generate` bodies + /// (`SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES`). + pub enable_request_header_overrides: bool, /// Streamed `/generate` frames carry per-step deltas instead of cumulative /// text. Matches the Python `TokenizerManager`. pub incremental_streaming_output: bool, @@ -195,6 +201,8 @@ impl ServerArgs { skip_tokenizer_init, skip_server_warmup, health_generate_headers, + default_priority_value, + enable_request_header_overrides, incremental_streaming_output, disaggregation_mode, model_config, @@ -230,6 +238,8 @@ impl ServerArgs { skip_tokenizer_init: bool, skip_server_warmup: bool, health_generate_headers: BTreeMap, + default_priority_value: Option, + enable_request_header_overrides: bool, incremental_streaming_output: bool, disaggregation_mode: DisaggregationMode, model_config: ModelConfig, @@ -263,6 +273,8 @@ impl ServerArgs { skip_tokenizer_init, skip_server_warmup, health_generate_headers, + default_priority_value, + enable_request_header_overrides, incremental_streaming_output, disaggregation_mode, model_config, @@ -304,6 +316,8 @@ impl Default for ServerArgs { skip_tokenizer_init: false, skip_server_warmup: false, health_generate_headers: BTreeMap::new(), + default_priority_value: None, + enable_request_header_overrides: false, incremental_streaming_output: false, disaggregation_mode: DisaggregationMode::Null, model_config: ModelConfig::default(), diff --git a/rust/sglang-server/src/message/io_struct.rs b/rust/sglang-server/src/message/io_struct.rs index 0c6499afca..92b65e0749 100644 --- a/rust/sglang-server/src/message/io_struct.rs +++ b/rust/sglang-server/src/message/io_struct.rs @@ -54,12 +54,12 @@ wire_struct! { routed_dp_rank: Option, disagg_prefill_dp_rank: Option, /// The scheduler decodes this header by position (Python's - /// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so the - /// fields between `disagg_prefill_dp_rank` and `extra_key` must be sent, - /// at their defaults, for `extra_key` to land in its slot. + /// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so + /// `routing_key` and `require_reasoning` are sent at their defaults for + /// `priority` and `extra_key` to land in their slots. routing_key: (), require_reasoning: bool, - priority: (), + priority: Option, extra_key: Option<&'a str>, } } @@ -133,7 +133,7 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> { disagg_prefill_dp_rank: req.disagg_prefill_dp_rank, routing_key: (), require_reasoning: false, - priority: (), + priority: req.priority, extra_key: req.extra_key.as_deref(), } } @@ -295,6 +295,7 @@ mod tests { routed_dp_rank: Some(3), disagg_prefill_dp_rank: Some(4), extra_key: Some("salt".into()), + priority: Some(-2), ..Default::default() }; let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap(); @@ -315,7 +316,7 @@ mod tests { assert_eq!(arr[31].as_i64(), Some(4), "disagg_prefill_dp_rank at 31"); assert!(arr[32].is_nil(), "routing_key at 32"); assert_eq!(arr[33].as_bool(), Some(false), "require_reasoning at 33"); - assert!(arr[34].is_nil(), "priority at 34"); + assert_eq!(arr[34].as_i64(), Some(-2), "priority at 34"); assert_eq!(arr[35].as_str(), Some("salt"), "extra_key at 35"); } } diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index deb11231b4..5fd96969f9 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -80,6 +80,7 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec Result<(Vec, + /// Scheduling priority, shared by every item of a batch. + pub priority: Option, /// Multimodal inputs. Consumed by the Encoding stage, which ships them to /// the MM worker pool; never read by the tokenizer or serialized onto the /// scheduler header. Boxed so the common text-only request doesn't grow @@ -825,7 +829,6 @@ mod tests { #[test] fn unported_generate_req_input_fields_are_ignored() { for field in [ - r#""priority": 3"#, r#""session_id": "s""#, r#""session_params": {"a": 1}"#, r#""return_sampling_mask": true"#, diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs index 452a25cf90..61a1efa50b 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs @@ -165,6 +165,7 @@ fn test_limits() -> Limits { num_reserved_tokens: 0, allow_auto_truncate: false, enable_return_hidden_states: false, + default_priority: None, } } @@ -183,6 +184,28 @@ fn generate_req(id: u64, sampling_params: SamplingParams) -> Request { } } +/// An unset priority takes the configured default; an explicit one is kept +/// (Python `_set_default_priority`). +#[test] +fn unset_priority_takes_the_default() { + let limits = Limits { + default_priority: Some(0), + ..test_limits() + }; + for (sent, want) in [(None, Some(0)), (Some(-2), Some(-2))] { + let mut req = generate_req(1, SamplingParams::default()); + let RequestKind::Generate(g) = &mut req.kind else { + unreachable!() + }; + g.priority = sent; + validate(&mut req, &limits).unwrap(); + let RequestKind::Generate(g) = &req.kind else { + unreachable!() + }; + assert_eq!(g.priority, want); + } +} + /// `input + max_new_tokens` past the context window is an actionable 400, not a /// silently truncated 200 (Python `TokenizerManager._validate_one_request`). /// The message names both halves so the client can fix the right one. diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_types.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_types.rs index f452918007..4c67d66131 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_types.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_types.rs @@ -31,6 +31,9 @@ pub struct Limits { pub allow_auto_truncate: bool, /// Whether the server can produce hidden states at all. pub enable_return_hidden_states: bool, + /// Priority for a generation request that sets none (see + /// `ServerArgs::default_priority_value`). + pub default_priority: Option, } impl From<&ServerArgs> for Limits { @@ -42,6 +45,7 @@ impl From<&ServerArgs> for Limits { num_reserved_tokens: sa.num_reserved_tokens, allow_auto_truncate: sa.allow_auto_truncate, enable_return_hidden_states: sa.enable_return_hidden_states, + default_priority: sa.default_priority_value, } } } diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs index e3613db361..8cf2199acd 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs @@ -41,6 +41,14 @@ pub(super) fn validate(req: &mut Request, limits: &Limits) -> Result<(), Error> )); } + // Python `_set_default_priority`: an unset priority takes the configured + // default instead of ranking as the scheduler's lowest. + if let RequestKind::Generate(g) = &mut req.kind + && g.priority.is_none() + { + g.priority = limits.default_priority; + } + // Multimodal processors may consume sentinel ids outside the vocabulary. // Validate their resulting ids at PreSendValidating instead. Non-MM client // ids can be rejected now. diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 888a3eb453..68605119d8 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -68,7 +68,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "no_logs", "num_items_assigned", "positional_embed_overrides", - "priority", "received_time", "require_reasoning", "return_bytes", diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index 4a19421406..ca52aa5101 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -29,11 +29,27 @@ def _scheduler_for_typed_config(): @pytest.mark.parametrize( - "legacy_args,expected", - [({}, 50051), ({"smg_grpc_mode": True}, None), ({"grpc_mode": True}, None)], - ids=["native", "legacy-smg", "deprecated-legacy-smg"], + "overrides,kwarg,expected", + [ + ({}, "grpc_port", 50051), + ({"smg_grpc_mode": True}, "grpc_port", None), + ({"grpc_mode": True}, "grpc_port", None), + ({"default_priority_value": 5}, "default_priority_value", None), + ( + {"enable_priority_scheduling": True, "default_priority_value": 5}, + "default_priority_value", + 5, + ), + ], + ids=[ + "native-grpc", + "legacy-smg", + "deprecated-legacy-smg", + "priority-off", + "priority-on", + ], ) -def test_typed_config_only_forwards_native_grpc_port(legacy_args, expected): +def test_typed_config_forwards_resolved_launch_fields(overrides, kwarg, expected): extension = SimpleNamespace( DisaggregationMode=SimpleNamespace( Null="null", Prefill="prefill", Decode="decode" @@ -43,7 +59,7 @@ def test_typed_config_only_forwards_native_grpc_port(legacy_args, expected): ServerArgs=MagicMock(return_value="server-args"), ) with ( - get_context().override_server_args(grpc_port=50051, **legacy_args), + get_context().override_server_args(grpc_port=50051, **overrides), patch.object(rust_extensions, "load_rust_extension", return_value=extension), patch.object(rust_config, "compute_num_reserved_tokens", return_value=0), ): @@ -52,7 +68,7 @@ def test_typed_config_only_forwards_native_grpc_port(legacy_args, expected): == "server-args" ) - assert extension.ServerArgs.call_args.kwargs["grpc_port"] == expected + assert extension.ServerArgs.call_args.kwargs[kwarg] == expected @pytest.mark.parametrize("enable_metrics", [False, True])