[Feat] rust-server: Apply the x-override-* request headers (#44312)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
shodoco
parent
4e0609ea27
commit
0b1a73599a
@@ -258,6 +258,7 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
|
||||
video_data: Optional[MediaInput] = None
|
||||
audio_data: Optional[MediaInput] = None
|
||||
extra_key: Optional[StringOrList] = None
|
||||
priority: Optional[int] = None
|
||||
|
||||
@classmethod
|
||||
def from_json_value(cls, value: Any) -> "GenerateRequest":
|
||||
@@ -1099,6 +1100,8 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest:
|
||||
kw["audio_data"] = None if x is None else decode_MediaInput(x)
|
||||
elif key == "extra_key":
|
||||
kw["extra_key"] = None if x is None else decode_StringOrList(x)
|
||||
elif key == "priority":
|
||||
kw["priority"] = None if x is None else _expect_int(x, expected="i64")
|
||||
return GenerateRequest(**kw)
|
||||
|
||||
|
||||
@@ -1150,6 +1153,8 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]:
|
||||
d["audio_data"] = encode_MediaInput(v.audio_data)
|
||||
if v.extra_key is not None:
|
||||
d["extra_key"] = encode_StringOrList(v.extra_key)
|
||||
if v.priority is not None:
|
||||
d["priority"] = v.priority
|
||||
return d
|
||||
|
||||
|
||||
|
||||
@@ -9,12 +9,14 @@ from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.utils import compute_num_reserved_tokens
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_mm,
|
||||
get_model,
|
||||
get_observability,
|
||||
get_schedule,
|
||||
get_serving,
|
||||
)
|
||||
from sglang.version import __version__
|
||||
@@ -78,6 +80,12 @@ def _build_server_args(
|
||||
skip_tokenizer_init=get_serving().skip_tokenizer_init,
|
||||
skip_server_warmup=get_serving().skip_server_warmup,
|
||||
health_generate_headers=dict(health_generate_headers or {}),
|
||||
default_priority_value=(
|
||||
get_schedule().default_priority_value
|
||||
if get_schedule().enable_priority_scheduling
|
||||
else None
|
||||
),
|
||||
enable_request_header_overrides=envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.get(),
|
||||
incremental_streaming_output=get_serving().incremental_streaming_output,
|
||||
disaggregation_mode=disaggregation_mode,
|
||||
model_config=ext.ModelConfig(
|
||||
|
||||
Reference in New Issue
Block a user