[Feat] rust-server: Attach configured metadata headers to a healthy /health_generate (#44315)
Co-authored-by: fei-xx <fei-xx@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
fei-xx
shodoco
parent
173e0f2e9b
commit
02a43eeef6
@@ -6,7 +6,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Callable, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.managers.utils import compute_num_reserved_tokens
|
||||
@@ -27,7 +27,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_server_args(
|
||||
scheduler: Scheduler, *, extension: Optional[ModuleType] = None
|
||||
scheduler: Scheduler,
|
||||
*,
|
||||
extension: Optional[ModuleType] = None,
|
||||
health_generate_headers: Optional[Dict[str, str]] = None,
|
||||
) -> ServerArgs:
|
||||
"""The typed launch configuration for the scheduler's embedded Rust server:
|
||||
the ``server_args`` fields it reads, the already-resolved
|
||||
@@ -74,6 +77,7 @@ def _build_server_args(
|
||||
detokenizer_worker_num=get_serving().detokenizer_worker_num,
|
||||
skip_tokenizer_init=get_serving().skip_tokenizer_init,
|
||||
skip_server_warmup=get_serving().skip_server_warmup,
|
||||
health_generate_headers=dict(health_generate_headers or {}),
|
||||
incremental_streaming_output=get_serving().incremental_streaming_output,
|
||||
disaggregation_mode=disaggregation_mode,
|
||||
model_config=ext.ModelConfig(
|
||||
|
||||
@@ -68,6 +68,12 @@ class RustServer:
|
||||
self._multimodal_enabled = mm_spec is not None
|
||||
self._max_per_poll = max_per_poll
|
||||
|
||||
@classmethod
|
||||
def _health_generate_headers(cls, scheduler: Scheduler) -> Dict[str, str]:
|
||||
"""Headers a healthy ``/health_generate`` carries; a deployment's
|
||||
router may read its metadata off the deep health probe."""
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _load_extension(cls) -> ModuleType:
|
||||
from sglang.srt.rust_extensions import load_rust_extension
|
||||
@@ -167,7 +173,11 @@ class RustServer:
|
||||
|
||||
extension = cls._load_extension()
|
||||
server = extension.Server(
|
||||
_build_server_args(scheduler, extension=extension),
|
||||
_build_server_args(
|
||||
scheduler,
|
||||
extension=extension,
|
||||
health_generate_headers=cls._health_generate_headers(scheduler),
|
||||
),
|
||||
# None runs unpinned; otherwise the list carries the pinning decision.
|
||||
cores=server_cores,
|
||||
port_offset=local_dp_rank,
|
||||
|
||||
Reference in New Issue
Block a user