feat(grpc): expose follower metadata and node-local KV sources (#39659)
Signed-off-by: jain-ria <riajain@NVIDIA.com> Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
This commit is contained in:
@@ -13,6 +13,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tonic_build::configure()
|
||||
.build_server(true)
|
||||
.build_client(true)
|
||||
// The follower implements only GetServerInfo; all other RPCs return
|
||||
// UNIMPLEMENTED without constructing the inference bridge.
|
||||
.generate_default_stubs(true)
|
||||
.protoc_arg("--experimental_allow_proto3_optional")
|
||||
.compile_protos(&[proto_path], &["."])?;
|
||||
|
||||
|
||||
@@ -80,7 +80,8 @@ class Serving(msgspec.Struct):
|
||||
Optional[int],
|
||||
"Port for the native gRPC server, started alongside HTTP. Setting this "
|
||||
"(or SGLANG_GRPC_PORT) enables the native gRPC server; it is off by "
|
||||
"default. In legacy --smg-grpc-mode this is the SMG server port and "
|
||||
"default. Follower nodes expose GetServerInfo only. "
|
||||
"In legacy --smg-grpc-mode this is the SMG server port and "
|
||||
"defaults to --port + 10000.",
|
||||
] = None
|
||||
grpc_response_timeout_secs: A[
|
||||
|
||||
@@ -20,6 +20,7 @@ KV caching events
|
||||
import atexit
|
||||
import enum
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
@@ -329,6 +330,10 @@ class EventPublisher(ABC):
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the publisher."""
|
||||
|
||||
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
|
||||
"""Describe a source accessible to a separate local process, if any."""
|
||||
return None
|
||||
|
||||
|
||||
class NullEventPublisher(EventPublisher):
|
||||
"""No-op implementation (default when disabled)."""
|
||||
@@ -349,7 +354,7 @@ class ZmqEventPublisher(EventPublisher):
|
||||
----------
|
||||
endpoint:
|
||||
PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to
|
||||
connect.
|
||||
connect, unless overridden by ``bind``.
|
||||
replay_endpoint:
|
||||
Optional ROUTER address for replay requests. When given, subscribers can
|
||||
request missed batches by sending the starting sequence number as an
|
||||
@@ -362,6 +367,9 @@ class ZmqEventPublisher(EventPublisher):
|
||||
Maximum number of events to buffer in memory.
|
||||
topic:
|
||||
Topic to publish events to.
|
||||
bind:
|
||||
Whether to bind the PUB socket. When unset, preserve the endpoint-based
|
||||
heuristic: bind wildcard, IPC and inproc addresses; connect otherwise.
|
||||
"""
|
||||
|
||||
SHUTDOWN_TIMEOUT: float = 1.0
|
||||
@@ -376,6 +384,7 @@ class ZmqEventPublisher(EventPublisher):
|
||||
hwm: int = 100_000,
|
||||
max_queue_size: int = 100_000,
|
||||
topic: str = "",
|
||||
bind: Optional[bool] = None,
|
||||
) -> None:
|
||||
# Storage
|
||||
self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size)
|
||||
@@ -384,9 +393,12 @@ class ZmqEventPublisher(EventPublisher):
|
||||
# ZMQ sockets
|
||||
self._ctx = zmq.Context.instance()
|
||||
self._pub: Optional[zmq.Socket] = None
|
||||
self._local_pub_endpoint: Optional[str] = None
|
||||
self._local_replay_endpoint: Optional[str] = None
|
||||
self._replay: Optional[zmq.Socket] = None
|
||||
self._dp_rank = attn_dp_rank
|
||||
self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank)
|
||||
self._bind = bind
|
||||
self._replay_endpoint = self.offset_endpoint_port(
|
||||
replay_endpoint, self._dp_rank
|
||||
)
|
||||
@@ -415,6 +427,43 @@ class ZmqEventPublisher(EventPublisher):
|
||||
events.attn_dp_rank = self._dp_rank
|
||||
self._event_queue.put(events)
|
||||
|
||||
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
|
||||
"""Report the bound socket, not a port reconstructed from global DP size.
|
||||
|
||||
Connect-style publishers and inproc sockets have no subscribable local
|
||||
endpoint. They keep working as before, but are not advertised to external
|
||||
processes. Wildcard binds are reachable on loopback from this node.
|
||||
"""
|
||||
|
||||
def local_endpoint(endpoint: Optional[str]) -> Optional[str]:
|
||||
if endpoint is None:
|
||||
return None
|
||||
if endpoint.startswith("ipc://"):
|
||||
return endpoint
|
||||
if parse_advertisable_tcp(endpoint) is None:
|
||||
return None
|
||||
address = NetworkAddress.parse(endpoint[len("tcp://") :])
|
||||
host = address.host
|
||||
if host in ("*", "0.0.0.0"):
|
||||
host = "127.0.0.1"
|
||||
elif host == "::":
|
||||
host = "::1"
|
||||
return NetworkAddress(host, address.port).to_tcp()
|
||||
|
||||
endpoint = local_endpoint(self._local_pub_endpoint)
|
||||
if endpoint is None:
|
||||
return None
|
||||
source = {
|
||||
"dp_rank": self._dp_rank,
|
||||
"endpoint": endpoint,
|
||||
"topic": self._topic_bytes.decode("utf-8"),
|
||||
"block_size": block_size,
|
||||
}
|
||||
replay_endpoint = local_endpoint(self._local_replay_endpoint)
|
||||
if replay_endpoint is not None:
|
||||
source["replay_endpoint"] = replay_endpoint
|
||||
return source
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the publisher thread and clean up resources."""
|
||||
self._running = False
|
||||
@@ -453,23 +502,27 @@ class ZmqEventPublisher(EventPublisher):
|
||||
if self._pub is None:
|
||||
self._pub = self._ctx.socket(zmq.PUB)
|
||||
self._pub.set_hwm(self._hwm)
|
||||
# Heuristic: bind if wildcard / * present, else connect.
|
||||
# Default heuristic: bind if wildcard / * present, else connect.
|
||||
# bind stable, connect volatile convention.
|
||||
# ``0.0.0.0`` is the IPv4 bind-all wildcard alongside ``*``
|
||||
# and ``::``; ``/server_info`` advertises it as a wildcard,
|
||||
# so the publisher must bind it for the advertised endpoint
|
||||
# to actually be listening.
|
||||
if (
|
||||
"*" in self._endpoint
|
||||
or "::" in self._endpoint
|
||||
or "0.0.0.0" in self._endpoint
|
||||
or self._endpoint.startswith("ipc://")
|
||||
or self._endpoint.startswith("inproc://")
|
||||
):
|
||||
should_bind = self._bind
|
||||
if should_bind is None:
|
||||
should_bind = (
|
||||
"*" in self._endpoint
|
||||
or "::" in self._endpoint
|
||||
or "0.0.0.0" in self._endpoint
|
||||
or self._endpoint.startswith("ipc://")
|
||||
or self._endpoint.startswith("inproc://")
|
||||
)
|
||||
if should_bind:
|
||||
logger.debug(
|
||||
f"ZmqEventPublisher socket publisher_endpoint bind to {self._endpoint}"
|
||||
)
|
||||
self._pub.bind(self._endpoint)
|
||||
self._local_pub_endpoint = self._bound_endpoint(self._pub)
|
||||
else:
|
||||
self._pub.connect(self._endpoint)
|
||||
|
||||
@@ -483,6 +536,19 @@ class ZmqEventPublisher(EventPublisher):
|
||||
f"ZmqEventPublisher socket replay_endpoint bind to {self._replay_endpoint}"
|
||||
)
|
||||
self._replay.bind(self._replay_endpoint)
|
||||
self._local_replay_endpoint = self._bound_endpoint(self._replay)
|
||||
|
||||
@staticmethod
|
||||
def _bound_endpoint(socket: zmq.Socket) -> str:
|
||||
endpoint = socket.getsockopt_string(zmq.LAST_ENDPOINT)
|
||||
if endpoint.startswith("ipc://"):
|
||||
path = endpoint[len("ipc://") :]
|
||||
# Capture the publisher's working directory at bind time so a
|
||||
# separate process can use the address. Abstract IPC names are
|
||||
# independent of the filesystem and must remain unchanged.
|
||||
if not path.startswith("@"):
|
||||
endpoint = f"ipc://{os.path.abspath(path)}"
|
||||
return endpoint
|
||||
|
||||
def _publisher_thread(self) -> None:
|
||||
"""Background thread that processes the event queue."""
|
||||
@@ -589,6 +655,11 @@ class KVEventsConfig(BaseModel):
|
||||
"""The zmq endpoint to use for publishing kv events.
|
||||
"""
|
||||
|
||||
bind: Optional[bool] = None
|
||||
"""Bind (true) or connect (false) the PUB socket. When unset, infer from
|
||||
the endpoint as before. Use true to bind a concrete address such as loopback.
|
||||
"""
|
||||
|
||||
replay_endpoint: Optional[str] = None
|
||||
"""The zmq endpoint to use for replaying kv events.
|
||||
"""
|
||||
|
||||
@@ -140,6 +140,7 @@ from sglang.srt.utils import (
|
||||
numa_utils,
|
||||
set_prometheus_multiproc_dir,
|
||||
set_ulimit,
|
||||
start_follower_grpc_server,
|
||||
)
|
||||
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
|
||||
from sglang.srt.utils.network import (
|
||||
@@ -173,6 +174,12 @@ class SchedulerInitResult:
|
||||
wait_for_ready: Callable[[], None] = lambda: None
|
||||
block_until_scheduler_exits: Callable[[], None] = lambda: None
|
||||
engine_info_bootstrap_server: Optional[Any] = None
|
||||
grpc_server: Optional[Any] = None
|
||||
|
||||
def stop_grpc_server(self) -> None:
|
||||
if self.grpc_server is not None:
|
||||
self.grpc_server.shutdown()
|
||||
self.grpc_server = None
|
||||
|
||||
|
||||
def init_tokenizer_manager(
|
||||
@@ -951,6 +958,17 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
|
||||
def wait_for_ready():
|
||||
infos = _wait_for_scheduler_ready(scheduler_pipe_readers, scheduler_procs)
|
||||
if any("kv_event_sources" in info for info in infos):
|
||||
# Both gRPC entrypoints consume the first scheduler info. Keep
|
||||
# the sources from every local scheduler, not just the first.
|
||||
infos[0]["kv_event_sources"] = sorted(
|
||||
(
|
||||
source
|
||||
for info in infos
|
||||
for source in info.get("kv_event_sources", [])
|
||||
),
|
||||
key=lambda source: source["dp_rank"],
|
||||
)
|
||||
scheduler_infos.extend(infos)
|
||||
if use_dp_controller:
|
||||
for info in infos:
|
||||
@@ -1173,6 +1191,18 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
# Non-zero-rank nodes do not run tokenizer processes.
|
||||
scheduler_init_result.wait_for_ready()
|
||||
|
||||
try:
|
||||
scheduler_init_result.grpc_server = start_follower_grpc_server(
|
||||
server_args, scheduler_init_result.scheduler_infos[0]
|
||||
)
|
||||
except BaseException:
|
||||
# Engine.__init__ has not received these handles yet. Do not
|
||||
# leave GPU workers behind if binding the metadata port fails.
|
||||
for proc in scheduler_procs or []:
|
||||
kill_process_tree(proc.pid, wait_timeout=60)
|
||||
cls._terminate_weight_cache_daemons(weight_cache_daemon_procs)
|
||||
raise
|
||||
|
||||
if os.getenv("SGLANG_BLOCK_NONZERO_RANK_CHILDREN") == "0":
|
||||
# When using `Engine` as a Python API, we don't want to block here.
|
||||
return (
|
||||
@@ -1202,14 +1232,16 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
rust_server_owns_base_port = (
|
||||
envs.SGLANG_RUST_SERVER.get() and node_hosts_rust_server()
|
||||
)
|
||||
if not rust_server_owns_base_port:
|
||||
launch_dummy_health_check_server(
|
||||
get_serving().host,
|
||||
get_serving().port,
|
||||
get_observability().enable_metrics,
|
||||
)
|
||||
|
||||
scheduler_init_result.block_until_scheduler_exits()
|
||||
try:
|
||||
if not rust_server_owns_base_port:
|
||||
launch_dummy_health_check_server(
|
||||
get_serving().host,
|
||||
get_serving().port,
|
||||
get_observability().enable_metrics,
|
||||
)
|
||||
scheduler_init_result.block_until_scheduler_exits()
|
||||
finally:
|
||||
scheduler_init_result.stop_grpc_server()
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
@@ -1384,6 +1416,9 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
pass
|
||||
self._multi_tokenizer_shm = None
|
||||
try:
|
||||
scheduler_init_result = getattr(self, "_scheduler_init_result", None)
|
||||
if scheduler_init_result is not None:
|
||||
scheduler_init_result.stop_grpc_server()
|
||||
if (
|
||||
self.tokenizer_manager is not None
|
||||
and self.tokenizer_manager._subprocess_watchdog is not None
|
||||
@@ -1789,7 +1824,6 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
|
||||
|
||||
def _set_envs_and_config(server_args: ServerArgs):
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
# Set global environments
|
||||
# MNNVL fabric (GB200/GB300) multi-node: cross-node NVLink needs NCCL's
|
||||
|
||||
@@ -16,11 +16,8 @@ from pydantic import ValidationError
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
|
||||
from sglang.srt.runtime_context import (
|
||||
describe_kv_events_publisher,
|
||||
get_lora,
|
||||
get_serving,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_lora, get_serving
|
||||
from sglang.srt.utils.common import build_server_info
|
||||
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -238,9 +235,7 @@ class RuntimeHandle:
|
||||
return self._openai_serving_classes
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
from sglang.srt.entrypoints.openai.serving_classify import (
|
||||
OpenAIServingClassify,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
|
||||
from sglang.srt.entrypoints.openai.serving_completions import (
|
||||
OpenAIServingCompletion,
|
||||
)
|
||||
@@ -430,17 +425,14 @@ class RuntimeHandle:
|
||||
return json.dumps(result, default=str)
|
||||
|
||||
def get_server_info(self) -> str:
|
||||
result: Dict[str, Any] = self.tokenizer_manager.server_args.resolved_dict()
|
||||
# `resolved_dict` answers with what resolution decided; the launch
|
||||
# command answers with what was asked for, and the two are not
|
||||
# derivable from each other. The HTTP and in-process readbacks both
|
||||
# carry it, so this one does too.
|
||||
result["launch_command"] = self.tokenizer_manager.server_args.launch_command
|
||||
result.update(self.scheduler_info)
|
||||
result["kv_events"] = describe_kv_events_publisher(
|
||||
self.tokenizer_manager.server_args
|
||||
return json.dumps(
|
||||
msgspec_to_builtins(
|
||||
build_server_info(
|
||||
self.tokenizer_manager.server_args, self.scheduler_info
|
||||
)
|
||||
),
|
||||
default=str,
|
||||
)
|
||||
return json.dumps(msgspec_to_builtins(result), default=str)
|
||||
|
||||
def health_check(self) -> bool:
|
||||
from sglang.srt.managers.tokenizer_manager import ServerStatus
|
||||
|
||||
@@ -177,6 +177,7 @@ from sglang.srt.server_args import PortArgs, ServerArgs
|
||||
from sglang.srt.utils import (
|
||||
add_prometheus_middleware,
|
||||
add_prometheus_track_response_middleware,
|
||||
build_server_info,
|
||||
delete_directory,
|
||||
get_bool_env_var,
|
||||
is_mps,
|
||||
@@ -502,7 +503,6 @@ app.include_router(v1_loads_router)
|
||||
from sglang.srt.arg_groups.serving_hook import ssl_verify_of
|
||||
from sglang.srt.entrypoints.elastic_ep import router as elastic_ep_router
|
||||
from sglang.srt.runtime_context import (
|
||||
describe_kv_events_publisher,
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_lora,
|
||||
@@ -859,17 +859,11 @@ async def server_info():
|
||||
|
||||
return msgspec_to_builtins(
|
||||
{
|
||||
**server_args.resolved_dict(),
|
||||
"launch_command": server_args.launch_command,
|
||||
**_global_state.scheduler_info,
|
||||
**build_server_info(server_args, _global_state.scheduler_info),
|
||||
"startup_time": _global_state.tokenizer_manager.startup_time,
|
||||
"internal_states": internal_states,
|
||||
"version": __version__,
|
||||
"frontend": "python",
|
||||
# Structured KV-event publisher descriptor for KV-aware routers.
|
||||
# `None` when publishing is disabled or misconfigured; see
|
||||
# `runtime_context.describe_kv_events_publisher` for the contract.
|
||||
"kv_events": describe_kv_events_publisher(server_args),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -200,6 +200,7 @@ class DataParallelController:
|
||||
|
||||
# Launch data parallel workers
|
||||
self.scheduler_procs = []
|
||||
self.local_kv_event_sources = []
|
||||
self.workers: list[zmq.Socket | None] = [None] * self.max_dp_size
|
||||
self.status: list[bool] = list(self.dp_active)
|
||||
self._active_workers: list[int] = list(range(self.launch_dp_size))
|
||||
@@ -723,6 +724,15 @@ class DataParallelController:
|
||||
for i in range(len(scheduler_pipe_readers)):
|
||||
scheduler_info.append(scheduler_pipe_readers[i].recv())
|
||||
|
||||
# Pure-DP TP groups launch concurrently. Their ready replies contain
|
||||
# only publishers owned by schedulers on this node.
|
||||
with self.env_lock:
|
||||
self.local_kv_event_sources.extend(
|
||||
source
|
||||
for info in scheduler_info
|
||||
for source in info.get("kv_event_sources", [])
|
||||
)
|
||||
|
||||
self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"]
|
||||
self.max_req_input_len = scheduler_info[0]["max_req_input_len"]
|
||||
self.startup_time = aggregate_scheduler_startup_times(
|
||||
@@ -836,15 +846,20 @@ def run_data_parallel_controller_process(
|
||||
scheduler_pids = [
|
||||
proc.pid for proc in controller.scheduler_procs if proc is not None
|
||||
]
|
||||
pipe_writer.send(
|
||||
{
|
||||
"status": "ready",
|
||||
"max_total_num_tokens": controller.max_total_num_tokens,
|
||||
"max_req_input_len": controller.max_req_input_len,
|
||||
"startup_time": controller.startup_time,
|
||||
SCHEDULER_PIDS_ARG: scheduler_pids,
|
||||
}
|
||||
)
|
||||
init_info = {
|
||||
"status": "ready",
|
||||
"max_total_num_tokens": controller.max_total_num_tokens,
|
||||
"max_req_input_len": controller.max_req_input_len,
|
||||
"startup_time": controller.startup_time,
|
||||
SCHEDULER_PIDS_ARG: scheduler_pids,
|
||||
}
|
||||
if get_serving().grpc_port is not None and not (
|
||||
get_serving().smg_grpc_mode or get_serving().grpc_mode
|
||||
):
|
||||
init_info["kv_event_sources"] = sorted(
|
||||
controller.local_kv_event_sources, key=lambda source: source["dp_rank"]
|
||||
)
|
||||
pipe_writer.send(init_info)
|
||||
# The primary owns routing for the expanded scheduler set.
|
||||
if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
|
||||
controller.event_loop()
|
||||
|
||||
@@ -1826,6 +1826,14 @@ class Scheduler(
|
||||
"startup_time": self.startup_time,
|
||||
}
|
||||
|
||||
if get_serving().grpc_port is not None and not (
|
||||
get_serving().smg_grpc_mode or get_serving().grpc_mode
|
||||
):
|
||||
result_dict["kv_event_sources"] = (
|
||||
self.kv_events_publisher.local_kv_event_sources(
|
||||
self.page_size * get_parallel().dcp_size
|
||||
)
|
||||
)
|
||||
return result_dict
|
||||
|
||||
def release_host_resources(self) -> None:
|
||||
|
||||
@@ -89,6 +89,12 @@ class SchedulerKvEventsPublisher:
|
||||
if not self.send_metrics_from_scheduler.closed:
|
||||
sock_send(self.send_metrics_from_scheduler, kv_metrics)
|
||||
|
||||
def local_kv_event_sources(self, block_size: int) -> list[dict[str, Any]]:
|
||||
if self.kv_event_publisher is None:
|
||||
return []
|
||||
source = self.kv_event_publisher.describe_local_source(block_size)
|
||||
return [source] if source is not None else []
|
||||
|
||||
def publish_kv_events(self):
|
||||
if not self.enable_kv_cache_events:
|
||||
return
|
||||
|
||||
@@ -101,13 +101,16 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.observability.func_timer import enable_func_timer
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import (
|
||||
describe_kv_events_publisher,
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
get_serving,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
|
||||
from sglang.srt.utils.video_decoder import _BACKEND, VideoDecoderWrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -3649,6 +3652,41 @@ def _configure_uvicorn_access_log_filter(
|
||||
filters_list.append(filter_name)
|
||||
|
||||
|
||||
def build_server_info(server_args, scheduler_info: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build server metadata shared by HTTP and native gRPC.
|
||||
|
||||
Callers add transport-specific fields and serialize the result.
|
||||
"""
|
||||
result = server_args.resolved_dict()
|
||||
result["launch_command"] = server_args.launch_command
|
||||
result.update(scheduler_info)
|
||||
result["kv_events"] = describe_kv_events_publisher(server_args)
|
||||
return result
|
||||
|
||||
|
||||
def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]):
|
||||
"""Expose GetServerInfo only, without constructing a TokenizerManager.
|
||||
|
||||
The snapshot is taken after scheduler readiness. This does not launch a
|
||||
sidecar or enable inference/control RPCs.
|
||||
"""
|
||||
serving = get_serving()
|
||||
if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode:
|
||||
return None
|
||||
|
||||
from sglang.srt.rust_extensions import load_rust_extension
|
||||
|
||||
grpc_native = load_rust_extension("sglang.srt.rust_extensions._grpc")
|
||||
return grpc_native.start_metadata_server(
|
||||
host=serving.host,
|
||||
port=serving.grpc_port,
|
||||
server_info_json=json.dumps(
|
||||
msgspec_to_builtins(build_server_info(server_args, scheduler_info)),
|
||||
default=str,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def launch_dummy_health_check_server(host, port, enable_metrics):
|
||||
import asyncio
|
||||
|
||||
|
||||
+80
-39
@@ -36,6 +36,76 @@ impl GrpcServerHandle {
|
||||
}
|
||||
}
|
||||
|
||||
fn bind_listener(host: &str, port: u16) -> PyResult<TcpListener> {
|
||||
let addr: SocketAddr = format!("{}:{}", host, port)
|
||||
.parse()
|
||||
.map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {}", e)))?;
|
||||
let listener = TcpListener::bind(addr).map_err(|e| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(format!(
|
||||
"Failed to bind gRPC server to {}: {}",
|
||||
addr, e
|
||||
))
|
||||
})?;
|
||||
listener.set_nonblocking(true).map_err(|e| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(format!(
|
||||
"Failed to configure gRPC listener for {}: {}",
|
||||
addr, e
|
||||
))
|
||||
})?;
|
||||
Ok(listener)
|
||||
}
|
||||
|
||||
fn start_server_thread(
|
||||
rt: tokio::runtime::Runtime,
|
||||
listener: TcpListener,
|
||||
service: server::ServerMode,
|
||||
) -> PyResult<GrpcServerHandle> {
|
||||
let shutdown = Arc::new(Notify::new());
|
||||
let shutdown_clone = shutdown.clone();
|
||||
let join_handle = std::thread::Builder::new()
|
||||
.name("sglang-grpc".to_string())
|
||||
.spawn(move || {
|
||||
if let Err(e) = rt.block_on(server::run_grpc_server(listener, service, shutdown_clone))
|
||||
{
|
||||
tracing::error!("gRPC server exited with error: {}", e);
|
||||
}
|
||||
})
|
||||
.map_err(|e| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to spawn gRPC thread: {}", e))
|
||||
})?;
|
||||
|
||||
Ok(GrpcServerHandle {
|
||||
shutdown,
|
||||
join_handle: Some(join_handle),
|
||||
})
|
||||
}
|
||||
|
||||
/// Start a follower server exposing only GetServerInfo, without an inference bridge.
|
||||
#[pyfunction]
|
||||
fn start_metadata_server(
|
||||
host: String,
|
||||
port: u16,
|
||||
server_info_json: String,
|
||||
) -> PyResult<GrpcServerHandle> {
|
||||
let listener = bind_listener(&host, port)?;
|
||||
// Metadata is a startup snapshot; it needs neither Python callbacks nor a
|
||||
// worker pool. The dedicated server thread drives this runtime directly.
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(format!(
|
||||
"Failed to build Tokio runtime for gRPC metadata server: {}",
|
||||
err
|
||||
))
|
||||
})?;
|
||||
start_server_thread(
|
||||
rt,
|
||||
listener,
|
||||
server::ServerMode::Metadata(server::MetadataService { server_info_json }),
|
||||
)
|
||||
}
|
||||
|
||||
struct TokenizerInfo {
|
||||
tokenizer_path: Option<String>,
|
||||
tokenizer_mode: Option<String>,
|
||||
@@ -160,9 +230,6 @@ fn start_server(
|
||||
)
|
||||
.try_init();
|
||||
|
||||
let addr: SocketAddr = format!("{}:{}", host, port)
|
||||
.parse()
|
||||
.map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {}", e)))?;
|
||||
let worker_threads = worker_threads.max(1);
|
||||
let response_channel_capacity = if response_channel_capacity == 0 {
|
||||
tracing::warn!(
|
||||
@@ -183,18 +250,7 @@ fn start_server(
|
||||
response_timeout_secs
|
||||
};
|
||||
let response_timeout = Duration::from_secs(response_timeout_secs);
|
||||
let listener = TcpListener::bind(addr).map_err(|e| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(format!(
|
||||
"Failed to bind gRPC server to {}: {}",
|
||||
addr, e
|
||||
))
|
||||
})?;
|
||||
listener.set_nonblocking(true).map_err(|e| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(format!(
|
||||
"Failed to configure gRPC listener for {}: {}",
|
||||
addr, e
|
||||
))
|
||||
})?;
|
||||
let listener = bind_listener(&host, port)?;
|
||||
|
||||
let tokenizer_info = extract_tokenizer_info(&runtime_handle)?;
|
||||
|
||||
@@ -226,35 +282,20 @@ fn start_server(
|
||||
response_channel_capacity,
|
||||
tokio_handle,
|
||||
));
|
||||
let shutdown = Arc::new(Notify::new());
|
||||
let shutdown_clone = shutdown.clone();
|
||||
let bridge_clone = bridge.clone();
|
||||
|
||||
let join_handle = std::thread::Builder::new()
|
||||
.name("sglang-grpc".to_string())
|
||||
.spawn(move || {
|
||||
if let Err(e) = rt.block_on(server::run_grpc_server(
|
||||
listener,
|
||||
bridge_clone,
|
||||
shutdown_clone,
|
||||
response_timeout,
|
||||
)) {
|
||||
tracing::error!("gRPC server exited with error: {}", e);
|
||||
}
|
||||
})
|
||||
.map_err(|e| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to spawn gRPC thread: {}", e))
|
||||
})?;
|
||||
|
||||
Ok(GrpcServerHandle {
|
||||
shutdown,
|
||||
join_handle: Some(join_handle),
|
||||
})
|
||||
start_server_thread(
|
||||
rt,
|
||||
listener,
|
||||
server::ServerMode::Inference {
|
||||
bridge,
|
||||
response_timeout,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[pymodule]
|
||||
fn _grpc(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(start_server, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(start_metadata_server, m)?)?;
|
||||
m.add_class::<GrpcServerHandle>()?;
|
||||
m.add_class::<ChunkSendStatus>()?;
|
||||
Ok(())
|
||||
|
||||
@@ -26,6 +26,25 @@ pub struct SglangServiceImpl {
|
||||
stream_shutdown: watch::Receiver<bool>,
|
||||
}
|
||||
|
||||
/// A follower has no tokenizer manager or inference bridge. It exposes the same
|
||||
/// discovery RPC as the leader, with a node-local startup snapshot. Generated
|
||||
/// default handlers return UNIMPLEMENTED for every other RPC.
|
||||
pub struct MetadataService {
|
||||
pub server_info_json: String,
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl proto::sglang_service_server::SglangService for MetadataService {
|
||||
async fn get_server_info(
|
||||
&self,
|
||||
_request: Request<proto::GetServerInfoRequest>,
|
||||
) -> Result<Response<proto::GetServerInfoResponse>, Status> {
|
||||
Ok(Response::new(proto::GetServerInfoResponse {
|
||||
json_info: self.server_info_json.clone(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
type StreamResult<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
|
||||
pub const DEFAULT_RESPONSE_TIMEOUT_SECS: u64 = 300;
|
||||
|
||||
@@ -300,12 +319,10 @@ fn openai_status_code(meta_info: &HashMap<String, String>, default: i32) -> i32
|
||||
impl proto::sglang_service_server::SglangService for SglangServiceImpl {
|
||||
// --- SGLang-native RPCs: TextGenerate / Generate ---
|
||||
|
||||
type TextGenerateStream = StreamResult<proto::TextGenerateResponse>;
|
||||
|
||||
async fn text_generate(
|
||||
&self,
|
||||
request: Request<proto::TextGenerateRequest>,
|
||||
) -> Result<Response<Self::TextGenerateStream>, Status> {
|
||||
) -> Result<Response<StreamResult<proto::TextGenerateResponse>>, Status> {
|
||||
let req = request.into_inner();
|
||||
let rid = req
|
||||
.rid
|
||||
@@ -369,12 +386,10 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl {
|
||||
Ok(Response::new(Box::pin(stream)))
|
||||
}
|
||||
|
||||
type GenerateStream = StreamResult<proto::GenerateResponse>;
|
||||
|
||||
async fn generate(
|
||||
&self,
|
||||
request: Request<proto::GenerateRequest>,
|
||||
) -> Result<Response<Self::GenerateStream>, Status> {
|
||||
) -> Result<Response<StreamResult<proto::GenerateResponse>>, Status> {
|
||||
let req = request.into_inner();
|
||||
let rid = req
|
||||
.rid
|
||||
@@ -645,12 +660,10 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl {
|
||||
Ok(Response::new(proto::HealthCheckResponse { healthy }))
|
||||
}
|
||||
|
||||
type WatchEngineStateStream = StreamResult<proto::EngineStateSnapshot>;
|
||||
|
||||
async fn watch_engine_state(
|
||||
&self,
|
||||
_request: Request<proto::WatchEngineStateRequest>,
|
||||
) -> Result<Response<Self::WatchEngineStateStream>, Status> {
|
||||
) -> Result<Response<StreamResult<proto::EngineStateSnapshot>>, Status> {
|
||||
let mut receiver = self.engine_state.subscribe();
|
||||
let mut shutdown = self.stream_shutdown.clone();
|
||||
let stream = async_stream::stream! {
|
||||
@@ -831,22 +844,18 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl {
|
||||
|
||||
// --- OpenAI-compatible RPCs (JSON pass-through) ---
|
||||
|
||||
type ChatCompleteStream = StreamResult<proto::OpenAiStreamChunk>;
|
||||
|
||||
async fn chat_complete(
|
||||
&self,
|
||||
request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<Self::ChatCompleteStream>, Status> {
|
||||
) -> Result<Response<StreamResult<proto::OpenAiStreamChunk>>, Status> {
|
||||
self.openai_streaming_rpc(request, "submit_openai_chat")
|
||||
.await
|
||||
}
|
||||
|
||||
type CompleteStream = StreamResult<proto::OpenAiStreamChunk>;
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<Self::CompleteStream>, Status> {
|
||||
) -> Result<Response<StreamResult<proto::OpenAiStreamChunk>>, Status> {
|
||||
self.openai_streaming_rpc(request, "submit_openai_complete")
|
||||
.await
|
||||
}
|
||||
@@ -1079,6 +1088,14 @@ async fn recv_json_response(
|
||||
}
|
||||
}
|
||||
|
||||
pub enum ServerMode {
|
||||
Inference {
|
||||
bridge: Arc<PyBridge>,
|
||||
response_timeout: Duration,
|
||||
},
|
||||
Metadata(MetadataService),
|
||||
}
|
||||
|
||||
/// Start the Tonic gRPC server on the given address.
|
||||
//
|
||||
// TODO(grpc-auth): this listener is currently unauthenticated. Before exposing
|
||||
@@ -1086,12 +1103,18 @@ async fn recv_json_response(
|
||||
// checks the HTTP server applies (see issue tracking gRPC auth parity).
|
||||
pub async fn run_grpc_server(
|
||||
listener: std::net::TcpListener,
|
||||
bridge: Arc<PyBridge>,
|
||||
mode: ServerMode,
|
||||
shutdown: Arc<Notify>,
|
||||
response_timeout: Duration,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let addr = listener.local_addr()?;
|
||||
let listener = tokio::net::TcpListener::from_std(listener)?;
|
||||
let (bridge, response_timeout) = match mode {
|
||||
ServerMode::Metadata(service) => {
|
||||
return serve_grpc(listener, service, shutdown, None).await;
|
||||
}
|
||||
ServerMode::Inference {
|
||||
bridge,
|
||||
response_timeout,
|
||||
} => (bridge, response_timeout),
|
||||
};
|
||||
let (state_changed_tx, mut state_changed_rx) = tokio::sync::mpsc::channel(1);
|
||||
bridge.set_engine_state_changed_callback(state_changed_tx)?;
|
||||
let engine_state = EngineStatePublisher::new(bridge.clone()).await?;
|
||||
@@ -1112,6 +1135,19 @@ pub async fn run_grpc_server(
|
||||
}
|
||||
});
|
||||
|
||||
let result = serve_grpc(listener, service, shutdown, Some(stream_shutdown_tx)).await;
|
||||
monitor.abort();
|
||||
result
|
||||
}
|
||||
|
||||
async fn serve_grpc(
|
||||
listener: std::net::TcpListener,
|
||||
service: impl proto::sglang_service_server::SglangService,
|
||||
shutdown: Arc<Notify>,
|
||||
stream_shutdown: Option<watch::Sender<bool>>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let addr = listener.local_addr()?;
|
||||
let listener = tokio::net::TcpListener::from_std(listener)?;
|
||||
let max_message_size = resolve_max_message_size();
|
||||
let svc = proto::sglang_service_server::SglangServiceServer::new(service)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
@@ -1123,11 +1159,12 @@ pub async fn run_grpc_server(
|
||||
.add_service(svc)
|
||||
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async move {
|
||||
shutdown.notified().await;
|
||||
stream_shutdown_tx.send_replace(true);
|
||||
if let Some(stream_shutdown) = stream_shutdown {
|
||||
stream_shutdown.send_replace(true);
|
||||
}
|
||||
tracing::info!("gRPC server shutting down");
|
||||
})
|
||||
.await;
|
||||
monitor.abort();
|
||||
result?;
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -6,6 +6,10 @@ use crate::bridge::TerminalError;
|
||||
use std::collections::HashMap;
|
||||
use tonic::Code;
|
||||
|
||||
// The metadata wire test starts a real server that reads SGLANG_TONIC_PAYLOAD.
|
||||
// Keep that read separate from the environment override test below.
|
||||
static PAYLOAD_ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
#[test]
|
||||
fn openai_status_code_uses_forwarded_status_when_present() {
|
||||
let meta_info = HashMap::from([(String::from("status_code"), String::from("429"))]);
|
||||
@@ -42,6 +46,7 @@ fn terminal_error_status_maps_abort_to_cancelled() {
|
||||
// serial test so they don't race each other under `cargo test`'s default parallelism.
|
||||
#[test]
|
||||
fn resolve_max_message_size_honors_env_var() {
|
||||
let _guard = PAYLOAD_ENV_LOCK.blocking_lock();
|
||||
const VAR: &str = "SGLANG_TONIC_PAYLOAD";
|
||||
|
||||
// Unset → default.
|
||||
@@ -73,3 +78,102 @@ fn resolve_max_message_size_honors_env_var() {
|
||||
std::env::remove_var(VAR);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn follower_metadata_server_exposes_only_server_info_and_shuts_down() {
|
||||
use sglang_grpc_types::sglang::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse};
|
||||
use tonic::codec::ProstCodec;
|
||||
|
||||
let _guard = PAYLOAD_ENV_LOCK.lock().await;
|
||||
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap();
|
||||
let server_info_json = serde_json::json!({
|
||||
"node_rank": 1,
|
||||
"nnodes": 2,
|
||||
"dp_size": 8,
|
||||
"kv_event_sources": [{
|
||||
"dp_rank": 4,
|
||||
"endpoint": "tcp://127.0.0.1:5557",
|
||||
"topic": "kv-events",
|
||||
"block_size": 16
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
// Exercise the metadata service without the Python-facing startup wrapper,
|
||||
// which requires libpython when linked into a standalone test executable.
|
||||
let shutdown = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
let server_shutdown = shutdown.clone();
|
||||
let service = super::MetadataService {
|
||||
server_info_json: server_info_json.clone(),
|
||||
};
|
||||
let handle = std::thread::spawn(move || {
|
||||
runtime
|
||||
.block_on(super::serve_grpc(listener, service, server_shutdown, None))
|
||||
.unwrap();
|
||||
});
|
||||
assert!(!handle.is_finished());
|
||||
|
||||
let channel = tonic::transport::Endpoint::from_shared(format!("http://{addr}"))
|
||||
.unwrap()
|
||||
.connect_timeout(std::time::Duration::from_secs(5))
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.connect()
|
||||
.await
|
||||
.unwrap();
|
||||
let mut client = tonic::client::Grpc::new(channel);
|
||||
client.ready().await.unwrap();
|
||||
let response: tonic::Response<GetServerInfoResponse> = client
|
||||
.unary(
|
||||
tonic::Request::new(GetServerInfoRequest {}),
|
||||
tonic::codegen::http::uri::PathAndQuery::from_static(
|
||||
"/sglang.runtime.v1.SglangService/GetServerInfo",
|
||||
),
|
||||
ProstCodec::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.into_inner().json_info, server_info_json);
|
||||
|
||||
// These cover generation (including streaming), model queries, health,
|
||||
// and controls. The follower must never advertise an inference frontend.
|
||||
for method in [
|
||||
"Generate",
|
||||
"ChatComplete",
|
||||
"GetModelInfo",
|
||||
"HealthCheck",
|
||||
"FlushCache",
|
||||
"Abort",
|
||||
] {
|
||||
client.ready().await.unwrap();
|
||||
let result: Result<tonic::Response<GetServerInfoResponse>, _> = client
|
||||
.unary(
|
||||
tonic::Request::new(GetServerInfoRequest {}),
|
||||
format!("/sglang.runtime.v1.SglangService/{method}")
|
||||
.parse()
|
||||
.unwrap(),
|
||||
ProstCodec::default(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.unwrap_err().code(), Code::Unimplemented, "{method}");
|
||||
}
|
||||
drop(client);
|
||||
// Keep the client's runtime driving connection teardown while joining the
|
||||
// server thread during graceful shutdown.
|
||||
shutdown.notify_one();
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
tokio::task::spawn_blocking(move || {
|
||||
handle.join().unwrap();
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(std::net::TcpStream::connect(addr).is_err());
|
||||
}
|
||||
|
||||
@@ -87,12 +87,10 @@ fn unimplemented_rpc(name: &'static str) -> Status {
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl SglangService for GrpcService {
|
||||
type TextGenerateStream = ResponseStream<proto::TextGenerateResponse>;
|
||||
|
||||
async fn text_generate(
|
||||
&self,
|
||||
request: Request<proto::TextGenerateRequest>,
|
||||
) -> Result<Response<Self::TextGenerateStream>, Status> {
|
||||
) -> Result<Response<ResponseStream<proto::TextGenerateResponse>>, Status> {
|
||||
let request = convert::text_generate(
|
||||
request.into_inner(),
|
||||
self.config.preferred_sampling_params.as_ref(),
|
||||
@@ -110,12 +108,10 @@ impl SglangService for GrpcService {
|
||||
)))
|
||||
}
|
||||
|
||||
type GenerateStream = ResponseStream<proto::GenerateResponse>;
|
||||
|
||||
async fn generate(
|
||||
&self,
|
||||
request: Request<proto::GenerateRequest>,
|
||||
) -> Result<Response<Self::GenerateStream>, Status> {
|
||||
) -> Result<Response<ResponseStream<proto::GenerateResponse>>, Status> {
|
||||
let request = convert::generate(
|
||||
request.into_inner(),
|
||||
self.config.preferred_sampling_params.as_ref(),
|
||||
@@ -228,12 +224,10 @@ impl SglangService for GrpcService {
|
||||
Err(unimplemented_rpc("pause_generation"))
|
||||
}
|
||||
|
||||
type WatchEngineStateStream = ResponseStream<proto::EngineStateSnapshot>;
|
||||
|
||||
async fn watch_engine_state(
|
||||
&self,
|
||||
_request: Request<proto::WatchEngineStateRequest>,
|
||||
) -> Result<Response<Self::WatchEngineStateStream>, Status> {
|
||||
) -> Result<Response<ResponseStream<proto::EngineStateSnapshot>>, Status> {
|
||||
Err(unimplemented_rpc("watch_engine_state"))
|
||||
}
|
||||
|
||||
@@ -244,21 +238,17 @@ impl SglangService for GrpcService {
|
||||
Err(unimplemented_rpc("continue_generation"))
|
||||
}
|
||||
|
||||
type ChatCompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
|
||||
|
||||
async fn chat_complete(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<Self::ChatCompleteStream>, Status> {
|
||||
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
|
||||
Err(unimplemented_rpc("chat_complete"))
|
||||
}
|
||||
|
||||
type CompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<Self::CompleteStream>, Status> {
|
||||
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
|
||||
Err(unimplemented_rpc("complete"))
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,12 @@ the router can subscribe per replica (the `dp_size` it reads from
|
||||
`/server_info`).
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
@@ -16,7 +21,9 @@ from sglang.srt.disaggregation.kv_events import (
|
||||
AllBlocksCleared,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
EventPublisherFactory,
|
||||
KVEventBatch,
|
||||
NullEventPublisher,
|
||||
StorageMedium,
|
||||
ZmqEventPublisher,
|
||||
resolve_load_pub_range,
|
||||
@@ -31,6 +38,101 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestLocalKvEventSource(CustomTestCase):
|
||||
def _publisher(self, attn_dp_rank=0, **config):
|
||||
publisher = EventPublisherFactory.create(
|
||||
json.dumps({"publisher": "zmq", **config}), attn_dp_rank=attn_dp_rank
|
||||
)
|
||||
atexit.unregister(publisher.shutdown)
|
||||
self.addCleanup(publisher.shutdown)
|
||||
return publisher
|
||||
|
||||
def test_advertised_source_delivers_events(self):
|
||||
for host, bind, topic in (("*", None, "kv"), ("127.0.0.1", True, "")):
|
||||
with self.subTest(host=host):
|
||||
with zmq.Context.instance().socket(zmq.PUB) as probe:
|
||||
port = probe.bind_to_random_port("tcp://127.0.0.1")
|
||||
publisher = self._publisher(
|
||||
attn_dp_rank=4,
|
||||
endpoint=f"tcp://{host}:{port - 4}",
|
||||
bind=bind,
|
||||
topic=topic,
|
||||
)
|
||||
source = publisher.describe_local_source(64)
|
||||
self.assertEqual(
|
||||
source,
|
||||
dict(
|
||||
dp_rank=4,
|
||||
endpoint=f"tcp://127.0.0.1:{port}",
|
||||
topic=topic,
|
||||
block_size=64,
|
||||
),
|
||||
)
|
||||
self._assert_event_received(publisher, source)
|
||||
|
||||
def _assert_event_received(self, publisher, source):
|
||||
with zmq.Context() as context, context.socket(zmq.SUB) as subscriber:
|
||||
subscriber.setsockopt_string(zmq.SUBSCRIBE, source["topic"])
|
||||
subscriber.connect(source["endpoint"])
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
publisher.publish(KVEventBatch(ts=1.0, events=[AllBlocksCleared()]))
|
||||
if subscriber.poll(100):
|
||||
topic, _, payload = subscriber.recv_multipart()
|
||||
self.assertEqual(topic, source["topic"].encode())
|
||||
batch = msgspec.msgpack.decode(payload, type=KVEventBatch)
|
||||
self.assertEqual(batch.attn_dp_rank, source["dp_rank"])
|
||||
self.assertEqual(batch.events, [AllBlocksCleared()])
|
||||
break
|
||||
else:
|
||||
self.fail("No event received from the advertised local source")
|
||||
|
||||
def test_ipc_is_advertised_only_when_bound(self):
|
||||
directory = tempfile.TemporaryDirectory(prefix="kv-", dir="/tmp")
|
||||
self.addCleanup(directory.cleanup)
|
||||
endpoint = f"ipc://{directory.name}/events"
|
||||
for bind in (None, False):
|
||||
with self.subTest(bind=bind):
|
||||
publisher = self._publisher(endpoint=endpoint, bind=bind)
|
||||
source = publisher.describe_local_source(64)
|
||||
if bind is False:
|
||||
self.assertIsNone(source)
|
||||
else:
|
||||
self.assertEqual(source["endpoint"], endpoint)
|
||||
|
||||
def test_ephemeral_bind_and_replay_report_resolved_ports(self):
|
||||
publisher = self._publisher(
|
||||
attn_dp_rank=0,
|
||||
endpoint="tcp://0.0.0.0:0",
|
||||
replay_endpoint="tcp://*:0",
|
||||
)
|
||||
source = publisher.describe_local_source(64)
|
||||
self.assertEqual(
|
||||
source["endpoint"],
|
||||
publisher._pub.getsockopt_string(zmq.LAST_ENDPOINT).replace(
|
||||
"0.0.0.0", "127.0.0.1"
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
source["replay_endpoint"],
|
||||
publisher._replay.getsockopt_string(zmq.LAST_ENDPOINT).replace(
|
||||
"0.0.0.0", "127.0.0.1"
|
||||
),
|
||||
)
|
||||
self.assertNotEqual(source["endpoint"], source["replay_endpoint"])
|
||||
self.assertFalse(source["endpoint"].endswith(":0"))
|
||||
|
||||
def test_non_subscribable_publishers_are_not_advertised(self):
|
||||
for endpoint in (
|
||||
"tcp://127.0.0.1:5557", # Connect-style PUB, not a listening source.
|
||||
f"inproc://kv-source-{uuid.uuid4().hex}", # Same-process only.
|
||||
):
|
||||
with self.subTest(endpoint=endpoint):
|
||||
publisher = self._publisher(attn_dp_rank=0, endpoint=endpoint)
|
||||
self.assertIsNone(publisher.describe_local_source(64))
|
||||
self.assertIsNone(NullEventPublisher().describe_local_source(64))
|
||||
|
||||
|
||||
class TestResolveLoadPubRange(CustomTestCase):
|
||||
"""The single source of truth both the bind and /server_info route through."""
|
||||
|
||||
|
||||
@@ -24,16 +24,19 @@ import asyncio
|
||||
import json
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import msgspec
|
||||
import msgspec.structs
|
||||
|
||||
from sglang.srt.arg_groups.validation_hook import check_load_publish_args
|
||||
from sglang.srt.entrypoints import http_server
|
||||
from sglang.srt.entrypoints.grpc_bridge import RuntimeHandle
|
||||
from sglang.srt.lora.lora_registry import LoRARef
|
||||
from sglang.srt.managers.tokenizer_manager import TokenizerManager
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils import common
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -147,6 +150,46 @@ def _call_server_info_with(
|
||||
reset_context()
|
||||
|
||||
|
||||
class TestServerInfoTransportParity(CustomTestCase):
|
||||
def test_http_and_both_grpc_roles_share_startup_metadata(self):
|
||||
for config in (None, '{"publisher": "zmq", "endpoint": "tcp://*:5557"}'):
|
||||
with self.subTest(kv_events_config=config):
|
||||
args = ServerArgs(
|
||||
model_path="dummy", kv_events_config=config, page_size=64
|
||||
)
|
||||
http_info = _call_server_info_with(args)
|
||||
self.assertEqual(http_info["frontend"], "python")
|
||||
for field in ("startup_time", "internal_states", "version", "frontend"):
|
||||
self.assertIn(field, http_info)
|
||||
http_info.pop(field)
|
||||
|
||||
bridge = RuntimeHandle.__new__(RuntimeHandle)
|
||||
bridge.tokenizer_manager = SimpleNamespace(server_args=args)
|
||||
bridge.scheduler_info = {"max_req_input_len": 1024}
|
||||
leader_json = bridge.get_server_info()
|
||||
self.assertEqual(
|
||||
json.loads(leader_json), json.loads(json.dumps(http_info))
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
common,
|
||||
"get_serving",
|
||||
return_value=SimpleNamespace(
|
||||
host="127.0.0.1",
|
||||
grpc_port=50051,
|
||||
smg_grpc_mode=False,
|
||||
grpc_mode=False,
|
||||
),
|
||||
),
|
||||
patch("sglang.srt.rust_extensions.load_rust_extension") as load,
|
||||
):
|
||||
common.start_follower_grpc_server(args, bridge.scheduler_info)
|
||||
load.return_value.start_metadata_server.assert_called_once_with(
|
||||
host="127.0.0.1", port=50051, server_info_json=leader_json
|
||||
)
|
||||
|
||||
|
||||
class TestServerInfoKvEventsField(CustomTestCase):
|
||||
"""The new `kv_events` field is wired correctly across the full
|
||||
`_build_kv_events_block` input matrix.
|
||||
|
||||
@@ -11,10 +11,12 @@ is exercised as the real method, no mock. The refresh-throttle tests inject
|
||||
`load_snapshot_reader` and `_last_refresh_time` on top of those.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec.structs
|
||||
|
||||
@@ -29,6 +31,7 @@ from sglang.srt.managers.data_parallel_controller import (
|
||||
LoadBalanceMethod,
|
||||
)
|
||||
from sglang.srt.managers.load_snapshot import LoadSnapshot
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
|
||||
|
||||
@@ -64,6 +67,97 @@ def _req(routed_dp_rank=None, bootstrap_room=None, input_ids=None):
|
||||
)
|
||||
|
||||
|
||||
class TestLocalKvEventSources(CustomTestCase):
|
||||
@staticmethod
|
||||
def _source(rank, block_size=64):
|
||||
return {
|
||||
"dp_rank": rank,
|
||||
"endpoint": f"tcp://127.0.0.1:{5557 + rank}",
|
||||
"topic": "kv",
|
||||
"block_size": block_size,
|
||||
}
|
||||
|
||||
def test_scheduler_reports_logical_block_size_or_no_source(self):
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.managers.scheduler_components.kv_events_publisher import (
|
||||
SchedulerKvEventsPublisher,
|
||||
)
|
||||
|
||||
component = SchedulerKvEventsPublisher.__new__(SchedulerKvEventsPublisher)
|
||||
scheduler = Scheduler.__new__(Scheduler)
|
||||
scheduler.max_total_num_tokens = 64
|
||||
scheduler.max_req_input_len = 32
|
||||
scheduler.startup_time = {}
|
||||
scheduler.page_size = 64
|
||||
scheduler.kv_events_publisher = component
|
||||
publisher = MagicMock()
|
||||
publisher.describe_local_source.return_value = self._source(4, 256)
|
||||
for owned_publisher, expected in (
|
||||
(publisher, [self._source(4, 256)]),
|
||||
(None, []),
|
||||
):
|
||||
with (
|
||||
self.subTest(has_publisher=owned_publisher is not None),
|
||||
get_context().override_server_args(grpc_port=50051, dcp_size=4),
|
||||
):
|
||||
component.kv_event_publisher = owned_publisher
|
||||
self.assertEqual(
|
||||
scheduler.get_init_info()["kv_event_sources"], expected
|
||||
)
|
||||
publisher.describe_local_source.assert_called_once_with(256)
|
||||
|
||||
def test_controller_collects_all_scheduler_sources(self):
|
||||
from sglang.srt.managers import data_parallel_controller as module
|
||||
|
||||
controller = DataParallelController.__new__(DataParallelController)
|
||||
controller.env_lock = threading.Lock()
|
||||
controller.scheduler_procs = []
|
||||
controller.local_kv_event_sources = []
|
||||
controller.run_scheduler_process_func = MagicMock()
|
||||
# TP8/DP4 over two nodes: this follower's four schedulers own ranks 2,3.
|
||||
readers = [
|
||||
MagicMock(
|
||||
recv=MagicMock(
|
||||
return_value={
|
||||
"max_total_num_tokens": 64,
|
||||
"max_req_input_len": 32,
|
||||
"kv_event_sources": sources,
|
||||
}
|
||||
)
|
||||
)
|
||||
for sources in ([self._source(2)], [], [self._source(3)], [])
|
||||
]
|
||||
with (
|
||||
get_context().override_server_args(
|
||||
tp_size=8,
|
||||
dp_size=4,
|
||||
enable_dp_attention=True,
|
||||
nnodes=2,
|
||||
node_rank=1,
|
||||
pp_size=1,
|
||||
attn_cp_size=1,
|
||||
),
|
||||
patch.object(module.mp, "Process"),
|
||||
patch.object(
|
||||
module.mp,
|
||||
"Pipe",
|
||||
side_effect=[(reader, MagicMock()) for reader in readers],
|
||||
),
|
||||
patch.object(module.PortArgs, "init_new", return_value=MagicMock()),
|
||||
patch.object(module.TorchMemorySaverAdapter, "create"),
|
||||
patch.object(module.numa_utils, "configure_subprocess"),
|
||||
patch.object(
|
||||
module, "maybe_reindex_device_id", return_value=nullcontext(0)
|
||||
),
|
||||
):
|
||||
controller.launch_tensor_parallel_group(
|
||||
MagicMock(), MagicMock(), 0, None, [7000, 7001, 7002, 7003]
|
||||
)
|
||||
self.assertEqual(
|
||||
controller.local_kv_event_sources, [self._source(2), self._source(3)]
|
||||
)
|
||||
|
||||
|
||||
class TestDPBudgetUpdateBudget(CustomTestCase):
|
||||
def test_maps_running_plus_waiting_to_total_requests(self):
|
||||
budget = DPBudget(num_dp_ranks=2)
|
||||
|
||||
Reference in New Issue
Block a user