feat(grpc): expose follower metadata and node-local KV sources (#39659)
Signed-off-by: jain-ria <riajain@NVIDIA.com> Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
This commit is contained in:
@@ -80,7 +80,8 @@ class Serving(msgspec.Struct):
|
||||
Optional[int],
|
||||
"Port for the native gRPC server, started alongside HTTP. Setting this "
|
||||
"(or SGLANG_GRPC_PORT) enables the native gRPC server; it is off by "
|
||||
"default. In legacy --smg-grpc-mode this is the SMG server port and "
|
||||
"default. Follower nodes expose GetServerInfo only. "
|
||||
"In legacy --smg-grpc-mode this is the SMG server port and "
|
||||
"defaults to --port + 10000.",
|
||||
] = None
|
||||
grpc_response_timeout_secs: A[
|
||||
|
||||
@@ -20,6 +20,7 @@ KV caching events
|
||||
import atexit
|
||||
import enum
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
@@ -329,6 +330,10 @@ class EventPublisher(ABC):
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the publisher."""
|
||||
|
||||
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
|
||||
"""Describe a source accessible to a separate local process, if any."""
|
||||
return None
|
||||
|
||||
|
||||
class NullEventPublisher(EventPublisher):
|
||||
"""No-op implementation (default when disabled)."""
|
||||
@@ -349,7 +354,7 @@ class ZmqEventPublisher(EventPublisher):
|
||||
----------
|
||||
endpoint:
|
||||
PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to
|
||||
connect.
|
||||
connect, unless overridden by ``bind``.
|
||||
replay_endpoint:
|
||||
Optional ROUTER address for replay requests. When given, subscribers can
|
||||
request missed batches by sending the starting sequence number as an
|
||||
@@ -362,6 +367,9 @@ class ZmqEventPublisher(EventPublisher):
|
||||
Maximum number of events to buffer in memory.
|
||||
topic:
|
||||
Topic to publish events to.
|
||||
bind:
|
||||
Whether to bind the PUB socket. When unset, preserve the endpoint-based
|
||||
heuristic: bind wildcard, IPC and inproc addresses; connect otherwise.
|
||||
"""
|
||||
|
||||
SHUTDOWN_TIMEOUT: float = 1.0
|
||||
@@ -376,6 +384,7 @@ class ZmqEventPublisher(EventPublisher):
|
||||
hwm: int = 100_000,
|
||||
max_queue_size: int = 100_000,
|
||||
topic: str = "",
|
||||
bind: Optional[bool] = None,
|
||||
) -> None:
|
||||
# Storage
|
||||
self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size)
|
||||
@@ -384,9 +393,12 @@ class ZmqEventPublisher(EventPublisher):
|
||||
# ZMQ sockets
|
||||
self._ctx = zmq.Context.instance()
|
||||
self._pub: Optional[zmq.Socket] = None
|
||||
self._local_pub_endpoint: Optional[str] = None
|
||||
self._local_replay_endpoint: Optional[str] = None
|
||||
self._replay: Optional[zmq.Socket] = None
|
||||
self._dp_rank = attn_dp_rank
|
||||
self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank)
|
||||
self._bind = bind
|
||||
self._replay_endpoint = self.offset_endpoint_port(
|
||||
replay_endpoint, self._dp_rank
|
||||
)
|
||||
@@ -415,6 +427,43 @@ class ZmqEventPublisher(EventPublisher):
|
||||
events.attn_dp_rank = self._dp_rank
|
||||
self._event_queue.put(events)
|
||||
|
||||
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
|
||||
"""Report the bound socket, not a port reconstructed from global DP size.
|
||||
|
||||
Connect-style publishers and inproc sockets have no subscribable local
|
||||
endpoint. They keep working as before, but are not advertised to external
|
||||
processes. Wildcard binds are reachable on loopback from this node.
|
||||
"""
|
||||
|
||||
def local_endpoint(endpoint: Optional[str]) -> Optional[str]:
|
||||
if endpoint is None:
|
||||
return None
|
||||
if endpoint.startswith("ipc://"):
|
||||
return endpoint
|
||||
if parse_advertisable_tcp(endpoint) is None:
|
||||
return None
|
||||
address = NetworkAddress.parse(endpoint[len("tcp://") :])
|
||||
host = address.host
|
||||
if host in ("*", "0.0.0.0"):
|
||||
host = "127.0.0.1"
|
||||
elif host == "::":
|
||||
host = "::1"
|
||||
return NetworkAddress(host, address.port).to_tcp()
|
||||
|
||||
endpoint = local_endpoint(self._local_pub_endpoint)
|
||||
if endpoint is None:
|
||||
return None
|
||||
source = {
|
||||
"dp_rank": self._dp_rank,
|
||||
"endpoint": endpoint,
|
||||
"topic": self._topic_bytes.decode("utf-8"),
|
||||
"block_size": block_size,
|
||||
}
|
||||
replay_endpoint = local_endpoint(self._local_replay_endpoint)
|
||||
if replay_endpoint is not None:
|
||||
source["replay_endpoint"] = replay_endpoint
|
||||
return source
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Stop the publisher thread and clean up resources."""
|
||||
self._running = False
|
||||
@@ -453,23 +502,27 @@ class ZmqEventPublisher(EventPublisher):
|
||||
if self._pub is None:
|
||||
self._pub = self._ctx.socket(zmq.PUB)
|
||||
self._pub.set_hwm(self._hwm)
|
||||
# Heuristic: bind if wildcard / * present, else connect.
|
||||
# Default heuristic: bind if wildcard / * present, else connect.
|
||||
# bind stable, connect volatile convention.
|
||||
# ``0.0.0.0`` is the IPv4 bind-all wildcard alongside ``*``
|
||||
# and ``::``; ``/server_info`` advertises it as a wildcard,
|
||||
# so the publisher must bind it for the advertised endpoint
|
||||
# to actually be listening.
|
||||
if (
|
||||
"*" in self._endpoint
|
||||
or "::" in self._endpoint
|
||||
or "0.0.0.0" in self._endpoint
|
||||
or self._endpoint.startswith("ipc://")
|
||||
or self._endpoint.startswith("inproc://")
|
||||
):
|
||||
should_bind = self._bind
|
||||
if should_bind is None:
|
||||
should_bind = (
|
||||
"*" in self._endpoint
|
||||
or "::" in self._endpoint
|
||||
or "0.0.0.0" in self._endpoint
|
||||
or self._endpoint.startswith("ipc://")
|
||||
or self._endpoint.startswith("inproc://")
|
||||
)
|
||||
if should_bind:
|
||||
logger.debug(
|
||||
f"ZmqEventPublisher socket publisher_endpoint bind to {self._endpoint}"
|
||||
)
|
||||
self._pub.bind(self._endpoint)
|
||||
self._local_pub_endpoint = self._bound_endpoint(self._pub)
|
||||
else:
|
||||
self._pub.connect(self._endpoint)
|
||||
|
||||
@@ -483,6 +536,19 @@ class ZmqEventPublisher(EventPublisher):
|
||||
f"ZmqEventPublisher socket replay_endpoint bind to {self._replay_endpoint}"
|
||||
)
|
||||
self._replay.bind(self._replay_endpoint)
|
||||
self._local_replay_endpoint = self._bound_endpoint(self._replay)
|
||||
|
||||
@staticmethod
|
||||
def _bound_endpoint(socket: zmq.Socket) -> str:
|
||||
endpoint = socket.getsockopt_string(zmq.LAST_ENDPOINT)
|
||||
if endpoint.startswith("ipc://"):
|
||||
path = endpoint[len("ipc://") :]
|
||||
# Capture the publisher's working directory at bind time so a
|
||||
# separate process can use the address. Abstract IPC names are
|
||||
# independent of the filesystem and must remain unchanged.
|
||||
if not path.startswith("@"):
|
||||
endpoint = f"ipc://{os.path.abspath(path)}"
|
||||
return endpoint
|
||||
|
||||
def _publisher_thread(self) -> None:
|
||||
"""Background thread that processes the event queue."""
|
||||
@@ -589,6 +655,11 @@ class KVEventsConfig(BaseModel):
|
||||
"""The zmq endpoint to use for publishing kv events.
|
||||
"""
|
||||
|
||||
bind: Optional[bool] = None
|
||||
"""Bind (true) or connect (false) the PUB socket. When unset, infer from
|
||||
the endpoint as before. Use true to bind a concrete address such as loopback.
|
||||
"""
|
||||
|
||||
replay_endpoint: Optional[str] = None
|
||||
"""The zmq endpoint to use for replaying kv events.
|
||||
"""
|
||||
|
||||
@@ -140,6 +140,7 @@ from sglang.srt.utils import (
|
||||
numa_utils,
|
||||
set_prometheus_multiproc_dir,
|
||||
set_ulimit,
|
||||
start_follower_grpc_server,
|
||||
)
|
||||
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
|
||||
from sglang.srt.utils.network import (
|
||||
@@ -173,6 +174,12 @@ class SchedulerInitResult:
|
||||
wait_for_ready: Callable[[], None] = lambda: None
|
||||
block_until_scheduler_exits: Callable[[], None] = lambda: None
|
||||
engine_info_bootstrap_server: Optional[Any] = None
|
||||
grpc_server: Optional[Any] = None
|
||||
|
||||
def stop_grpc_server(self) -> None:
|
||||
if self.grpc_server is not None:
|
||||
self.grpc_server.shutdown()
|
||||
self.grpc_server = None
|
||||
|
||||
|
||||
def init_tokenizer_manager(
|
||||
@@ -951,6 +958,17 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
|
||||
def wait_for_ready():
|
||||
infos = _wait_for_scheduler_ready(scheduler_pipe_readers, scheduler_procs)
|
||||
if any("kv_event_sources" in info for info in infos):
|
||||
# Both gRPC entrypoints consume the first scheduler info. Keep
|
||||
# the sources from every local scheduler, not just the first.
|
||||
infos[0]["kv_event_sources"] = sorted(
|
||||
(
|
||||
source
|
||||
for info in infos
|
||||
for source in info.get("kv_event_sources", [])
|
||||
),
|
||||
key=lambda source: source["dp_rank"],
|
||||
)
|
||||
scheduler_infos.extend(infos)
|
||||
if use_dp_controller:
|
||||
for info in infos:
|
||||
@@ -1173,6 +1191,18 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
# Non-zero-rank nodes do not run tokenizer processes.
|
||||
scheduler_init_result.wait_for_ready()
|
||||
|
||||
try:
|
||||
scheduler_init_result.grpc_server = start_follower_grpc_server(
|
||||
server_args, scheduler_init_result.scheduler_infos[0]
|
||||
)
|
||||
except BaseException:
|
||||
# Engine.__init__ has not received these handles yet. Do not
|
||||
# leave GPU workers behind if binding the metadata port fails.
|
||||
for proc in scheduler_procs or []:
|
||||
kill_process_tree(proc.pid, wait_timeout=60)
|
||||
cls._terminate_weight_cache_daemons(weight_cache_daemon_procs)
|
||||
raise
|
||||
|
||||
if os.getenv("SGLANG_BLOCK_NONZERO_RANK_CHILDREN") == "0":
|
||||
# When using `Engine` as a Python API, we don't want to block here.
|
||||
return (
|
||||
@@ -1202,14 +1232,16 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
rust_server_owns_base_port = (
|
||||
envs.SGLANG_RUST_SERVER.get() and node_hosts_rust_server()
|
||||
)
|
||||
if not rust_server_owns_base_port:
|
||||
launch_dummy_health_check_server(
|
||||
get_serving().host,
|
||||
get_serving().port,
|
||||
get_observability().enable_metrics,
|
||||
)
|
||||
|
||||
scheduler_init_result.block_until_scheduler_exits()
|
||||
try:
|
||||
if not rust_server_owns_base_port:
|
||||
launch_dummy_health_check_server(
|
||||
get_serving().host,
|
||||
get_serving().port,
|
||||
get_observability().enable_metrics,
|
||||
)
|
||||
scheduler_init_result.block_until_scheduler_exits()
|
||||
finally:
|
||||
scheduler_init_result.stop_grpc_server()
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
@@ -1384,6 +1416,9 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
pass
|
||||
self._multi_tokenizer_shm = None
|
||||
try:
|
||||
scheduler_init_result = getattr(self, "_scheduler_init_result", None)
|
||||
if scheduler_init_result is not None:
|
||||
scheduler_init_result.stop_grpc_server()
|
||||
if (
|
||||
self.tokenizer_manager is not None
|
||||
and self.tokenizer_manager._subprocess_watchdog is not None
|
||||
@@ -1789,7 +1824,6 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
|
||||
|
||||
def _set_envs_and_config(server_args: ServerArgs):
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
# Set global environments
|
||||
# MNNVL fabric (GB200/GB300) multi-node: cross-node NVLink needs NCCL's
|
||||
|
||||
@@ -16,11 +16,8 @@ from pydantic import ValidationError
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
|
||||
from sglang.srt.runtime_context import (
|
||||
describe_kv_events_publisher,
|
||||
get_lora,
|
||||
get_serving,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_lora, get_serving
|
||||
from sglang.srt.utils.common import build_server_info
|
||||
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -238,9 +235,7 @@ class RuntimeHandle:
|
||||
return self._openai_serving_classes
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
from sglang.srt.entrypoints.openai.serving_classify import (
|
||||
OpenAIServingClassify,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
|
||||
from sglang.srt.entrypoints.openai.serving_completions import (
|
||||
OpenAIServingCompletion,
|
||||
)
|
||||
@@ -430,17 +425,14 @@ class RuntimeHandle:
|
||||
return json.dumps(result, default=str)
|
||||
|
||||
def get_server_info(self) -> str:
|
||||
result: Dict[str, Any] = self.tokenizer_manager.server_args.resolved_dict()
|
||||
# `resolved_dict` answers with what resolution decided; the launch
|
||||
# command answers with what was asked for, and the two are not
|
||||
# derivable from each other. The HTTP and in-process readbacks both
|
||||
# carry it, so this one does too.
|
||||
result["launch_command"] = self.tokenizer_manager.server_args.launch_command
|
||||
result.update(self.scheduler_info)
|
||||
result["kv_events"] = describe_kv_events_publisher(
|
||||
self.tokenizer_manager.server_args
|
||||
return json.dumps(
|
||||
msgspec_to_builtins(
|
||||
build_server_info(
|
||||
self.tokenizer_manager.server_args, self.scheduler_info
|
||||
)
|
||||
),
|
||||
default=str,
|
||||
)
|
||||
return json.dumps(msgspec_to_builtins(result), default=str)
|
||||
|
||||
def health_check(self) -> bool:
|
||||
from sglang.srt.managers.tokenizer_manager import ServerStatus
|
||||
|
||||
@@ -177,6 +177,7 @@ from sglang.srt.server_args import PortArgs, ServerArgs
|
||||
from sglang.srt.utils import (
|
||||
add_prometheus_middleware,
|
||||
add_prometheus_track_response_middleware,
|
||||
build_server_info,
|
||||
delete_directory,
|
||||
get_bool_env_var,
|
||||
is_mps,
|
||||
@@ -502,7 +503,6 @@ app.include_router(v1_loads_router)
|
||||
from sglang.srt.arg_groups.serving_hook import ssl_verify_of
|
||||
from sglang.srt.entrypoints.elastic_ep import router as elastic_ep_router
|
||||
from sglang.srt.runtime_context import (
|
||||
describe_kv_events_publisher,
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_lora,
|
||||
@@ -859,17 +859,11 @@ async def server_info():
|
||||
|
||||
return msgspec_to_builtins(
|
||||
{
|
||||
**server_args.resolved_dict(),
|
||||
"launch_command": server_args.launch_command,
|
||||
**_global_state.scheduler_info,
|
||||
**build_server_info(server_args, _global_state.scheduler_info),
|
||||
"startup_time": _global_state.tokenizer_manager.startup_time,
|
||||
"internal_states": internal_states,
|
||||
"version": __version__,
|
||||
"frontend": "python",
|
||||
# Structured KV-event publisher descriptor for KV-aware routers.
|
||||
# `None` when publishing is disabled or misconfigured; see
|
||||
# `runtime_context.describe_kv_events_publisher` for the contract.
|
||||
"kv_events": describe_kv_events_publisher(server_args),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -200,6 +200,7 @@ class DataParallelController:
|
||||
|
||||
# Launch data parallel workers
|
||||
self.scheduler_procs = []
|
||||
self.local_kv_event_sources = []
|
||||
self.workers: list[zmq.Socket | None] = [None] * self.max_dp_size
|
||||
self.status: list[bool] = list(self.dp_active)
|
||||
self._active_workers: list[int] = list(range(self.launch_dp_size))
|
||||
@@ -723,6 +724,15 @@ class DataParallelController:
|
||||
for i in range(len(scheduler_pipe_readers)):
|
||||
scheduler_info.append(scheduler_pipe_readers[i].recv())
|
||||
|
||||
# Pure-DP TP groups launch concurrently. Their ready replies contain
|
||||
# only publishers owned by schedulers on this node.
|
||||
with self.env_lock:
|
||||
self.local_kv_event_sources.extend(
|
||||
source
|
||||
for info in scheduler_info
|
||||
for source in info.get("kv_event_sources", [])
|
||||
)
|
||||
|
||||
self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"]
|
||||
self.max_req_input_len = scheduler_info[0]["max_req_input_len"]
|
||||
self.startup_time = aggregate_scheduler_startup_times(
|
||||
@@ -836,15 +846,20 @@ def run_data_parallel_controller_process(
|
||||
scheduler_pids = [
|
||||
proc.pid for proc in controller.scheduler_procs if proc is not None
|
||||
]
|
||||
pipe_writer.send(
|
||||
{
|
||||
"status": "ready",
|
||||
"max_total_num_tokens": controller.max_total_num_tokens,
|
||||
"max_req_input_len": controller.max_req_input_len,
|
||||
"startup_time": controller.startup_time,
|
||||
SCHEDULER_PIDS_ARG: scheduler_pids,
|
||||
}
|
||||
)
|
||||
init_info = {
|
||||
"status": "ready",
|
||||
"max_total_num_tokens": controller.max_total_num_tokens,
|
||||
"max_req_input_len": controller.max_req_input_len,
|
||||
"startup_time": controller.startup_time,
|
||||
SCHEDULER_PIDS_ARG: scheduler_pids,
|
||||
}
|
||||
if get_serving().grpc_port is not None and not (
|
||||
get_serving().smg_grpc_mode or get_serving().grpc_mode
|
||||
):
|
||||
init_info["kv_event_sources"] = sorted(
|
||||
controller.local_kv_event_sources, key=lambda source: source["dp_rank"]
|
||||
)
|
||||
pipe_writer.send(init_info)
|
||||
# The primary owns routing for the expanded scheduler set.
|
||||
if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
|
||||
controller.event_loop()
|
||||
|
||||
@@ -1826,6 +1826,14 @@ class Scheduler(
|
||||
"startup_time": self.startup_time,
|
||||
}
|
||||
|
||||
if get_serving().grpc_port is not None and not (
|
||||
get_serving().smg_grpc_mode or get_serving().grpc_mode
|
||||
):
|
||||
result_dict["kv_event_sources"] = (
|
||||
self.kv_events_publisher.local_kv_event_sources(
|
||||
self.page_size * get_parallel().dcp_size
|
||||
)
|
||||
)
|
||||
return result_dict
|
||||
|
||||
def release_host_resources(self) -> None:
|
||||
|
||||
@@ -89,6 +89,12 @@ class SchedulerKvEventsPublisher:
|
||||
if not self.send_metrics_from_scheduler.closed:
|
||||
sock_send(self.send_metrics_from_scheduler, kv_metrics)
|
||||
|
||||
def local_kv_event_sources(self, block_size: int) -> list[dict[str, Any]]:
|
||||
if self.kv_event_publisher is None:
|
||||
return []
|
||||
source = self.kv_event_publisher.describe_local_source(block_size)
|
||||
return [source] if source is not None else []
|
||||
|
||||
def publish_kv_events(self):
|
||||
if not self.enable_kv_cache_events:
|
||||
return
|
||||
|
||||
@@ -101,13 +101,16 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.observability.func_timer import enable_func_timer
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import (
|
||||
describe_kv_events_publisher,
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
get_serving,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
|
||||
from sglang.srt.utils.video_decoder import _BACKEND, VideoDecoderWrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -3649,6 +3652,41 @@ def _configure_uvicorn_access_log_filter(
|
||||
filters_list.append(filter_name)
|
||||
|
||||
|
||||
def build_server_info(server_args, scheduler_info: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build server metadata shared by HTTP and native gRPC.
|
||||
|
||||
Callers add transport-specific fields and serialize the result.
|
||||
"""
|
||||
result = server_args.resolved_dict()
|
||||
result["launch_command"] = server_args.launch_command
|
||||
result.update(scheduler_info)
|
||||
result["kv_events"] = describe_kv_events_publisher(server_args)
|
||||
return result
|
||||
|
||||
|
||||
def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]):
|
||||
"""Expose GetServerInfo only, without constructing a TokenizerManager.
|
||||
|
||||
The snapshot is taken after scheduler readiness. This does not launch a
|
||||
sidecar or enable inference/control RPCs.
|
||||
"""
|
||||
serving = get_serving()
|
||||
if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode:
|
||||
return None
|
||||
|
||||
from sglang.srt.rust_extensions import load_rust_extension
|
||||
|
||||
grpc_native = load_rust_extension("sglang.srt.rust_extensions._grpc")
|
||||
return grpc_native.start_metadata_server(
|
||||
host=serving.host,
|
||||
port=serving.grpc_port,
|
||||
server_info_json=json.dumps(
|
||||
msgspec_to_builtins(build_server_info(server_args, scheduler_info)),
|
||||
default=str,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def launch_dummy_health_check_server(host, port, enable_metrics):
|
||||
import asyncio
|
||||
|
||||
|
||||
Reference in New Issue
Block a user