feat(grpc): expose follower metadata and node-local KV sources (#39659)

Signed-off-by: jain-ria <riajain@NVIDIA.com>
Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
This commit is contained in:
jain-ria
2026-10-06 16:39:01 -07:00
committed by GitHub
co-authored by ishandhanani
parent 90aa1cb0fb
commit fd62ee84f7
17 changed files with 703 additions and 130 deletions
@@ -80,7 +80,8 @@ class Serving(msgspec.Struct):
Optional[int],
"Port for the native gRPC server, started alongside HTTP. Setting this "
"(or SGLANG_GRPC_PORT) enables the native gRPC server; it is off by "
"default. In legacy --smg-grpc-mode this is the SMG server port and "
"default. Follower nodes expose GetServerInfo only. "
"In legacy --smg-grpc-mode this is the SMG server port and "
"defaults to --port + 10000.",
] = None
grpc_response_timeout_secs: A[
+80 -9
View File
@@ -20,6 +20,7 @@ KV caching events
import atexit
import enum
import logging
import os
import queue
import threading
import time
@@ -329,6 +330,10 @@ class EventPublisher(ABC):
def shutdown(self) -> None:
"""Shutdown the publisher."""
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
"""Describe a source accessible to a separate local process, if any."""
return None
class NullEventPublisher(EventPublisher):
"""No-op implementation (default when disabled)."""
@@ -349,7 +354,7 @@ class ZmqEventPublisher(EventPublisher):
----------
endpoint:
PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to
connect.
connect, unless overridden by ``bind``.
replay_endpoint:
Optional ROUTER address for replay requests. When given, subscribers can
request missed batches by sending the starting sequence number as an
@@ -362,6 +367,9 @@ class ZmqEventPublisher(EventPublisher):
Maximum number of events to buffer in memory.
topic:
Topic to publish events to.
bind:
Whether to bind the PUB socket. When unset, preserve the endpoint-based
heuristic: bind wildcard, IPC and inproc addresses; connect otherwise.
"""
SHUTDOWN_TIMEOUT: float = 1.0
@@ -376,6 +384,7 @@ class ZmqEventPublisher(EventPublisher):
hwm: int = 100_000,
max_queue_size: int = 100_000,
topic: str = "",
bind: Optional[bool] = None,
) -> None:
# Storage
self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size)
@@ -384,9 +393,12 @@ class ZmqEventPublisher(EventPublisher):
# ZMQ sockets
self._ctx = zmq.Context.instance()
self._pub: Optional[zmq.Socket] = None
self._local_pub_endpoint: Optional[str] = None
self._local_replay_endpoint: Optional[str] = None
self._replay: Optional[zmq.Socket] = None
self._dp_rank = attn_dp_rank
self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank)
self._bind = bind
self._replay_endpoint = self.offset_endpoint_port(
replay_endpoint, self._dp_rank
)
@@ -415,6 +427,43 @@ class ZmqEventPublisher(EventPublisher):
events.attn_dp_rank = self._dp_rank
self._event_queue.put(events)
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
"""Report the bound socket, not a port reconstructed from global DP size.
Connect-style publishers and inproc sockets have no subscribable local
endpoint. They keep working as before, but are not advertised to external
processes. Wildcard binds are reachable on loopback from this node.
"""
def local_endpoint(endpoint: Optional[str]) -> Optional[str]:
if endpoint is None:
return None
if endpoint.startswith("ipc://"):
return endpoint
if parse_advertisable_tcp(endpoint) is None:
return None
address = NetworkAddress.parse(endpoint[len("tcp://") :])
host = address.host
if host in ("*", "0.0.0.0"):
host = "127.0.0.1"
elif host == "::":
host = "::1"
return NetworkAddress(host, address.port).to_tcp()
endpoint = local_endpoint(self._local_pub_endpoint)
if endpoint is None:
return None
source = {
"dp_rank": self._dp_rank,
"endpoint": endpoint,
"topic": self._topic_bytes.decode("utf-8"),
"block_size": block_size,
}
replay_endpoint = local_endpoint(self._local_replay_endpoint)
if replay_endpoint is not None:
source["replay_endpoint"] = replay_endpoint
return source
def shutdown(self) -> None:
"""Stop the publisher thread and clean up resources."""
self._running = False
@@ -453,23 +502,27 @@ class ZmqEventPublisher(EventPublisher):
if self._pub is None:
self._pub = self._ctx.socket(zmq.PUB)
self._pub.set_hwm(self._hwm)
# Heuristic: bind if wildcard / * present, else connect.
# Default heuristic: bind if wildcard / * present, else connect.
# bind stable, connect volatile convention.
# ``0.0.0.0`` is the IPv4 bind-all wildcard alongside ``*``
# and ``::``; ``/server_info`` advertises it as a wildcard,
# so the publisher must bind it for the advertised endpoint
# to actually be listening.
if (
"*" in self._endpoint
or "::" in self._endpoint
or "0.0.0.0" in self._endpoint
or self._endpoint.startswith("ipc://")
or self._endpoint.startswith("inproc://")
):
should_bind = self._bind
if should_bind is None:
should_bind = (
"*" in self._endpoint
or "::" in self._endpoint
or "0.0.0.0" in self._endpoint
or self._endpoint.startswith("ipc://")
or self._endpoint.startswith("inproc://")
)
if should_bind:
logger.debug(
f"ZmqEventPublisher socket publisher_endpoint bind to {self._endpoint}"
)
self._pub.bind(self._endpoint)
self._local_pub_endpoint = self._bound_endpoint(self._pub)
else:
self._pub.connect(self._endpoint)
@@ -483,6 +536,19 @@ class ZmqEventPublisher(EventPublisher):
f"ZmqEventPublisher socket replay_endpoint bind to {self._replay_endpoint}"
)
self._replay.bind(self._replay_endpoint)
self._local_replay_endpoint = self._bound_endpoint(self._replay)
@staticmethod
def _bound_endpoint(socket: zmq.Socket) -> str:
endpoint = socket.getsockopt_string(zmq.LAST_ENDPOINT)
if endpoint.startswith("ipc://"):
path = endpoint[len("ipc://") :]
# Capture the publisher's working directory at bind time so a
# separate process can use the address. Abstract IPC names are
# independent of the filesystem and must remain unchanged.
if not path.startswith("@"):
endpoint = f"ipc://{os.path.abspath(path)}"
return endpoint
def _publisher_thread(self) -> None:
"""Background thread that processes the event queue."""
@@ -589,6 +655,11 @@ class KVEventsConfig(BaseModel):
"""The zmq endpoint to use for publishing kv events.
"""
bind: Optional[bool] = None
"""Bind (true) or connect (false) the PUB socket. When unset, infer from
the endpoint as before. Use true to bind a concrete address such as loopback.
"""
replay_endpoint: Optional[str] = None
"""The zmq endpoint to use for replaying kv events.
"""
+43 -9
View File
@@ -140,6 +140,7 @@ from sglang.srt.utils import (
numa_utils,
set_prometheus_multiproc_dir,
set_ulimit,
start_follower_grpc_server,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
from sglang.srt.utils.network import (
@@ -173,6 +174,12 @@ class SchedulerInitResult:
wait_for_ready: Callable[[], None] = lambda: None
block_until_scheduler_exits: Callable[[], None] = lambda: None
engine_info_bootstrap_server: Optional[Any] = None
grpc_server: Optional[Any] = None
def stop_grpc_server(self) -> None:
if self.grpc_server is not None:
self.grpc_server.shutdown()
self.grpc_server = None
def init_tokenizer_manager(
@@ -951,6 +958,17 @@ class Engine(EngineScoreMixin, EngineBase):
def wait_for_ready():
infos = _wait_for_scheduler_ready(scheduler_pipe_readers, scheduler_procs)
if any("kv_event_sources" in info for info in infos):
# Both gRPC entrypoints consume the first scheduler info. Keep
# the sources from every local scheduler, not just the first.
infos[0]["kv_event_sources"] = sorted(
(
source
for info in infos
for source in info.get("kv_event_sources", [])
),
key=lambda source: source["dp_rank"],
)
scheduler_infos.extend(infos)
if use_dp_controller:
for info in infos:
@@ -1173,6 +1191,18 @@ class Engine(EngineScoreMixin, EngineBase):
# Non-zero-rank nodes do not run tokenizer processes.
scheduler_init_result.wait_for_ready()
try:
scheduler_init_result.grpc_server = start_follower_grpc_server(
server_args, scheduler_init_result.scheduler_infos[0]
)
except BaseException:
# Engine.__init__ has not received these handles yet. Do not
# leave GPU workers behind if binding the metadata port fails.
for proc in scheduler_procs or []:
kill_process_tree(proc.pid, wait_timeout=60)
cls._terminate_weight_cache_daemons(weight_cache_daemon_procs)
raise
if os.getenv("SGLANG_BLOCK_NONZERO_RANK_CHILDREN") == "0":
# When using `Engine` as a Python API, we don't want to block here.
return (
@@ -1202,14 +1232,16 @@ class Engine(EngineScoreMixin, EngineBase):
rust_server_owns_base_port = (
envs.SGLANG_RUST_SERVER.get() and node_hosts_rust_server()
)
if not rust_server_owns_base_port:
launch_dummy_health_check_server(
get_serving().host,
get_serving().port,
get_observability().enable_metrics,
)
scheduler_init_result.block_until_scheduler_exits()
try:
if not rust_server_owns_base_port:
launch_dummy_health_check_server(
get_serving().host,
get_serving().port,
get_observability().enable_metrics,
)
scheduler_init_result.block_until_scheduler_exits()
finally:
scheduler_init_result.stop_grpc_server()
return (
None,
None,
@@ -1384,6 +1416,9 @@ class Engine(EngineScoreMixin, EngineBase):
pass
self._multi_tokenizer_shm = None
try:
scheduler_init_result = getattr(self, "_scheduler_init_result", None)
if scheduler_init_result is not None:
scheduler_init_result.stop_grpc_server()
if (
self.tokenizer_manager is not None
and self.tokenizer_manager._subprocess_watchdog is not None
@@ -1789,7 +1824,6 @@ class Engine(EngineScoreMixin, EngineBase):
def _set_envs_and_config(server_args: ServerArgs):
cfg = resolving_view(server_args)
# Set global environments
# MNNVL fabric (GB200/GB300) multi-node: cross-node NVLink needs NCCL's
+10 -18
View File
@@ -16,11 +16,8 @@ from pydantic import ValidationError
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_lora,
get_serving,
)
from sglang.srt.runtime_context import get_lora, get_serving
from sglang.srt.utils.common import build_server_info
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
logger = logging.getLogger(__name__)
@@ -238,9 +235,7 @@ class RuntimeHandle:
return self._openai_serving_classes
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.entrypoints.openai.serving_classify import (
OpenAIServingClassify,
)
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
from sglang.srt.entrypoints.openai.serving_completions import (
OpenAIServingCompletion,
)
@@ -430,17 +425,14 @@ class RuntimeHandle:
return json.dumps(result, default=str)
def get_server_info(self) -> str:
result: Dict[str, Any] = self.tokenizer_manager.server_args.resolved_dict()
# `resolved_dict` answers with what resolution decided; the launch
# command answers with what was asked for, and the two are not
# derivable from each other. The HTTP and in-process readbacks both
# carry it, so this one does too.
result["launch_command"] = self.tokenizer_manager.server_args.launch_command
result.update(self.scheduler_info)
result["kv_events"] = describe_kv_events_publisher(
self.tokenizer_manager.server_args
return json.dumps(
msgspec_to_builtins(
build_server_info(
self.tokenizer_manager.server_args, self.scheduler_info
)
),
default=str,
)
return json.dumps(msgspec_to_builtins(result), default=str)
def health_check(self) -> bool:
from sglang.srt.managers.tokenizer_manager import ServerStatus
+2 -8
View File
@@ -177,6 +177,7 @@ from sglang.srt.server_args import PortArgs, ServerArgs
from sglang.srt.utils import (
add_prometheus_middleware,
add_prometheus_track_response_middleware,
build_server_info,
delete_directory,
get_bool_env_var,
is_mps,
@@ -502,7 +503,6 @@ app.include_router(v1_loads_router)
from sglang.srt.arg_groups.serving_hook import ssl_verify_of
from sglang.srt.entrypoints.elastic_ep import router as elastic_ep_router
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_disagg,
get_exec,
get_lora,
@@ -859,17 +859,11 @@ async def server_info():
return msgspec_to_builtins(
{
**server_args.resolved_dict(),
"launch_command": server_args.launch_command,
**_global_state.scheduler_info,
**build_server_info(server_args, _global_state.scheduler_info),
"startup_time": _global_state.tokenizer_manager.startup_time,
"internal_states": internal_states,
"version": __version__,
"frontend": "python",
# Structured KV-event publisher descriptor for KV-aware routers.
# `None` when publishing is disabled or misconfigured; see
# `runtime_context.describe_kv_events_publisher` for the contract.
"kv_events": describe_kv_events_publisher(server_args),
}
)
@@ -200,6 +200,7 @@ class DataParallelController:
# Launch data parallel workers
self.scheduler_procs = []
self.local_kv_event_sources = []
self.workers: list[zmq.Socket | None] = [None] * self.max_dp_size
self.status: list[bool] = list(self.dp_active)
self._active_workers: list[int] = list(range(self.launch_dp_size))
@@ -723,6 +724,15 @@ class DataParallelController:
for i in range(len(scheduler_pipe_readers)):
scheduler_info.append(scheduler_pipe_readers[i].recv())
# Pure-DP TP groups launch concurrently. Their ready replies contain
# only publishers owned by schedulers on this node.
with self.env_lock:
self.local_kv_event_sources.extend(
source
for info in scheduler_info
for source in info.get("kv_event_sources", [])
)
self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"]
self.max_req_input_len = scheduler_info[0]["max_req_input_len"]
self.startup_time = aggregate_scheduler_startup_times(
@@ -836,15 +846,20 @@ def run_data_parallel_controller_process(
scheduler_pids = [
proc.pid for proc in controller.scheduler_procs if proc is not None
]
pipe_writer.send(
{
"status": "ready",
"max_total_num_tokens": controller.max_total_num_tokens,
"max_req_input_len": controller.max_req_input_len,
"startup_time": controller.startup_time,
SCHEDULER_PIDS_ARG: scheduler_pids,
}
)
init_info = {
"status": "ready",
"max_total_num_tokens": controller.max_total_num_tokens,
"max_req_input_len": controller.max_req_input_len,
"startup_time": controller.startup_time,
SCHEDULER_PIDS_ARG: scheduler_pids,
}
if get_serving().grpc_port is not None and not (
get_serving().smg_grpc_mode or get_serving().grpc_mode
):
init_info["kv_event_sources"] = sorted(
controller.local_kv_event_sources, key=lambda source: source["dp_rank"]
)
pipe_writer.send(init_info)
# The primary owns routing for the expanded scheduler set.
if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
controller.event_loop()
+8
View File
@@ -1826,6 +1826,14 @@ class Scheduler(
"startup_time": self.startup_time,
}
if get_serving().grpc_port is not None and not (
get_serving().smg_grpc_mode or get_serving().grpc_mode
):
result_dict["kv_event_sources"] = (
self.kv_events_publisher.local_kv_event_sources(
self.page_size * get_parallel().dcp_size
)
)
return result_dict
def release_host_resources(self) -> None:
@@ -89,6 +89,12 @@ class SchedulerKvEventsPublisher:
if not self.send_metrics_from_scheduler.closed:
sock_send(self.send_metrics_from_scheduler, kv_metrics)
def local_kv_event_sources(self, block_size: int) -> list[dict[str, Any]]:
if self.kv_event_publisher is None:
return []
source = self.kv_event_publisher.describe_local_source(block_size)
return [source] if source is not None else []
def publish_kv_events(self):
if not self.enable_kv_cache_events:
return
+38
View File
@@ -101,13 +101,16 @@ from sglang.srt.environ import envs
from sglang.srt.observability.func_timer import enable_func_timer
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_exec,
get_flags,
get_model,
get_parallel,
get_platform,
get_serving,
get_spec,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
from sglang.srt.utils.video_decoder import _BACKEND, VideoDecoderWrapper
if TYPE_CHECKING:
@@ -3649,6 +3652,41 @@ def _configure_uvicorn_access_log_filter(
filters_list.append(filter_name)
def build_server_info(server_args, scheduler_info: Dict[str, Any]) -> Dict[str, Any]:
"""Build server metadata shared by HTTP and native gRPC.
Callers add transport-specific fields and serialize the result.
"""
result = server_args.resolved_dict()
result["launch_command"] = server_args.launch_command
result.update(scheduler_info)
result["kv_events"] = describe_kv_events_publisher(server_args)
return result
def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]):
"""Expose GetServerInfo only, without constructing a TokenizerManager.
The snapshot is taken after scheduler readiness. This does not launch a
sidecar or enable inference/control RPCs.
"""
serving = get_serving()
if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode:
return None
from sglang.srt.rust_extensions import load_rust_extension
grpc_native = load_rust_extension("sglang.srt.rust_extensions._grpc")
return grpc_native.start_metadata_server(
host=serving.host,
port=serving.grpc_port,
server_info_json=json.dumps(
msgspec_to_builtins(build_server_info(server_args, scheduler_info)),
default=str,
),
)
def launch_dummy_health_check_server(host, port, enable_metrics):
import asyncio