feat(grpc): expose follower metadata and node-local KV sources (#39659)

Signed-off-by: jain-ria <riajain@NVIDIA.com>
Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
This commit is contained in:
jain-ria
2026-10-06 16:39:01 -07:00
committed by GitHub
co-authored by ishandhanani
parent 90aa1cb0fb
commit fd62ee84f7
17 changed files with 703 additions and 130 deletions
+3
View File
@@ -13,6 +13,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
tonic_build::configure()
.build_server(true)
.build_client(true)
// The follower implements only GetServerInfo; all other RPCs return
// UNIMPLEMENTED without constructing the inference bridge.
.generate_default_stubs(true)
.protoc_arg("--experimental_allow_proto3_optional")
.compile_protos(&[proto_path], &["."])?;
@@ -80,7 +80,8 @@ class Serving(msgspec.Struct):
Optional[int],
"Port for the native gRPC server, started alongside HTTP. Setting this "
"(or SGLANG_GRPC_PORT) enables the native gRPC server; it is off by "
"default. In legacy --smg-grpc-mode this is the SMG server port and "
"default. Follower nodes expose GetServerInfo only. "
"In legacy --smg-grpc-mode this is the SMG server port and "
"defaults to --port + 10000.",
] = None
grpc_response_timeout_secs: A[
+80 -9
View File
@@ -20,6 +20,7 @@ KV caching events
import atexit
import enum
import logging
import os
import queue
import threading
import time
@@ -329,6 +330,10 @@ class EventPublisher(ABC):
def shutdown(self) -> None:
"""Shutdown the publisher."""
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
"""Describe a source accessible to a separate local process, if any."""
return None
class NullEventPublisher(EventPublisher):
"""No-op implementation (default when disabled)."""
@@ -349,7 +354,7 @@ class ZmqEventPublisher(EventPublisher):
----------
endpoint:
PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to
connect.
connect, unless overridden by ``bind``.
replay_endpoint:
Optional ROUTER address for replay requests. When given, subscribers can
request missed batches by sending the starting sequence number as an
@@ -362,6 +367,9 @@ class ZmqEventPublisher(EventPublisher):
Maximum number of events to buffer in memory.
topic:
Topic to publish events to.
bind:
Whether to bind the PUB socket. When unset, preserve the endpoint-based
heuristic: bind wildcard, IPC and inproc addresses; connect otherwise.
"""
SHUTDOWN_TIMEOUT: float = 1.0
@@ -376,6 +384,7 @@ class ZmqEventPublisher(EventPublisher):
hwm: int = 100_000,
max_queue_size: int = 100_000,
topic: str = "",
bind: Optional[bool] = None,
) -> None:
# Storage
self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size)
@@ -384,9 +393,12 @@ class ZmqEventPublisher(EventPublisher):
# ZMQ sockets
self._ctx = zmq.Context.instance()
self._pub: Optional[zmq.Socket] = None
self._local_pub_endpoint: Optional[str] = None
self._local_replay_endpoint: Optional[str] = None
self._replay: Optional[zmq.Socket] = None
self._dp_rank = attn_dp_rank
self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank)
self._bind = bind
self._replay_endpoint = self.offset_endpoint_port(
replay_endpoint, self._dp_rank
)
@@ -415,6 +427,43 @@ class ZmqEventPublisher(EventPublisher):
events.attn_dp_rank = self._dp_rank
self._event_queue.put(events)
def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
"""Report the bound socket, not a port reconstructed from global DP size.
Connect-style publishers and inproc sockets have no subscribable local
endpoint. They keep working as before, but are not advertised to external
processes. Wildcard binds are reachable on loopback from this node.
"""
def local_endpoint(endpoint: Optional[str]) -> Optional[str]:
if endpoint is None:
return None
if endpoint.startswith("ipc://"):
return endpoint
if parse_advertisable_tcp(endpoint) is None:
return None
address = NetworkAddress.parse(endpoint[len("tcp://") :])
host = address.host
if host in ("*", "0.0.0.0"):
host = "127.0.0.1"
elif host == "::":
host = "::1"
return NetworkAddress(host, address.port).to_tcp()
endpoint = local_endpoint(self._local_pub_endpoint)
if endpoint is None:
return None
source = {
"dp_rank": self._dp_rank,
"endpoint": endpoint,
"topic": self._topic_bytes.decode("utf-8"),
"block_size": block_size,
}
replay_endpoint = local_endpoint(self._local_replay_endpoint)
if replay_endpoint is not None:
source["replay_endpoint"] = replay_endpoint
return source
def shutdown(self) -> None:
"""Stop the publisher thread and clean up resources."""
self._running = False
@@ -453,23 +502,27 @@ class ZmqEventPublisher(EventPublisher):
if self._pub is None:
self._pub = self._ctx.socket(zmq.PUB)
self._pub.set_hwm(self._hwm)
# Heuristic: bind if wildcard / * present, else connect.
# Default heuristic: bind if wildcard / * present, else connect.
# bind stable, connect volatile convention.
# ``0.0.0.0`` is the IPv4 bind-all wildcard alongside ``*``
# and ``::``; ``/server_info`` advertises it as a wildcard,
# so the publisher must bind it for the advertised endpoint
# to actually be listening.
if (
"*" in self._endpoint
or "::" in self._endpoint
or "0.0.0.0" in self._endpoint
or self._endpoint.startswith("ipc://")
or self._endpoint.startswith("inproc://")
):
should_bind = self._bind
if should_bind is None:
should_bind = (
"*" in self._endpoint
or "::" in self._endpoint
or "0.0.0.0" in self._endpoint
or self._endpoint.startswith("ipc://")
or self._endpoint.startswith("inproc://")
)
if should_bind:
logger.debug(
f"ZmqEventPublisher socket publisher_endpoint bind to {self._endpoint}"
)
self._pub.bind(self._endpoint)
self._local_pub_endpoint = self._bound_endpoint(self._pub)
else:
self._pub.connect(self._endpoint)
@@ -483,6 +536,19 @@ class ZmqEventPublisher(EventPublisher):
f"ZmqEventPublisher socket replay_endpoint bind to {self._replay_endpoint}"
)
self._replay.bind(self._replay_endpoint)
self._local_replay_endpoint = self._bound_endpoint(self._replay)
@staticmethod
def _bound_endpoint(socket: zmq.Socket) -> str:
endpoint = socket.getsockopt_string(zmq.LAST_ENDPOINT)
if endpoint.startswith("ipc://"):
path = endpoint[len("ipc://") :]
# Capture the publisher's working directory at bind time so a
# separate process can use the address. Abstract IPC names are
# independent of the filesystem and must remain unchanged.
if not path.startswith("@"):
endpoint = f"ipc://{os.path.abspath(path)}"
return endpoint
def _publisher_thread(self) -> None:
"""Background thread that processes the event queue."""
@@ -589,6 +655,11 @@ class KVEventsConfig(BaseModel):
"""The zmq endpoint to use for publishing kv events.
"""
bind: Optional[bool] = None
"""Bind (true) or connect (false) the PUB socket. When unset, infer from
the endpoint as before. Use true to bind a concrete address such as loopback.
"""
replay_endpoint: Optional[str] = None
"""The zmq endpoint to use for replaying kv events.
"""
+43 -9
View File
@@ -140,6 +140,7 @@ from sglang.srt.utils import (
numa_utils,
set_prometheus_multiproc_dir,
set_ulimit,
start_follower_grpc_server,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
from sglang.srt.utils.network import (
@@ -173,6 +174,12 @@ class SchedulerInitResult:
wait_for_ready: Callable[[], None] = lambda: None
block_until_scheduler_exits: Callable[[], None] = lambda: None
engine_info_bootstrap_server: Optional[Any] = None
grpc_server: Optional[Any] = None
def stop_grpc_server(self) -> None:
if self.grpc_server is not None:
self.grpc_server.shutdown()
self.grpc_server = None
def init_tokenizer_manager(
@@ -951,6 +958,17 @@ class Engine(EngineScoreMixin, EngineBase):
def wait_for_ready():
infos = _wait_for_scheduler_ready(scheduler_pipe_readers, scheduler_procs)
if any("kv_event_sources" in info for info in infos):
# Both gRPC entrypoints consume the first scheduler info. Keep
# the sources from every local scheduler, not just the first.
infos[0]["kv_event_sources"] = sorted(
(
source
for info in infos
for source in info.get("kv_event_sources", [])
),
key=lambda source: source["dp_rank"],
)
scheduler_infos.extend(infos)
if use_dp_controller:
for info in infos:
@@ -1173,6 +1191,18 @@ class Engine(EngineScoreMixin, EngineBase):
# Non-zero-rank nodes do not run tokenizer processes.
scheduler_init_result.wait_for_ready()
try:
scheduler_init_result.grpc_server = start_follower_grpc_server(
server_args, scheduler_init_result.scheduler_infos[0]
)
except BaseException:
# Engine.__init__ has not received these handles yet. Do not
# leave GPU workers behind if binding the metadata port fails.
for proc in scheduler_procs or []:
kill_process_tree(proc.pid, wait_timeout=60)
cls._terminate_weight_cache_daemons(weight_cache_daemon_procs)
raise
if os.getenv("SGLANG_BLOCK_NONZERO_RANK_CHILDREN") == "0":
# When using `Engine` as a Python API, we don't want to block here.
return (
@@ -1202,14 +1232,16 @@ class Engine(EngineScoreMixin, EngineBase):
rust_server_owns_base_port = (
envs.SGLANG_RUST_SERVER.get() and node_hosts_rust_server()
)
if not rust_server_owns_base_port:
launch_dummy_health_check_server(
get_serving().host,
get_serving().port,
get_observability().enable_metrics,
)
scheduler_init_result.block_until_scheduler_exits()
try:
if not rust_server_owns_base_port:
launch_dummy_health_check_server(
get_serving().host,
get_serving().port,
get_observability().enable_metrics,
)
scheduler_init_result.block_until_scheduler_exits()
finally:
scheduler_init_result.stop_grpc_server()
return (
None,
None,
@@ -1384,6 +1416,9 @@ class Engine(EngineScoreMixin, EngineBase):
pass
self._multi_tokenizer_shm = None
try:
scheduler_init_result = getattr(self, "_scheduler_init_result", None)
if scheduler_init_result is not None:
scheduler_init_result.stop_grpc_server()
if (
self.tokenizer_manager is not None
and self.tokenizer_manager._subprocess_watchdog is not None
@@ -1789,7 +1824,6 @@ class Engine(EngineScoreMixin, EngineBase):
def _set_envs_and_config(server_args: ServerArgs):
cfg = resolving_view(server_args)
# Set global environments
# MNNVL fabric (GB200/GB300) multi-node: cross-node NVLink needs NCCL's
+10 -18
View File
@@ -16,11 +16,8 @@ from pydantic import ValidationError
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_lora,
get_serving,
)
from sglang.srt.runtime_context import get_lora, get_serving
from sglang.srt.utils.common import build_server_info
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
logger = logging.getLogger(__name__)
@@ -238,9 +235,7 @@ class RuntimeHandle:
return self._openai_serving_classes
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.entrypoints.openai.serving_classify import (
OpenAIServingClassify,
)
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
from sglang.srt.entrypoints.openai.serving_completions import (
OpenAIServingCompletion,
)
@@ -430,17 +425,14 @@ class RuntimeHandle:
return json.dumps(result, default=str)
def get_server_info(self) -> str:
result: Dict[str, Any] = self.tokenizer_manager.server_args.resolved_dict()
# `resolved_dict` answers with what resolution decided; the launch
# command answers with what was asked for, and the two are not
# derivable from each other. The HTTP and in-process readbacks both
# carry it, so this one does too.
result["launch_command"] = self.tokenizer_manager.server_args.launch_command
result.update(self.scheduler_info)
result["kv_events"] = describe_kv_events_publisher(
self.tokenizer_manager.server_args
return json.dumps(
msgspec_to_builtins(
build_server_info(
self.tokenizer_manager.server_args, self.scheduler_info
)
),
default=str,
)
return json.dumps(msgspec_to_builtins(result), default=str)
def health_check(self) -> bool:
from sglang.srt.managers.tokenizer_manager import ServerStatus
+2 -8
View File
@@ -177,6 +177,7 @@ from sglang.srt.server_args import PortArgs, ServerArgs
from sglang.srt.utils import (
add_prometheus_middleware,
add_prometheus_track_response_middleware,
build_server_info,
delete_directory,
get_bool_env_var,
is_mps,
@@ -502,7 +503,6 @@ app.include_router(v1_loads_router)
from sglang.srt.arg_groups.serving_hook import ssl_verify_of
from sglang.srt.entrypoints.elastic_ep import router as elastic_ep_router
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_disagg,
get_exec,
get_lora,
@@ -859,17 +859,11 @@ async def server_info():
return msgspec_to_builtins(
{
**server_args.resolved_dict(),
"launch_command": server_args.launch_command,
**_global_state.scheduler_info,
**build_server_info(server_args, _global_state.scheduler_info),
"startup_time": _global_state.tokenizer_manager.startup_time,
"internal_states": internal_states,
"version": __version__,
"frontend": "python",
# Structured KV-event publisher descriptor for KV-aware routers.
# `None` when publishing is disabled or misconfigured; see
# `runtime_context.describe_kv_events_publisher` for the contract.
"kv_events": describe_kv_events_publisher(server_args),
}
)
@@ -200,6 +200,7 @@ class DataParallelController:
# Launch data parallel workers
self.scheduler_procs = []
self.local_kv_event_sources = []
self.workers: list[zmq.Socket | None] = [None] * self.max_dp_size
self.status: list[bool] = list(self.dp_active)
self._active_workers: list[int] = list(range(self.launch_dp_size))
@@ -723,6 +724,15 @@ class DataParallelController:
for i in range(len(scheduler_pipe_readers)):
scheduler_info.append(scheduler_pipe_readers[i].recv())
# Pure-DP TP groups launch concurrently. Their ready replies contain
# only publishers owned by schedulers on this node.
with self.env_lock:
self.local_kv_event_sources.extend(
source
for info in scheduler_info
for source in info.get("kv_event_sources", [])
)
self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"]
self.max_req_input_len = scheduler_info[0]["max_req_input_len"]
self.startup_time = aggregate_scheduler_startup_times(
@@ -836,15 +846,20 @@ def run_data_parallel_controller_process(
scheduler_pids = [
proc.pid for proc in controller.scheduler_procs if proc is not None
]
pipe_writer.send(
{
"status": "ready",
"max_total_num_tokens": controller.max_total_num_tokens,
"max_req_input_len": controller.max_req_input_len,
"startup_time": controller.startup_time,
SCHEDULER_PIDS_ARG: scheduler_pids,
}
)
init_info = {
"status": "ready",
"max_total_num_tokens": controller.max_total_num_tokens,
"max_req_input_len": controller.max_req_input_len,
"startup_time": controller.startup_time,
SCHEDULER_PIDS_ARG: scheduler_pids,
}
if get_serving().grpc_port is not None and not (
get_serving().smg_grpc_mode or get_serving().grpc_mode
):
init_info["kv_event_sources"] = sorted(
controller.local_kv_event_sources, key=lambda source: source["dp_rank"]
)
pipe_writer.send(init_info)
# The primary owns routing for the expanded scheduler set.
if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
controller.event_loop()
+8
View File
@@ -1826,6 +1826,14 @@ class Scheduler(
"startup_time": self.startup_time,
}
if get_serving().grpc_port is not None and not (
get_serving().smg_grpc_mode or get_serving().grpc_mode
):
result_dict["kv_event_sources"] = (
self.kv_events_publisher.local_kv_event_sources(
self.page_size * get_parallel().dcp_size
)
)
return result_dict
def release_host_resources(self) -> None:
@@ -89,6 +89,12 @@ class SchedulerKvEventsPublisher:
if not self.send_metrics_from_scheduler.closed:
sock_send(self.send_metrics_from_scheduler, kv_metrics)
def local_kv_event_sources(self, block_size: int) -> list[dict[str, Any]]:
if self.kv_event_publisher is None:
return []
source = self.kv_event_publisher.describe_local_source(block_size)
return [source] if source is not None else []
def publish_kv_events(self):
if not self.enable_kv_cache_events:
return
+38
View File
@@ -101,13 +101,16 @@ from sglang.srt.environ import envs
from sglang.srt.observability.func_timer import enable_func_timer
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_exec,
get_flags,
get_model,
get_parallel,
get_platform,
get_serving,
get_spec,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
from sglang.srt.utils.video_decoder import _BACKEND, VideoDecoderWrapper
if TYPE_CHECKING:
@@ -3649,6 +3652,41 @@ def _configure_uvicorn_access_log_filter(
filters_list.append(filter_name)
def build_server_info(server_args, scheduler_info: Dict[str, Any]) -> Dict[str, Any]:
"""Build server metadata shared by HTTP and native gRPC.
Callers add transport-specific fields and serialize the result.
"""
result = server_args.resolved_dict()
result["launch_command"] = server_args.launch_command
result.update(scheduler_info)
result["kv_events"] = describe_kv_events_publisher(server_args)
return result
def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]):
"""Expose GetServerInfo only, without constructing a TokenizerManager.
The snapshot is taken after scheduler readiness. This does not launch a
sidecar or enable inference/control RPCs.
"""
serving = get_serving()
if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode:
return None
from sglang.srt.rust_extensions import load_rust_extension
grpc_native = load_rust_extension("sglang.srt.rust_extensions._grpc")
return grpc_native.start_metadata_server(
host=serving.host,
port=serving.grpc_port,
server_info_json=json.dumps(
msgspec_to_builtins(build_server_info(server_args, scheduler_info)),
default=str,
),
)
def launch_dummy_health_check_server(host, port, enable_metrics):
import asyncio
+80 -39
View File
@@ -36,6 +36,76 @@ impl GrpcServerHandle {
}
}
fn bind_listener(host: &str, port: u16) -> PyResult<TcpListener> {
let addr: SocketAddr = format!("{}:{}", host, port)
.parse()
.map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {}", e)))?;
let listener = TcpListener::bind(addr).map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!(
"Failed to bind gRPC server to {}: {}",
addr, e
))
})?;
listener.set_nonblocking(true).map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!(
"Failed to configure gRPC listener for {}: {}",
addr, e
))
})?;
Ok(listener)
}
fn start_server_thread(
rt: tokio::runtime::Runtime,
listener: TcpListener,
service: server::ServerMode,
) -> PyResult<GrpcServerHandle> {
let shutdown = Arc::new(Notify::new());
let shutdown_clone = shutdown.clone();
let join_handle = std::thread::Builder::new()
.name("sglang-grpc".to_string())
.spawn(move || {
if let Err(e) = rt.block_on(server::run_grpc_server(listener, service, shutdown_clone))
{
tracing::error!("gRPC server exited with error: {}", e);
}
})
.map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to spawn gRPC thread: {}", e))
})?;
Ok(GrpcServerHandle {
shutdown,
join_handle: Some(join_handle),
})
}
/// Start a follower server exposing only GetServerInfo, without an inference bridge.
#[pyfunction]
fn start_metadata_server(
host: String,
port: u16,
server_info_json: String,
) -> PyResult<GrpcServerHandle> {
let listener = bind_listener(&host, port)?;
// Metadata is a startup snapshot; it needs neither Python callbacks nor a
// worker pool. The dedicated server thread drives this runtime directly.
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|err| {
pyo3::exceptions::PyRuntimeError::new_err(format!(
"Failed to build Tokio runtime for gRPC metadata server: {}",
err
))
})?;
start_server_thread(
rt,
listener,
server::ServerMode::Metadata(server::MetadataService { server_info_json }),
)
}
struct TokenizerInfo {
tokenizer_path: Option<String>,
tokenizer_mode: Option<String>,
@@ -160,9 +230,6 @@ fn start_server(
)
.try_init();
let addr: SocketAddr = format!("{}:{}", host, port)
.parse()
.map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {}", e)))?;
let worker_threads = worker_threads.max(1);
let response_channel_capacity = if response_channel_capacity == 0 {
tracing::warn!(
@@ -183,18 +250,7 @@ fn start_server(
response_timeout_secs
};
let response_timeout = Duration::from_secs(response_timeout_secs);
let listener = TcpListener::bind(addr).map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!(
"Failed to bind gRPC server to {}: {}",
addr, e
))
})?;
listener.set_nonblocking(true).map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!(
"Failed to configure gRPC listener for {}: {}",
addr, e
))
})?;
let listener = bind_listener(&host, port)?;
let tokenizer_info = extract_tokenizer_info(&runtime_handle)?;
@@ -226,35 +282,20 @@ fn start_server(
response_channel_capacity,
tokio_handle,
));
let shutdown = Arc::new(Notify::new());
let shutdown_clone = shutdown.clone();
let bridge_clone = bridge.clone();
let join_handle = std::thread::Builder::new()
.name("sglang-grpc".to_string())
.spawn(move || {
if let Err(e) = rt.block_on(server::run_grpc_server(
listener,
bridge_clone,
shutdown_clone,
response_timeout,
)) {
tracing::error!("gRPC server exited with error: {}", e);
}
})
.map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to spawn gRPC thread: {}", e))
})?;
Ok(GrpcServerHandle {
shutdown,
join_handle: Some(join_handle),
})
start_server_thread(
rt,
listener,
server::ServerMode::Inference {
bridge,
response_timeout,
},
)
}
#[pymodule]
fn _grpc(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(start_server, m)?)?;
m.add_function(wrap_pyfunction!(start_metadata_server, m)?)?;
m.add_class::<GrpcServerHandle>()?;
m.add_class::<ChunkSendStatus>()?;
Ok(())
+58 -21
View File
@@ -26,6 +26,25 @@ pub struct SglangServiceImpl {
stream_shutdown: watch::Receiver<bool>,
}
/// A follower has no tokenizer manager or inference bridge. It exposes the same
/// discovery RPC as the leader, with a node-local startup snapshot. Generated
/// default handlers return UNIMPLEMENTED for every other RPC.
pub struct MetadataService {
pub server_info_json: String,
}
#[tonic::async_trait]
impl proto::sglang_service_server::SglangService for MetadataService {
async fn get_server_info(
&self,
_request: Request<proto::GetServerInfoRequest>,
) -> Result<Response<proto::GetServerInfoResponse>, Status> {
Ok(Response::new(proto::GetServerInfoResponse {
json_info: self.server_info_json.clone(),
}))
}
}
type StreamResult<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
pub const DEFAULT_RESPONSE_TIMEOUT_SECS: u64 = 300;
@@ -300,12 +319,10 @@ fn openai_status_code(meta_info: &HashMap<String, String>, default: i32) -> i32
impl proto::sglang_service_server::SglangService for SglangServiceImpl {
// --- SGLang-native RPCs: TextGenerate / Generate ---
type TextGenerateStream = StreamResult<proto::TextGenerateResponse>;
async fn text_generate(
&self,
request: Request<proto::TextGenerateRequest>,
) -> Result<Response<Self::TextGenerateStream>, Status> {
) -> Result<Response<StreamResult<proto::TextGenerateResponse>>, Status> {
let req = request.into_inner();
let rid = req
.rid
@@ -369,12 +386,10 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl {
Ok(Response::new(Box::pin(stream)))
}
type GenerateStream = StreamResult<proto::GenerateResponse>;
async fn generate(
&self,
request: Request<proto::GenerateRequest>,
) -> Result<Response<Self::GenerateStream>, Status> {
) -> Result<Response<StreamResult<proto::GenerateResponse>>, Status> {
let req = request.into_inner();
let rid = req
.rid
@@ -645,12 +660,10 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl {
Ok(Response::new(proto::HealthCheckResponse { healthy }))
}
type WatchEngineStateStream = StreamResult<proto::EngineStateSnapshot>;
async fn watch_engine_state(
&self,
_request: Request<proto::WatchEngineStateRequest>,
) -> Result<Response<Self::WatchEngineStateStream>, Status> {
) -> Result<Response<StreamResult<proto::EngineStateSnapshot>>, Status> {
let mut receiver = self.engine_state.subscribe();
let mut shutdown = self.stream_shutdown.clone();
let stream = async_stream::stream! {
@@ -831,22 +844,18 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl {
// --- OpenAI-compatible RPCs (JSON pass-through) ---
type ChatCompleteStream = StreamResult<proto::OpenAiStreamChunk>;
async fn chat_complete(
&self,
request: Request<proto::OpenAiRequest>,
) -> Result<Response<Self::ChatCompleteStream>, Status> {
) -> Result<Response<StreamResult<proto::OpenAiStreamChunk>>, Status> {
self.openai_streaming_rpc(request, "submit_openai_chat")
.await
}
type CompleteStream = StreamResult<proto::OpenAiStreamChunk>;
async fn complete(
&self,
request: Request<proto::OpenAiRequest>,
) -> Result<Response<Self::CompleteStream>, Status> {
) -> Result<Response<StreamResult<proto::OpenAiStreamChunk>>, Status> {
self.openai_streaming_rpc(request, "submit_openai_complete")
.await
}
@@ -1079,6 +1088,14 @@ async fn recv_json_response(
}
}
pub enum ServerMode {
Inference {
bridge: Arc<PyBridge>,
response_timeout: Duration,
},
Metadata(MetadataService),
}
/// Start the Tonic gRPC server on the given address.
//
// TODO(grpc-auth): this listener is currently unauthenticated. Before exposing
@@ -1086,12 +1103,18 @@ async fn recv_json_response(
// checks the HTTP server applies (see issue tracking gRPC auth parity).
pub async fn run_grpc_server(
listener: std::net::TcpListener,
bridge: Arc<PyBridge>,
mode: ServerMode,
shutdown: Arc<Notify>,
response_timeout: Duration,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let addr = listener.local_addr()?;
let listener = tokio::net::TcpListener::from_std(listener)?;
let (bridge, response_timeout) = match mode {
ServerMode::Metadata(service) => {
return serve_grpc(listener, service, shutdown, None).await;
}
ServerMode::Inference {
bridge,
response_timeout,
} => (bridge, response_timeout),
};
let (state_changed_tx, mut state_changed_rx) = tokio::sync::mpsc::channel(1);
bridge.set_engine_state_changed_callback(state_changed_tx)?;
let engine_state = EngineStatePublisher::new(bridge.clone()).await?;
@@ -1112,6 +1135,19 @@ pub async fn run_grpc_server(
}
});
let result = serve_grpc(listener, service, shutdown, Some(stream_shutdown_tx)).await;
monitor.abort();
result
}
async fn serve_grpc(
listener: std::net::TcpListener,
service: impl proto::sglang_service_server::SglangService,
shutdown: Arc<Notify>,
stream_shutdown: Option<watch::Sender<bool>>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let addr = listener.local_addr()?;
let listener = tokio::net::TcpListener::from_std(listener)?;
let max_message_size = resolve_max_message_size();
let svc = proto::sglang_service_server::SglangServiceServer::new(service)
.max_decoding_message_size(max_message_size)
@@ -1123,11 +1159,12 @@ pub async fn run_grpc_server(
.add_service(svc)
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async move {
shutdown.notified().await;
stream_shutdown_tx.send_replace(true);
if let Some(stream_shutdown) = stream_shutdown {
stream_shutdown.send_replace(true);
}
tracing::info!("gRPC server shutting down");
})
.await;
monitor.abort();
result?;
Ok(())
+104
View File
@@ -6,6 +6,10 @@ use crate::bridge::TerminalError;
use std::collections::HashMap;
use tonic::Code;
// The metadata wire test starts a real server that reads SGLANG_TONIC_PAYLOAD.
// Keep that read separate from the environment override test below.
static PAYLOAD_ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[test]
fn openai_status_code_uses_forwarded_status_when_present() {
let meta_info = HashMap::from([(String::from("status_code"), String::from("429"))]);
@@ -42,6 +46,7 @@ fn terminal_error_status_maps_abort_to_cancelled() {
// serial test so they don't race each other under `cargo test`'s default parallelism.
#[test]
fn resolve_max_message_size_honors_env_var() {
let _guard = PAYLOAD_ENV_LOCK.blocking_lock();
const VAR: &str = "SGLANG_TONIC_PAYLOAD";
// Unset → default.
@@ -73,3 +78,102 @@ fn resolve_max_message_size_honors_env_var() {
std::env::remove_var(VAR);
}
}
#[tokio::test]
async fn follower_metadata_server_exposes_only_server_info_and_shuts_down() {
use sglang_grpc_types::sglang::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse};
use tonic::codec::ProstCodec;
let _guard = PAYLOAD_ENV_LOCK.lock().await;
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
listener.set_nonblocking(true).unwrap();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let server_info_json = serde_json::json!({
"node_rank": 1,
"nnodes": 2,
"dp_size": 8,
"kv_event_sources": [{
"dp_rank": 4,
"endpoint": "tcp://127.0.0.1:5557",
"topic": "kv-events",
"block_size": 16
}]
})
.to_string();
// Exercise the metadata service without the Python-facing startup wrapper,
// which requires libpython when linked into a standalone test executable.
let shutdown = std::sync::Arc::new(tokio::sync::Notify::new());
let server_shutdown = shutdown.clone();
let service = super::MetadataService {
server_info_json: server_info_json.clone(),
};
let handle = std::thread::spawn(move || {
runtime
.block_on(super::serve_grpc(listener, service, server_shutdown, None))
.unwrap();
});
assert!(!handle.is_finished());
let channel = tonic::transport::Endpoint::from_shared(format!("http://{addr}"))
.unwrap()
.connect_timeout(std::time::Duration::from_secs(5))
.timeout(std::time::Duration::from_secs(5))
.connect()
.await
.unwrap();
let mut client = tonic::client::Grpc::new(channel);
client.ready().await.unwrap();
let response: tonic::Response<GetServerInfoResponse> = client
.unary(
tonic::Request::new(GetServerInfoRequest {}),
tonic::codegen::http::uri::PathAndQuery::from_static(
"/sglang.runtime.v1.SglangService/GetServerInfo",
),
ProstCodec::default(),
)
.await
.unwrap();
assert_eq!(response.into_inner().json_info, server_info_json);
// These cover generation (including streaming), model queries, health,
// and controls. The follower must never advertise an inference frontend.
for method in [
"Generate",
"ChatComplete",
"GetModelInfo",
"HealthCheck",
"FlushCache",
"Abort",
] {
client.ready().await.unwrap();
let result: Result<tonic::Response<GetServerInfoResponse>, _> = client
.unary(
tonic::Request::new(GetServerInfoRequest {}),
format!("/sglang.runtime.v1.SglangService/{method}")
.parse()
.unwrap(),
ProstCodec::default(),
)
.await;
assert_eq!(result.unwrap_err().code(), Code::Unimplemented, "{method}");
}
drop(client);
// Keep the client's runtime driving connection teardown while joining the
// server thread during graceful shutdown.
shutdown.notify_one();
tokio::time::timeout(
std::time::Duration::from_secs(5),
tokio::task::spawn_blocking(move || {
handle.join().unwrap();
}),
)
.await
.unwrap()
.unwrap();
assert!(std::net::TcpStream::connect(addr).is_err());
}
+5 -15
View File
@@ -87,12 +87,10 @@ fn unimplemented_rpc(name: &'static str) -> Status {
#[tonic::async_trait]
impl SglangService for GrpcService {
type TextGenerateStream = ResponseStream<proto::TextGenerateResponse>;
async fn text_generate(
&self,
request: Request<proto::TextGenerateRequest>,
) -> Result<Response<Self::TextGenerateStream>, Status> {
) -> Result<Response<ResponseStream<proto::TextGenerateResponse>>, Status> {
let request = convert::text_generate(
request.into_inner(),
self.config.preferred_sampling_params.as_ref(),
@@ -110,12 +108,10 @@ impl SglangService for GrpcService {
)))
}
type GenerateStream = ResponseStream<proto::GenerateResponse>;
async fn generate(
&self,
request: Request<proto::GenerateRequest>,
) -> Result<Response<Self::GenerateStream>, Status> {
) -> Result<Response<ResponseStream<proto::GenerateResponse>>, Status> {
let request = convert::generate(
request.into_inner(),
self.config.preferred_sampling_params.as_ref(),
@@ -228,12 +224,10 @@ impl SglangService for GrpcService {
Err(unimplemented_rpc("pause_generation"))
}
type WatchEngineStateStream = ResponseStream<proto::EngineStateSnapshot>;
async fn watch_engine_state(
&self,
_request: Request<proto::WatchEngineStateRequest>,
) -> Result<Response<Self::WatchEngineStateStream>, Status> {
) -> Result<Response<ResponseStream<proto::EngineStateSnapshot>>, Status> {
Err(unimplemented_rpc("watch_engine_state"))
}
@@ -244,21 +238,17 @@ impl SglangService for GrpcService {
Err(unimplemented_rpc("continue_generation"))
}
type ChatCompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
async fn chat_complete(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<Self::ChatCompleteStream>, Status> {
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
Err(unimplemented_rpc("chat_complete"))
}
type CompleteStream = ResponseStream<proto::OpenAiStreamChunk>;
async fn complete(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<Self::CompleteStream>, Status> {
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
Err(unimplemented_rpc("complete"))
}
@@ -7,7 +7,12 @@ the router can subscribe per replica (the `dp_size` it reads from
`/server_info`).
"""
import atexit
import json
import tempfile
import time
import unittest
import uuid
import msgspec
import zmq
@@ -16,7 +21,9 @@ from sglang.srt.disaggregation.kv_events import (
AllBlocksCleared,
BlockRemoved,
BlockStored,
EventPublisherFactory,
KVEventBatch,
NullEventPublisher,
StorageMedium,
ZmqEventPublisher,
resolve_load_pub_range,
@@ -31,6 +38,101 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
class TestLocalKvEventSource(CustomTestCase):
def _publisher(self, attn_dp_rank=0, **config):
publisher = EventPublisherFactory.create(
json.dumps({"publisher": "zmq", **config}), attn_dp_rank=attn_dp_rank
)
atexit.unregister(publisher.shutdown)
self.addCleanup(publisher.shutdown)
return publisher
def test_advertised_source_delivers_events(self):
for host, bind, topic in (("*", None, "kv"), ("127.0.0.1", True, "")):
with self.subTest(host=host):
with zmq.Context.instance().socket(zmq.PUB) as probe:
port = probe.bind_to_random_port("tcp://127.0.0.1")
publisher = self._publisher(
attn_dp_rank=4,
endpoint=f"tcp://{host}:{port - 4}",
bind=bind,
topic=topic,
)
source = publisher.describe_local_source(64)
self.assertEqual(
source,
dict(
dp_rank=4,
endpoint=f"tcp://127.0.0.1:{port}",
topic=topic,
block_size=64,
),
)
self._assert_event_received(publisher, source)
def _assert_event_received(self, publisher, source):
with zmq.Context() as context, context.socket(zmq.SUB) as subscriber:
subscriber.setsockopt_string(zmq.SUBSCRIBE, source["topic"])
subscriber.connect(source["endpoint"])
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
publisher.publish(KVEventBatch(ts=1.0, events=[AllBlocksCleared()]))
if subscriber.poll(100):
topic, _, payload = subscriber.recv_multipart()
self.assertEqual(topic, source["topic"].encode())
batch = msgspec.msgpack.decode(payload, type=KVEventBatch)
self.assertEqual(batch.attn_dp_rank, source["dp_rank"])
self.assertEqual(batch.events, [AllBlocksCleared()])
break
else:
self.fail("No event received from the advertised local source")
def test_ipc_is_advertised_only_when_bound(self):
directory = tempfile.TemporaryDirectory(prefix="kv-", dir="/tmp")
self.addCleanup(directory.cleanup)
endpoint = f"ipc://{directory.name}/events"
for bind in (None, False):
with self.subTest(bind=bind):
publisher = self._publisher(endpoint=endpoint, bind=bind)
source = publisher.describe_local_source(64)
if bind is False:
self.assertIsNone(source)
else:
self.assertEqual(source["endpoint"], endpoint)
def test_ephemeral_bind_and_replay_report_resolved_ports(self):
publisher = self._publisher(
attn_dp_rank=0,
endpoint="tcp://0.0.0.0:0",
replay_endpoint="tcp://*:0",
)
source = publisher.describe_local_source(64)
self.assertEqual(
source["endpoint"],
publisher._pub.getsockopt_string(zmq.LAST_ENDPOINT).replace(
"0.0.0.0", "127.0.0.1"
),
)
self.assertEqual(
source["replay_endpoint"],
publisher._replay.getsockopt_string(zmq.LAST_ENDPOINT).replace(
"0.0.0.0", "127.0.0.1"
),
)
self.assertNotEqual(source["endpoint"], source["replay_endpoint"])
self.assertFalse(source["endpoint"].endswith(":0"))
def test_non_subscribable_publishers_are_not_advertised(self):
for endpoint in (
"tcp://127.0.0.1:5557", # Connect-style PUB, not a listening source.
f"inproc://kv-source-{uuid.uuid4().hex}", # Same-process only.
):
with self.subTest(endpoint=endpoint):
publisher = self._publisher(attn_dp_rank=0, endpoint=endpoint)
self.assertIsNone(publisher.describe_local_source(64))
self.assertIsNone(NullEventPublisher().describe_local_source(64))
class TestResolveLoadPubRange(CustomTestCase):
"""The single source of truth both the bind and /server_info route through."""
@@ -24,16 +24,19 @@ import asyncio
import json
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import msgspec
import msgspec.structs
from sglang.srt.arg_groups.validation_hook import check_load_publish_args
from sglang.srt.entrypoints import http_server
from sglang.srt.entrypoints.grpc_bridge import RuntimeHandle
from sglang.srt.lora.lora_registry import LoRARef
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import common
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -147,6 +150,46 @@ def _call_server_info_with(
reset_context()
class TestServerInfoTransportParity(CustomTestCase):
def test_http_and_both_grpc_roles_share_startup_metadata(self):
for config in (None, '{"publisher": "zmq", "endpoint": "tcp://*:5557"}'):
with self.subTest(kv_events_config=config):
args = ServerArgs(
model_path="dummy", kv_events_config=config, page_size=64
)
http_info = _call_server_info_with(args)
self.assertEqual(http_info["frontend"], "python")
for field in ("startup_time", "internal_states", "version", "frontend"):
self.assertIn(field, http_info)
http_info.pop(field)
bridge = RuntimeHandle.__new__(RuntimeHandle)
bridge.tokenizer_manager = SimpleNamespace(server_args=args)
bridge.scheduler_info = {"max_req_input_len": 1024}
leader_json = bridge.get_server_info()
self.assertEqual(
json.loads(leader_json), json.loads(json.dumps(http_info))
)
with (
patch.object(
common,
"get_serving",
return_value=SimpleNamespace(
host="127.0.0.1",
grpc_port=50051,
smg_grpc_mode=False,
grpc_mode=False,
),
),
patch("sglang.srt.rust_extensions.load_rust_extension") as load,
):
common.start_follower_grpc_server(args, bridge.scheduler_info)
load.return_value.start_metadata_server.assert_called_once_with(
host="127.0.0.1", port=50051, server_info_json=leader_json
)
class TestServerInfoKvEventsField(CustomTestCase):
"""The new `kv_events` field is wired correctly across the full
`_build_kv_events_block` input matrix.
@@ -11,10 +11,12 @@ is exercised as the real method, no mock. The refresh-throttle tests inject
`load_snapshot_reader` and `_last_refresh_time` on top of those.
"""
import threading
import time
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import msgspec.structs
@@ -29,6 +31,7 @@ from sglang.srt.managers.data_parallel_controller import (
LoadBalanceMethod,
)
from sglang.srt.managers.load_snapshot import LoadSnapshot
from sglang.srt.runtime_context import get_context
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
@@ -64,6 +67,97 @@ def _req(routed_dp_rank=None, bootstrap_room=None, input_ids=None):
)
class TestLocalKvEventSources(CustomTestCase):
@staticmethod
def _source(rank, block_size=64):
return {
"dp_rank": rank,
"endpoint": f"tcp://127.0.0.1:{5557 + rank}",
"topic": "kv",
"block_size": block_size,
}
def test_scheduler_reports_logical_block_size_or_no_source(self):
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.managers.scheduler_components.kv_events_publisher import (
SchedulerKvEventsPublisher,
)
component = SchedulerKvEventsPublisher.__new__(SchedulerKvEventsPublisher)
scheduler = Scheduler.__new__(Scheduler)
scheduler.max_total_num_tokens = 64
scheduler.max_req_input_len = 32
scheduler.startup_time = {}
scheduler.page_size = 64
scheduler.kv_events_publisher = component
publisher = MagicMock()
publisher.describe_local_source.return_value = self._source(4, 256)
for owned_publisher, expected in (
(publisher, [self._source(4, 256)]),
(None, []),
):
with (
self.subTest(has_publisher=owned_publisher is not None),
get_context().override_server_args(grpc_port=50051, dcp_size=4),
):
component.kv_event_publisher = owned_publisher
self.assertEqual(
scheduler.get_init_info()["kv_event_sources"], expected
)
publisher.describe_local_source.assert_called_once_with(256)
def test_controller_collects_all_scheduler_sources(self):
from sglang.srt.managers import data_parallel_controller as module
controller = DataParallelController.__new__(DataParallelController)
controller.env_lock = threading.Lock()
controller.scheduler_procs = []
controller.local_kv_event_sources = []
controller.run_scheduler_process_func = MagicMock()
# TP8/DP4 over two nodes: this follower's four schedulers own ranks 2,3.
readers = [
MagicMock(
recv=MagicMock(
return_value={
"max_total_num_tokens": 64,
"max_req_input_len": 32,
"kv_event_sources": sources,
}
)
)
for sources in ([self._source(2)], [], [self._source(3)], [])
]
with (
get_context().override_server_args(
tp_size=8,
dp_size=4,
enable_dp_attention=True,
nnodes=2,
node_rank=1,
pp_size=1,
attn_cp_size=1,
),
patch.object(module.mp, "Process"),
patch.object(
module.mp,
"Pipe",
side_effect=[(reader, MagicMock()) for reader in readers],
),
patch.object(module.PortArgs, "init_new", return_value=MagicMock()),
patch.object(module.TorchMemorySaverAdapter, "create"),
patch.object(module.numa_utils, "configure_subprocess"),
patch.object(
module, "maybe_reindex_device_id", return_value=nullcontext(0)
),
):
controller.launch_tensor_parallel_group(
MagicMock(), MagicMock(), 0, None, [7000, 7001, 7002, 7003]
)
self.assertEqual(
controller.local_kv_event_sources, [self._source(2), self._source(3)]
)
class TestDPBudgetUpdateBudget(CustomTestCase):
def test_maps_running_plus_waiting_to_total_requests(self):
budget = DPBudget(num_dp_ranks=2)