From fd62ee84f71fcea88a35bd507236b8fc5a108696 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Wed, 7 Oct 2026 07:39:01 +0800 Subject: [PATCH] feat(grpc): expose follower metadata and node-local KV sources (#39659) Signed-off-by: jain-ria Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com> --- proto/build.rs | 3 + .../sglang/srt/arg_groups/fields/serving.py | 3 +- python/sglang/srt/disaggregation/kv_events.py | 89 +++++++++++-- python/sglang/srt/entrypoints/engine.py | 52 ++++++-- python/sglang/srt/entrypoints/grpc_bridge.py | 28 ++--- python/sglang/srt/entrypoints/http_server.py | 10 +- .../srt/managers/data_parallel_controller.py | 33 +++-- python/sglang/srt/managers/scheduler.py | 8 ++ .../kv_events_publisher.py | 6 + python/sglang/srt/utils/common.py | 38 ++++++ rust/sglang-grpc/src/lib.rs | 119 ++++++++++++------ rust/sglang-grpc/src/server.rs | 79 ++++++++---- rust/sglang-grpc/src/server/tests.rs | 104 +++++++++++++++ rust/sglang-server/src/grpc/mod.rs | 20 +-- .../unit/disaggregation/test_kv_events.py | 102 +++++++++++++++ .../unit/entrypoints/test_server_info.py | 43 +++++++ .../managers/test_data_parallel_controller.py | 96 +++++++++++++- 17 files changed, 703 insertions(+), 130 deletions(-) diff --git a/proto/build.rs b/proto/build.rs index c095519802..c0f389ed9a 100644 --- a/proto/build.rs +++ b/proto/build.rs @@ -13,6 +13,9 @@ fn main() -> Result<(), Box> { tonic_build::configure() .build_server(true) .build_client(true) + // The follower implements only GetServerInfo; all other RPCs return + // UNIMPLEMENTED without constructing the inference bridge. + .generate_default_stubs(true) .protoc_arg("--experimental_allow_proto3_optional") .compile_protos(&[proto_path], &["."])?; diff --git a/python/sglang/srt/arg_groups/fields/serving.py b/python/sglang/srt/arg_groups/fields/serving.py index 3a00b879c9..f1b828ee05 100644 --- a/python/sglang/srt/arg_groups/fields/serving.py +++ b/python/sglang/srt/arg_groups/fields/serving.py @@ -80,7 +80,8 @@ class Serving(msgspec.Struct): Optional[int], "Port for the native gRPC server, started alongside HTTP. Setting this " "(or SGLANG_GRPC_PORT) enables the native gRPC server; it is off by " - "default. In legacy --smg-grpc-mode this is the SMG server port and " + "default. Follower nodes expose GetServerInfo only. " + "In legacy --smg-grpc-mode this is the SMG server port and " "defaults to --port + 10000.", ] = None grpc_response_timeout_secs: A[ diff --git a/python/sglang/srt/disaggregation/kv_events.py b/python/sglang/srt/disaggregation/kv_events.py index cbef506f3b..36013d4ef6 100644 --- a/python/sglang/srt/disaggregation/kv_events.py +++ b/python/sglang/srt/disaggregation/kv_events.py @@ -20,6 +20,7 @@ KV caching events import atexit import enum import logging +import os import queue import threading import time @@ -329,6 +330,10 @@ class EventPublisher(ABC): def shutdown(self) -> None: """Shutdown the publisher.""" + def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]: + """Describe a source accessible to a separate local process, if any.""" + return None + class NullEventPublisher(EventPublisher): """No-op implementation (default when disabled).""" @@ -349,7 +354,7 @@ class ZmqEventPublisher(EventPublisher): ---------- endpoint: PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to - connect. + connect, unless overridden by ``bind``. replay_endpoint: Optional ROUTER address for replay requests. When given, subscribers can request missed batches by sending the starting sequence number as an @@ -362,6 +367,9 @@ class ZmqEventPublisher(EventPublisher): Maximum number of events to buffer in memory. topic: Topic to publish events to. + bind: + Whether to bind the PUB socket. When unset, preserve the endpoint-based + heuristic: bind wildcard, IPC and inproc addresses; connect otherwise. """ SHUTDOWN_TIMEOUT: float = 1.0 @@ -376,6 +384,7 @@ class ZmqEventPublisher(EventPublisher): hwm: int = 100_000, max_queue_size: int = 100_000, topic: str = "", + bind: Optional[bool] = None, ) -> None: # Storage self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size) @@ -384,9 +393,12 @@ class ZmqEventPublisher(EventPublisher): # ZMQ sockets self._ctx = zmq.Context.instance() self._pub: Optional[zmq.Socket] = None + self._local_pub_endpoint: Optional[str] = None + self._local_replay_endpoint: Optional[str] = None self._replay: Optional[zmq.Socket] = None self._dp_rank = attn_dp_rank self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank) + self._bind = bind self._replay_endpoint = self.offset_endpoint_port( replay_endpoint, self._dp_rank ) @@ -415,6 +427,43 @@ class ZmqEventPublisher(EventPublisher): events.attn_dp_rank = self._dp_rank self._event_queue.put(events) + def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]: + """Report the bound socket, not a port reconstructed from global DP size. + + Connect-style publishers and inproc sockets have no subscribable local + endpoint. They keep working as before, but are not advertised to external + processes. Wildcard binds are reachable on loopback from this node. + """ + + def local_endpoint(endpoint: Optional[str]) -> Optional[str]: + if endpoint is None: + return None + if endpoint.startswith("ipc://"): + return endpoint + if parse_advertisable_tcp(endpoint) is None: + return None + address = NetworkAddress.parse(endpoint[len("tcp://") :]) + host = address.host + if host in ("*", "0.0.0.0"): + host = "127.0.0.1" + elif host == "::": + host = "::1" + return NetworkAddress(host, address.port).to_tcp() + + endpoint = local_endpoint(self._local_pub_endpoint) + if endpoint is None: + return None + source = { + "dp_rank": self._dp_rank, + "endpoint": endpoint, + "topic": self._topic_bytes.decode("utf-8"), + "block_size": block_size, + } + replay_endpoint = local_endpoint(self._local_replay_endpoint) + if replay_endpoint is not None: + source["replay_endpoint"] = replay_endpoint + return source + def shutdown(self) -> None: """Stop the publisher thread and clean up resources.""" self._running = False @@ -453,23 +502,27 @@ class ZmqEventPublisher(EventPublisher): if self._pub is None: self._pub = self._ctx.socket(zmq.PUB) self._pub.set_hwm(self._hwm) - # Heuristic: bind if wildcard / * present, else connect. + # Default heuristic: bind if wildcard / * present, else connect. # bind stable, connect volatile convention. # ``0.0.0.0`` is the IPv4 bind-all wildcard alongside ``*`` # and ``::``; ``/server_info`` advertises it as a wildcard, # so the publisher must bind it for the advertised endpoint # to actually be listening. - if ( - "*" in self._endpoint - or "::" in self._endpoint - or "0.0.0.0" in self._endpoint - or self._endpoint.startswith("ipc://") - or self._endpoint.startswith("inproc://") - ): + should_bind = self._bind + if should_bind is None: + should_bind = ( + "*" in self._endpoint + or "::" in self._endpoint + or "0.0.0.0" in self._endpoint + or self._endpoint.startswith("ipc://") + or self._endpoint.startswith("inproc://") + ) + if should_bind: logger.debug( f"ZmqEventPublisher socket publisher_endpoint bind to {self._endpoint}" ) self._pub.bind(self._endpoint) + self._local_pub_endpoint = self._bound_endpoint(self._pub) else: self._pub.connect(self._endpoint) @@ -483,6 +536,19 @@ class ZmqEventPublisher(EventPublisher): f"ZmqEventPublisher socket replay_endpoint bind to {self._replay_endpoint}" ) self._replay.bind(self._replay_endpoint) + self._local_replay_endpoint = self._bound_endpoint(self._replay) + + @staticmethod + def _bound_endpoint(socket: zmq.Socket) -> str: + endpoint = socket.getsockopt_string(zmq.LAST_ENDPOINT) + if endpoint.startswith("ipc://"): + path = endpoint[len("ipc://") :] + # Capture the publisher's working directory at bind time so a + # separate process can use the address. Abstract IPC names are + # independent of the filesystem and must remain unchanged. + if not path.startswith("@"): + endpoint = f"ipc://{os.path.abspath(path)}" + return endpoint def _publisher_thread(self) -> None: """Background thread that processes the event queue.""" @@ -589,6 +655,11 @@ class KVEventsConfig(BaseModel): """The zmq endpoint to use for publishing kv events. """ + bind: Optional[bool] = None + """Bind (true) or connect (false) the PUB socket. When unset, infer from + the endpoint as before. Use true to bind a concrete address such as loopback. + """ + replay_endpoint: Optional[str] = None """The zmq endpoint to use for replaying kv events. """ diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 47bad3e6ee..fb6db42920 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -140,6 +140,7 @@ from sglang.srt.utils import ( numa_utils, set_prometheus_multiproc_dir, set_ulimit, + start_follower_grpc_server, ) from sglang.srt.utils.msgspec_utils import msgspec_to_builtins from sglang.srt.utils.network import ( @@ -173,6 +174,12 @@ class SchedulerInitResult: wait_for_ready: Callable[[], None] = lambda: None block_until_scheduler_exits: Callable[[], None] = lambda: None engine_info_bootstrap_server: Optional[Any] = None + grpc_server: Optional[Any] = None + + def stop_grpc_server(self) -> None: + if self.grpc_server is not None: + self.grpc_server.shutdown() + self.grpc_server = None def init_tokenizer_manager( @@ -951,6 +958,17 @@ class Engine(EngineScoreMixin, EngineBase): def wait_for_ready(): infos = _wait_for_scheduler_ready(scheduler_pipe_readers, scheduler_procs) + if any("kv_event_sources" in info for info in infos): + # Both gRPC entrypoints consume the first scheduler info. Keep + # the sources from every local scheduler, not just the first. + infos[0]["kv_event_sources"] = sorted( + ( + source + for info in infos + for source in info.get("kv_event_sources", []) + ), + key=lambda source: source["dp_rank"], + ) scheduler_infos.extend(infos) if use_dp_controller: for info in infos: @@ -1173,6 +1191,18 @@ class Engine(EngineScoreMixin, EngineBase): # Non-zero-rank nodes do not run tokenizer processes. scheduler_init_result.wait_for_ready() + try: + scheduler_init_result.grpc_server = start_follower_grpc_server( + server_args, scheduler_init_result.scheduler_infos[0] + ) + except BaseException: + # Engine.__init__ has not received these handles yet. Do not + # leave GPU workers behind if binding the metadata port fails. + for proc in scheduler_procs or []: + kill_process_tree(proc.pid, wait_timeout=60) + cls._terminate_weight_cache_daemons(weight_cache_daemon_procs) + raise + if os.getenv("SGLANG_BLOCK_NONZERO_RANK_CHILDREN") == "0": # When using `Engine` as a Python API, we don't want to block here. return ( @@ -1202,14 +1232,16 @@ class Engine(EngineScoreMixin, EngineBase): rust_server_owns_base_port = ( envs.SGLANG_RUST_SERVER.get() and node_hosts_rust_server() ) - if not rust_server_owns_base_port: - launch_dummy_health_check_server( - get_serving().host, - get_serving().port, - get_observability().enable_metrics, - ) - - scheduler_init_result.block_until_scheduler_exits() + try: + if not rust_server_owns_base_port: + launch_dummy_health_check_server( + get_serving().host, + get_serving().port, + get_observability().enable_metrics, + ) + scheduler_init_result.block_until_scheduler_exits() + finally: + scheduler_init_result.stop_grpc_server() return ( None, None, @@ -1384,6 +1416,9 @@ class Engine(EngineScoreMixin, EngineBase): pass self._multi_tokenizer_shm = None try: + scheduler_init_result = getattr(self, "_scheduler_init_result", None) + if scheduler_init_result is not None: + scheduler_init_result.stop_grpc_server() if ( self.tokenizer_manager is not None and self.tokenizer_manager._subprocess_watchdog is not None @@ -1789,7 +1824,6 @@ class Engine(EngineScoreMixin, EngineBase): def _set_envs_and_config(server_args: ServerArgs): - cfg = resolving_view(server_args) # Set global environments # MNNVL fabric (GB200/GB300) multi-node: cross-node NVLink needs NCCL's diff --git a/python/sglang/srt/entrypoints/grpc_bridge.py b/python/sglang/srt/entrypoints/grpc_bridge.py index e8e9c3073a..6fb4acf220 100644 --- a/python/sglang/srt/entrypoints/grpc_bridge.py +++ b/python/sglang/srt/entrypoints/grpc_bridge.py @@ -16,11 +16,8 @@ from pydantic import ValidationError from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan -from sglang.srt.runtime_context import ( - describe_kv_events_publisher, - get_lora, - get_serving, -) +from sglang.srt.runtime_context import get_lora, get_serving +from sglang.srt.utils.common import build_server_info from sglang.srt.utils.msgspec_utils import msgspec_to_builtins logger = logging.getLogger(__name__) @@ -238,9 +235,7 @@ class RuntimeHandle: return self._openai_serving_classes from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat - from sglang.srt.entrypoints.openai.serving_classify import ( - OpenAIServingClassify, - ) + from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify from sglang.srt.entrypoints.openai.serving_completions import ( OpenAIServingCompletion, ) @@ -430,17 +425,14 @@ class RuntimeHandle: return json.dumps(result, default=str) def get_server_info(self) -> str: - result: Dict[str, Any] = self.tokenizer_manager.server_args.resolved_dict() - # `resolved_dict` answers with what resolution decided; the launch - # command answers with what was asked for, and the two are not - # derivable from each other. The HTTP and in-process readbacks both - # carry it, so this one does too. - result["launch_command"] = self.tokenizer_manager.server_args.launch_command - result.update(self.scheduler_info) - result["kv_events"] = describe_kv_events_publisher( - self.tokenizer_manager.server_args + return json.dumps( + msgspec_to_builtins( + build_server_info( + self.tokenizer_manager.server_args, self.scheduler_info + ) + ), + default=str, ) - return json.dumps(msgspec_to_builtins(result), default=str) def health_check(self) -> bool: from sglang.srt.managers.tokenizer_manager import ServerStatus diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 587b3e782f..0fb6f4813a 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -177,6 +177,7 @@ from sglang.srt.server_args import PortArgs, ServerArgs from sglang.srt.utils import ( add_prometheus_middleware, add_prometheus_track_response_middleware, + build_server_info, delete_directory, get_bool_env_var, is_mps, @@ -502,7 +503,6 @@ app.include_router(v1_loads_router) from sglang.srt.arg_groups.serving_hook import ssl_verify_of from sglang.srt.entrypoints.elastic_ep import router as elastic_ep_router from sglang.srt.runtime_context import ( - describe_kv_events_publisher, get_disagg, get_exec, get_lora, @@ -859,17 +859,11 @@ async def server_info(): return msgspec_to_builtins( { - **server_args.resolved_dict(), - "launch_command": server_args.launch_command, - **_global_state.scheduler_info, + **build_server_info(server_args, _global_state.scheduler_info), "startup_time": _global_state.tokenizer_manager.startup_time, "internal_states": internal_states, "version": __version__, "frontend": "python", - # Structured KV-event publisher descriptor for KV-aware routers. - # `None` when publishing is disabled or misconfigured; see - # `runtime_context.describe_kv_events_publisher` for the contract. - "kv_events": describe_kv_events_publisher(server_args), } ) diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 0396ab2c11..6e08c27e6e 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -200,6 +200,7 @@ class DataParallelController: # Launch data parallel workers self.scheduler_procs = [] + self.local_kv_event_sources = [] self.workers: list[zmq.Socket | None] = [None] * self.max_dp_size self.status: list[bool] = list(self.dp_active) self._active_workers: list[int] = list(range(self.launch_dp_size)) @@ -723,6 +724,15 @@ class DataParallelController: for i in range(len(scheduler_pipe_readers)): scheduler_info.append(scheduler_pipe_readers[i].recv()) + # Pure-DP TP groups launch concurrently. Their ready replies contain + # only publishers owned by schedulers on this node. + with self.env_lock: + self.local_kv_event_sources.extend( + source + for info in scheduler_info + for source in info.get("kv_event_sources", []) + ) + self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"] self.max_req_input_len = scheduler_info[0]["max_req_input_len"] self.startup_time = aggregate_scheduler_startup_times( @@ -836,15 +846,20 @@ def run_data_parallel_controller_process( scheduler_pids = [ proc.pid for proc in controller.scheduler_procs if proc is not None ] - pipe_writer.send( - { - "status": "ready", - "max_total_num_tokens": controller.max_total_num_tokens, - "max_req_input_len": controller.max_req_input_len, - "startup_time": controller.startup_time, - SCHEDULER_PIDS_ARG: scheduler_pids, - } - ) + init_info = { + "status": "ready", + "max_total_num_tokens": controller.max_total_num_tokens, + "max_req_input_len": controller.max_req_input_len, + "startup_time": controller.startup_time, + SCHEDULER_PIDS_ARG: scheduler_pids, + } + if get_serving().grpc_port is not None and not ( + get_serving().smg_grpc_mode or get_serving().grpc_mode + ): + init_info["kv_event_sources"] = sorted( + controller.local_kv_event_sources, key=lambda source: source["dp_rank"] + ) + pipe_writer.send(init_info) # The primary owns routing for the expanded scheduler set. if get_parallel().node_rank == 0 and not get_exec().moe.is_ep_scale_joiner: controller.event_loop() diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 60f6308213..0c254d7dc7 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1826,6 +1826,14 @@ class Scheduler( "startup_time": self.startup_time, } + if get_serving().grpc_port is not None and not ( + get_serving().smg_grpc_mode or get_serving().grpc_mode + ): + result_dict["kv_event_sources"] = ( + self.kv_events_publisher.local_kv_event_sources( + self.page_size * get_parallel().dcp_size + ) + ) return result_dict def release_host_resources(self) -> None: diff --git a/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py b/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py index 0bde503f4b..d236da1de9 100644 --- a/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py +++ b/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py @@ -89,6 +89,12 @@ class SchedulerKvEventsPublisher: if not self.send_metrics_from_scheduler.closed: sock_send(self.send_metrics_from_scheduler, kv_metrics) + def local_kv_event_sources(self, block_size: int) -> list[dict[str, Any]]: + if self.kv_event_publisher is None: + return [] + source = self.kv_event_publisher.describe_local_source(block_size) + return [source] if source is not None else [] + def publish_kv_events(self): if not self.enable_kv_cache_events: return diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index b6ba5788d6..7b289705d4 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -101,13 +101,16 @@ from sglang.srt.environ import envs from sglang.srt.observability.func_timer import enable_func_timer from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( + describe_kv_events_publisher, get_exec, get_flags, get_model, get_parallel, get_platform, + get_serving, get_spec, ) +from sglang.srt.utils.msgspec_utils import msgspec_to_builtins from sglang.srt.utils.video_decoder import _BACKEND, VideoDecoderWrapper if TYPE_CHECKING: @@ -3649,6 +3652,41 @@ def _configure_uvicorn_access_log_filter( filters_list.append(filter_name) +def build_server_info(server_args, scheduler_info: Dict[str, Any]) -> Dict[str, Any]: + """Build server metadata shared by HTTP and native gRPC. + + Callers add transport-specific fields and serialize the result. + """ + result = server_args.resolved_dict() + result["launch_command"] = server_args.launch_command + result.update(scheduler_info) + result["kv_events"] = describe_kv_events_publisher(server_args) + return result + + +def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]): + """Expose GetServerInfo only, without constructing a TokenizerManager. + + The snapshot is taken after scheduler readiness. This does not launch a + sidecar or enable inference/control RPCs. + """ + serving = get_serving() + if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode: + return None + + from sglang.srt.rust_extensions import load_rust_extension + + grpc_native = load_rust_extension("sglang.srt.rust_extensions._grpc") + return grpc_native.start_metadata_server( + host=serving.host, + port=serving.grpc_port, + server_info_json=json.dumps( + msgspec_to_builtins(build_server_info(server_args, scheduler_info)), + default=str, + ), + ) + + def launch_dummy_health_check_server(host, port, enable_metrics): import asyncio diff --git a/rust/sglang-grpc/src/lib.rs b/rust/sglang-grpc/src/lib.rs index 6a2b42d4ed..627708869e 100644 --- a/rust/sglang-grpc/src/lib.rs +++ b/rust/sglang-grpc/src/lib.rs @@ -36,6 +36,76 @@ impl GrpcServerHandle { } } +fn bind_listener(host: &str, port: u16) -> PyResult { + let addr: SocketAddr = format!("{}:{}", host, port) + .parse() + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {}", e)))?; + let listener = TcpListener::bind(addr).map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to bind gRPC server to {}: {}", + addr, e + )) + })?; + listener.set_nonblocking(true).map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to configure gRPC listener for {}: {}", + addr, e + )) + })?; + Ok(listener) +} + +fn start_server_thread( + rt: tokio::runtime::Runtime, + listener: TcpListener, + service: server::ServerMode, +) -> PyResult { + let shutdown = Arc::new(Notify::new()); + let shutdown_clone = shutdown.clone(); + let join_handle = std::thread::Builder::new() + .name("sglang-grpc".to_string()) + .spawn(move || { + if let Err(e) = rt.block_on(server::run_grpc_server(listener, service, shutdown_clone)) + { + tracing::error!("gRPC server exited with error: {}", e); + } + }) + .map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to spawn gRPC thread: {}", e)) + })?; + + Ok(GrpcServerHandle { + shutdown, + join_handle: Some(join_handle), + }) +} + +/// Start a follower server exposing only GetServerInfo, without an inference bridge. +#[pyfunction] +fn start_metadata_server( + host: String, + port: u16, + server_info_json: String, +) -> PyResult { + let listener = bind_listener(&host, port)?; + // Metadata is a startup snapshot; it needs neither Python callbacks nor a + // worker pool. The dedicated server thread drives this runtime directly. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|err| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to build Tokio runtime for gRPC metadata server: {}", + err + )) + })?; + start_server_thread( + rt, + listener, + server::ServerMode::Metadata(server::MetadataService { server_info_json }), + ) +} + struct TokenizerInfo { tokenizer_path: Option, tokenizer_mode: Option, @@ -160,9 +230,6 @@ fn start_server( ) .try_init(); - let addr: SocketAddr = format!("{}:{}", host, port) - .parse() - .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {}", e)))?; let worker_threads = worker_threads.max(1); let response_channel_capacity = if response_channel_capacity == 0 { tracing::warn!( @@ -183,18 +250,7 @@ fn start_server( response_timeout_secs }; let response_timeout = Duration::from_secs(response_timeout_secs); - let listener = TcpListener::bind(addr).map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!( - "Failed to bind gRPC server to {}: {}", - addr, e - )) - })?; - listener.set_nonblocking(true).map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!( - "Failed to configure gRPC listener for {}: {}", - addr, e - )) - })?; + let listener = bind_listener(&host, port)?; let tokenizer_info = extract_tokenizer_info(&runtime_handle)?; @@ -226,35 +282,20 @@ fn start_server( response_channel_capacity, tokio_handle, )); - let shutdown = Arc::new(Notify::new()); - let shutdown_clone = shutdown.clone(); - let bridge_clone = bridge.clone(); - - let join_handle = std::thread::Builder::new() - .name("sglang-grpc".to_string()) - .spawn(move || { - if let Err(e) = rt.block_on(server::run_grpc_server( - listener, - bridge_clone, - shutdown_clone, - response_timeout, - )) { - tracing::error!("gRPC server exited with error: {}", e); - } - }) - .map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to spawn gRPC thread: {}", e)) - })?; - - Ok(GrpcServerHandle { - shutdown, - join_handle: Some(join_handle), - }) + start_server_thread( + rt, + listener, + server::ServerMode::Inference { + bridge, + response_timeout, + }, + ) } #[pymodule] fn _grpc(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(start_server, m)?)?; + m.add_function(wrap_pyfunction!(start_metadata_server, m)?)?; m.add_class::()?; m.add_class::()?; Ok(()) diff --git a/rust/sglang-grpc/src/server.rs b/rust/sglang-grpc/src/server.rs index 3e72497ea2..ce5e5763df 100644 --- a/rust/sglang-grpc/src/server.rs +++ b/rust/sglang-grpc/src/server.rs @@ -26,6 +26,25 @@ pub struct SglangServiceImpl { stream_shutdown: watch::Receiver, } +/// A follower has no tokenizer manager or inference bridge. It exposes the same +/// discovery RPC as the leader, with a node-local startup snapshot. Generated +/// default handlers return UNIMPLEMENTED for every other RPC. +pub struct MetadataService { + pub server_info_json: String, +} + +#[tonic::async_trait] +impl proto::sglang_service_server::SglangService for MetadataService { + async fn get_server_info( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(proto::GetServerInfoResponse { + json_info: self.server_info_json.clone(), + })) + } +} + type StreamResult = Pin> + Send + 'static>>; pub const DEFAULT_RESPONSE_TIMEOUT_SECS: u64 = 300; @@ -300,12 +319,10 @@ fn openai_status_code(meta_info: &HashMap, default: i32) -> i32 impl proto::sglang_service_server::SglangService for SglangServiceImpl { // --- SGLang-native RPCs: TextGenerate / Generate --- - type TextGenerateStream = StreamResult; - async fn text_generate( &self, request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { let req = request.into_inner(); let rid = req .rid @@ -369,12 +386,10 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl { Ok(Response::new(Box::pin(stream))) } - type GenerateStream = StreamResult; - async fn generate( &self, request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { let req = request.into_inner(); let rid = req .rid @@ -645,12 +660,10 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl { Ok(Response::new(proto::HealthCheckResponse { healthy })) } - type WatchEngineStateStream = StreamResult; - async fn watch_engine_state( &self, _request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { let mut receiver = self.engine_state.subscribe(); let mut shutdown = self.stream_shutdown.clone(); let stream = async_stream::stream! { @@ -831,22 +844,18 @@ impl proto::sglang_service_server::SglangService for SglangServiceImpl { // --- OpenAI-compatible RPCs (JSON pass-through) --- - type ChatCompleteStream = StreamResult; - async fn chat_complete( &self, request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { self.openai_streaming_rpc(request, "submit_openai_chat") .await } - type CompleteStream = StreamResult; - async fn complete( &self, request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { self.openai_streaming_rpc(request, "submit_openai_complete") .await } @@ -1079,6 +1088,14 @@ async fn recv_json_response( } } +pub enum ServerMode { + Inference { + bridge: Arc, + response_timeout: Duration, + }, + Metadata(MetadataService), +} + /// Start the Tonic gRPC server on the given address. // // TODO(grpc-auth): this listener is currently unauthenticated. Before exposing @@ -1086,12 +1103,18 @@ async fn recv_json_response( // checks the HTTP server applies (see issue tracking gRPC auth parity). pub async fn run_grpc_server( listener: std::net::TcpListener, - bridge: Arc, + mode: ServerMode, shutdown: Arc, - response_timeout: Duration, ) -> Result<(), Box> { - let addr = listener.local_addr()?; - let listener = tokio::net::TcpListener::from_std(listener)?; + let (bridge, response_timeout) = match mode { + ServerMode::Metadata(service) => { + return serve_grpc(listener, service, shutdown, None).await; + } + ServerMode::Inference { + bridge, + response_timeout, + } => (bridge, response_timeout), + }; let (state_changed_tx, mut state_changed_rx) = tokio::sync::mpsc::channel(1); bridge.set_engine_state_changed_callback(state_changed_tx)?; let engine_state = EngineStatePublisher::new(bridge.clone()).await?; @@ -1112,6 +1135,19 @@ pub async fn run_grpc_server( } }); + let result = serve_grpc(listener, service, shutdown, Some(stream_shutdown_tx)).await; + monitor.abort(); + result +} + +async fn serve_grpc( + listener: std::net::TcpListener, + service: impl proto::sglang_service_server::SglangService, + shutdown: Arc, + stream_shutdown: Option>, +) -> Result<(), Box> { + let addr = listener.local_addr()?; + let listener = tokio::net::TcpListener::from_std(listener)?; let max_message_size = resolve_max_message_size(); let svc = proto::sglang_service_server::SglangServiceServer::new(service) .max_decoding_message_size(max_message_size) @@ -1123,11 +1159,12 @@ pub async fn run_grpc_server( .add_service(svc) .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async move { shutdown.notified().await; - stream_shutdown_tx.send_replace(true); + if let Some(stream_shutdown) = stream_shutdown { + stream_shutdown.send_replace(true); + } tracing::info!("gRPC server shutting down"); }) .await; - monitor.abort(); result?; Ok(()) diff --git a/rust/sglang-grpc/src/server/tests.rs b/rust/sglang-grpc/src/server/tests.rs index 11ecb68769..53b0b0e794 100644 --- a/rust/sglang-grpc/src/server/tests.rs +++ b/rust/sglang-grpc/src/server/tests.rs @@ -6,6 +6,10 @@ use crate::bridge::TerminalError; use std::collections::HashMap; use tonic::Code; +// The metadata wire test starts a real server that reads SGLANG_TONIC_PAYLOAD. +// Keep that read separate from the environment override test below. +static PAYLOAD_ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + #[test] fn openai_status_code_uses_forwarded_status_when_present() { let meta_info = HashMap::from([(String::from("status_code"), String::from("429"))]); @@ -42,6 +46,7 @@ fn terminal_error_status_maps_abort_to_cancelled() { // serial test so they don't race each other under `cargo test`'s default parallelism. #[test] fn resolve_max_message_size_honors_env_var() { + let _guard = PAYLOAD_ENV_LOCK.blocking_lock(); const VAR: &str = "SGLANG_TONIC_PAYLOAD"; // Unset → default. @@ -73,3 +78,102 @@ fn resolve_max_message_size_honors_env_var() { std::env::remove_var(VAR); } } + +#[tokio::test] +async fn follower_metadata_server_exposes_only_server_info_and_shuts_down() { + use sglang_grpc_types::sglang::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse}; + use tonic::codec::ProstCodec; + + let _guard = PAYLOAD_ENV_LOCK.lock().await; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + listener.set_nonblocking(true).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let server_info_json = serde_json::json!({ + "node_rank": 1, + "nnodes": 2, + "dp_size": 8, + "kv_event_sources": [{ + "dp_rank": 4, + "endpoint": "tcp://127.0.0.1:5557", + "topic": "kv-events", + "block_size": 16 + }] + }) + .to_string(); + // Exercise the metadata service without the Python-facing startup wrapper, + // which requires libpython when linked into a standalone test executable. + let shutdown = std::sync::Arc::new(tokio::sync::Notify::new()); + let server_shutdown = shutdown.clone(); + let service = super::MetadataService { + server_info_json: server_info_json.clone(), + }; + let handle = std::thread::spawn(move || { + runtime + .block_on(super::serve_grpc(listener, service, server_shutdown, None)) + .unwrap(); + }); + assert!(!handle.is_finished()); + + let channel = tonic::transport::Endpoint::from_shared(format!("http://{addr}")) + .unwrap() + .connect_timeout(std::time::Duration::from_secs(5)) + .timeout(std::time::Duration::from_secs(5)) + .connect() + .await + .unwrap(); + let mut client = tonic::client::Grpc::new(channel); + client.ready().await.unwrap(); + let response: tonic::Response = client + .unary( + tonic::Request::new(GetServerInfoRequest {}), + tonic::codegen::http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/GetServerInfo", + ), + ProstCodec::default(), + ) + .await + .unwrap(); + assert_eq!(response.into_inner().json_info, server_info_json); + + // These cover generation (including streaming), model queries, health, + // and controls. The follower must never advertise an inference frontend. + for method in [ + "Generate", + "ChatComplete", + "GetModelInfo", + "HealthCheck", + "FlushCache", + "Abort", + ] { + client.ready().await.unwrap(); + let result: Result, _> = client + .unary( + tonic::Request::new(GetServerInfoRequest {}), + format!("/sglang.runtime.v1.SglangService/{method}") + .parse() + .unwrap(), + ProstCodec::default(), + ) + .await; + assert_eq!(result.unwrap_err().code(), Code::Unimplemented, "{method}"); + } + drop(client); + // Keep the client's runtime driving connection teardown while joining the + // server thread during graceful shutdown. + shutdown.notify_one(); + tokio::time::timeout( + std::time::Duration::from_secs(5), + tokio::task::spawn_blocking(move || { + handle.join().unwrap(); + }), + ) + .await + .unwrap() + .unwrap(); + assert!(std::net::TcpStream::connect(addr).is_err()); +} diff --git a/rust/sglang-server/src/grpc/mod.rs b/rust/sglang-server/src/grpc/mod.rs index 1a3a5e5f16..963c9ebe19 100644 --- a/rust/sglang-server/src/grpc/mod.rs +++ b/rust/sglang-server/src/grpc/mod.rs @@ -87,12 +87,10 @@ fn unimplemented_rpc(name: &'static str) -> Status { #[tonic::async_trait] impl SglangService for GrpcService { - type TextGenerateStream = ResponseStream; - async fn text_generate( &self, request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { let request = convert::text_generate( request.into_inner(), self.config.preferred_sampling_params.as_ref(), @@ -110,12 +108,10 @@ impl SglangService for GrpcService { ))) } - type GenerateStream = ResponseStream; - async fn generate( &self, request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { let request = convert::generate( request.into_inner(), self.config.preferred_sampling_params.as_ref(), @@ -228,12 +224,10 @@ impl SglangService for GrpcService { Err(unimplemented_rpc("pause_generation")) } - type WatchEngineStateStream = ResponseStream; - async fn watch_engine_state( &self, _request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { Err(unimplemented_rpc("watch_engine_state")) } @@ -244,21 +238,17 @@ impl SglangService for GrpcService { Err(unimplemented_rpc("continue_generation")) } - type ChatCompleteStream = ResponseStream; - async fn chat_complete( &self, _request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { Err(unimplemented_rpc("chat_complete")) } - type CompleteStream = ResponseStream; - async fn complete( &self, _request: Request, - ) -> Result, Status> { + ) -> Result>, Status> { Err(unimplemented_rpc("complete")) } diff --git a/test/registered/unit/disaggregation/test_kv_events.py b/test/registered/unit/disaggregation/test_kv_events.py index 93dfb8156a..81b3e2d2d4 100644 --- a/test/registered/unit/disaggregation/test_kv_events.py +++ b/test/registered/unit/disaggregation/test_kv_events.py @@ -7,7 +7,12 @@ the router can subscribe per replica (the `dp_size` it reads from `/server_info`). """ +import atexit +import json +import tempfile +import time import unittest +import uuid import msgspec import zmq @@ -16,7 +21,9 @@ from sglang.srt.disaggregation.kv_events import ( AllBlocksCleared, BlockRemoved, BlockStored, + EventPublisherFactory, KVEventBatch, + NullEventPublisher, StorageMedium, ZmqEventPublisher, resolve_load_pub_range, @@ -31,6 +38,101 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") +class TestLocalKvEventSource(CustomTestCase): + def _publisher(self, attn_dp_rank=0, **config): + publisher = EventPublisherFactory.create( + json.dumps({"publisher": "zmq", **config}), attn_dp_rank=attn_dp_rank + ) + atexit.unregister(publisher.shutdown) + self.addCleanup(publisher.shutdown) + return publisher + + def test_advertised_source_delivers_events(self): + for host, bind, topic in (("*", None, "kv"), ("127.0.0.1", True, "")): + with self.subTest(host=host): + with zmq.Context.instance().socket(zmq.PUB) as probe: + port = probe.bind_to_random_port("tcp://127.0.0.1") + publisher = self._publisher( + attn_dp_rank=4, + endpoint=f"tcp://{host}:{port - 4}", + bind=bind, + topic=topic, + ) + source = publisher.describe_local_source(64) + self.assertEqual( + source, + dict( + dp_rank=4, + endpoint=f"tcp://127.0.0.1:{port}", + topic=topic, + block_size=64, + ), + ) + self._assert_event_received(publisher, source) + + def _assert_event_received(self, publisher, source): + with zmq.Context() as context, context.socket(zmq.SUB) as subscriber: + subscriber.setsockopt_string(zmq.SUBSCRIBE, source["topic"]) + subscriber.connect(source["endpoint"]) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + publisher.publish(KVEventBatch(ts=1.0, events=[AllBlocksCleared()])) + if subscriber.poll(100): + topic, _, payload = subscriber.recv_multipart() + self.assertEqual(topic, source["topic"].encode()) + batch = msgspec.msgpack.decode(payload, type=KVEventBatch) + self.assertEqual(batch.attn_dp_rank, source["dp_rank"]) + self.assertEqual(batch.events, [AllBlocksCleared()]) + break + else: + self.fail("No event received from the advertised local source") + + def test_ipc_is_advertised_only_when_bound(self): + directory = tempfile.TemporaryDirectory(prefix="kv-", dir="/tmp") + self.addCleanup(directory.cleanup) + endpoint = f"ipc://{directory.name}/events" + for bind in (None, False): + with self.subTest(bind=bind): + publisher = self._publisher(endpoint=endpoint, bind=bind) + source = publisher.describe_local_source(64) + if bind is False: + self.assertIsNone(source) + else: + self.assertEqual(source["endpoint"], endpoint) + + def test_ephemeral_bind_and_replay_report_resolved_ports(self): + publisher = self._publisher( + attn_dp_rank=0, + endpoint="tcp://0.0.0.0:0", + replay_endpoint="tcp://*:0", + ) + source = publisher.describe_local_source(64) + self.assertEqual( + source["endpoint"], + publisher._pub.getsockopt_string(zmq.LAST_ENDPOINT).replace( + "0.0.0.0", "127.0.0.1" + ), + ) + self.assertEqual( + source["replay_endpoint"], + publisher._replay.getsockopt_string(zmq.LAST_ENDPOINT).replace( + "0.0.0.0", "127.0.0.1" + ), + ) + self.assertNotEqual(source["endpoint"], source["replay_endpoint"]) + self.assertFalse(source["endpoint"].endswith(":0")) + + def test_non_subscribable_publishers_are_not_advertised(self): + for endpoint in ( + "tcp://127.0.0.1:5557", # Connect-style PUB, not a listening source. + f"inproc://kv-source-{uuid.uuid4().hex}", # Same-process only. + ): + with self.subTest(endpoint=endpoint): + publisher = self._publisher(attn_dp_rank=0, endpoint=endpoint) + self.assertIsNone(publisher.describe_local_source(64)) + self.assertIsNone(NullEventPublisher().describe_local_source(64)) + + class TestResolveLoadPubRange(CustomTestCase): """The single source of truth both the bind and /server_info route through.""" diff --git a/test/registered/unit/entrypoints/test_server_info.py b/test/registered/unit/entrypoints/test_server_info.py index f159dec053..0191383b4a 100644 --- a/test/registered/unit/entrypoints/test_server_info.py +++ b/test/registered/unit/entrypoints/test_server_info.py @@ -24,16 +24,19 @@ import asyncio import json import unittest from types import SimpleNamespace +from unittest.mock import patch import msgspec import msgspec.structs from sglang.srt.arg_groups.validation_hook import check_load_publish_args from sglang.srt.entrypoints import http_server +from sglang.srt.entrypoints.grpc_bridge import RuntimeHandle from sglang.srt.lora.lora_registry import LoRARef from sglang.srt.managers.tokenizer_manager import TokenizerManager from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import common from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -147,6 +150,46 @@ def _call_server_info_with( reset_context() +class TestServerInfoTransportParity(CustomTestCase): + def test_http_and_both_grpc_roles_share_startup_metadata(self): + for config in (None, '{"publisher": "zmq", "endpoint": "tcp://*:5557"}'): + with self.subTest(kv_events_config=config): + args = ServerArgs( + model_path="dummy", kv_events_config=config, page_size=64 + ) + http_info = _call_server_info_with(args) + self.assertEqual(http_info["frontend"], "python") + for field in ("startup_time", "internal_states", "version", "frontend"): + self.assertIn(field, http_info) + http_info.pop(field) + + bridge = RuntimeHandle.__new__(RuntimeHandle) + bridge.tokenizer_manager = SimpleNamespace(server_args=args) + bridge.scheduler_info = {"max_req_input_len": 1024} + leader_json = bridge.get_server_info() + self.assertEqual( + json.loads(leader_json), json.loads(json.dumps(http_info)) + ) + + with ( + patch.object( + common, + "get_serving", + return_value=SimpleNamespace( + host="127.0.0.1", + grpc_port=50051, + smg_grpc_mode=False, + grpc_mode=False, + ), + ), + patch("sglang.srt.rust_extensions.load_rust_extension") as load, + ): + common.start_follower_grpc_server(args, bridge.scheduler_info) + load.return_value.start_metadata_server.assert_called_once_with( + host="127.0.0.1", port=50051, server_info_json=leader_json + ) + + class TestServerInfoKvEventsField(CustomTestCase): """The new `kv_events` field is wired correctly across the full `_build_kv_events_block` input matrix. diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index 1e735c7c3d..04a0dc069c 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -11,10 +11,12 @@ is exercised as the real method, no mock. The refresh-throttle tests inject `load_snapshot_reader` and `_last_refresh_time` on top of those. """ +import threading import time import unittest +from contextlib import nullcontext from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import msgspec.structs @@ -29,6 +31,7 @@ from sglang.srt.managers.data_parallel_controller import ( LoadBalanceMethod, ) from sglang.srt.managers.load_snapshot import LoadSnapshot +from sglang.srt.runtime_context import get_context register_cpu_ci(est_time=12, suite="base-a-test-cpu") @@ -64,6 +67,97 @@ def _req(routed_dp_rank=None, bootstrap_room=None, input_ids=None): ) +class TestLocalKvEventSources(CustomTestCase): + @staticmethod + def _source(rank, block_size=64): + return { + "dp_rank": rank, + "endpoint": f"tcp://127.0.0.1:{5557 + rank}", + "topic": "kv", + "block_size": block_size, + } + + def test_scheduler_reports_logical_block_size_or_no_source(self): + from sglang.srt.managers.scheduler import Scheduler + from sglang.srt.managers.scheduler_components.kv_events_publisher import ( + SchedulerKvEventsPublisher, + ) + + component = SchedulerKvEventsPublisher.__new__(SchedulerKvEventsPublisher) + scheduler = Scheduler.__new__(Scheduler) + scheduler.max_total_num_tokens = 64 + scheduler.max_req_input_len = 32 + scheduler.startup_time = {} + scheduler.page_size = 64 + scheduler.kv_events_publisher = component + publisher = MagicMock() + publisher.describe_local_source.return_value = self._source(4, 256) + for owned_publisher, expected in ( + (publisher, [self._source(4, 256)]), + (None, []), + ): + with ( + self.subTest(has_publisher=owned_publisher is not None), + get_context().override_server_args(grpc_port=50051, dcp_size=4), + ): + component.kv_event_publisher = owned_publisher + self.assertEqual( + scheduler.get_init_info()["kv_event_sources"], expected + ) + publisher.describe_local_source.assert_called_once_with(256) + + def test_controller_collects_all_scheduler_sources(self): + from sglang.srt.managers import data_parallel_controller as module + + controller = DataParallelController.__new__(DataParallelController) + controller.env_lock = threading.Lock() + controller.scheduler_procs = [] + controller.local_kv_event_sources = [] + controller.run_scheduler_process_func = MagicMock() + # TP8/DP4 over two nodes: this follower's four schedulers own ranks 2,3. + readers = [ + MagicMock( + recv=MagicMock( + return_value={ + "max_total_num_tokens": 64, + "max_req_input_len": 32, + "kv_event_sources": sources, + } + ) + ) + for sources in ([self._source(2)], [], [self._source(3)], []) + ] + with ( + get_context().override_server_args( + tp_size=8, + dp_size=4, + enable_dp_attention=True, + nnodes=2, + node_rank=1, + pp_size=1, + attn_cp_size=1, + ), + patch.object(module.mp, "Process"), + patch.object( + module.mp, + "Pipe", + side_effect=[(reader, MagicMock()) for reader in readers], + ), + patch.object(module.PortArgs, "init_new", return_value=MagicMock()), + patch.object(module.TorchMemorySaverAdapter, "create"), + patch.object(module.numa_utils, "configure_subprocess"), + patch.object( + module, "maybe_reindex_device_id", return_value=nullcontext(0) + ), + ): + controller.launch_tensor_parallel_group( + MagicMock(), MagicMock(), 0, None, [7000, 7001, 7002, 7003] + ) + self.assertEqual( + controller.local_kv_event_sources, [self._source(2), self._source(3)] + ) + + class TestDPBudgetUpdateBudget(CustomTestCase): def test_maps_running_plus_waiting_to_total_requests(self): budget = DPBudget(num_dp_ranks=2)