minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

feat: add native gRPC sidecar module launcher (#31076)

Browse Source 
Signed-off-by: Ishan Dhanani <ishandhanani@gmail.com>
Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: Connor Carpenter <connorc@nvidia.com>
This commit is contained in:
ishandhanani
2026-07-22 06:39:08 -07:00
committed by GitHub
co-authored by Connor Carpenter
parent 74338e94f1
commit 21065bc862
4 changed files with 309 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/http_server.py
+10
View File
@@ -265,6 +265,7 @@ async def init_multi_tokenizer() -> ServerArgs:
@asynccontextmanager
async def lifespan(fast_api_app: FastAPI):
grpc_handle = None
sidecar = None
warmup_thread = None
if getattr(fast_api_app, "is_single_tokenizer_mode", False):
server_args = fast_api_app.server_args
@@ -397,6 +398,10 @@ async def lifespan(fast_api_app: FastAPI):
template_manager=_global_state.template_manager,
scheduler_info=_global_state.scheduler_info,
)
if server_args.sidecar is not None:
from sglang.srt.entrypoints.sidecar import start_sidecar
sidecar = start_sidecar(server_args)
# Execute the general warmup
warmup_thread = threading.Thread(
@@ -408,6 +413,11 @@ async def lifespan(fast_api_app: FastAPI):
# Start the HTTP server
yield
finally:
if sidecar is not None:
try:
sidecar.stop()
except Exception:
logger.exception("Failed to stop sidecar")
_shutdown_native_grpc_server(grpc_handle)
if tool_server is not None and hasattr(tool_server, "aclose"):
await tool_server.aclose()
python/sglang/srt/entrypoints/sidecar.py
+130
View File
@@ -0,0 +1,130 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Lifecycle management for an optional local native gRPC sidecar."""
import argparse
import importlib
import logging
import multiprocessing as mp
import os
from sglang.srt.utils.common import kill_itself_when_parent_died, kill_process_tree
from sglang.srt.utils.network import NetworkAddress
from sglang.srt.utils.watchdog import SubprocessWatchdog
logger = logging.getLogger(__name__)
SGLANG_GRPC_ENDPOINT_ENV = "SGLANG_GRPC_ENDPOINT"
_DEFAULT_SIDECAR_SHUTDOWN_TIMEOUT = 45.0
def _loopback_host(host: str) -> str:
if not host or host == "0.0.0.0":
return "127.0.0.1"
if host in ("::", "[::]"):
return "::1"
return host
def build_sidecar_endpoint(server_args) -> str:
return NetworkAddress(
_loopback_host(server_args.host), server_args.grpc_port
).to_url()
def _parse_sidecar_args(args: list[str] | None) -> tuple[list[str], float]:
parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
parser.add_argument(
"--sidecar-shutdown-timeout",
type=float,
default=_DEFAULT_SIDECAR_SHUTDOWN_TIMEOUT,
)
parsed, provider_args = parser.parse_known_args(args or [])
if parsed.sidecar_shutdown_timeout <= 0:
raise ValueError("--sidecar-shutdown-timeout must be greater than 0.")
return provider_args, parsed.sidecar_shutdown_timeout
def _run_sidecar(module_name: str, args: list[str], endpoint: str) -> None:
kill_itself_when_parent_died()
os.environ[SGLANG_GRPC_ENDPOINT_ENV] = endpoint
try:
main = getattr(importlib.import_module(module_name), "main")
except (AttributeError, ImportError) as e:
raise RuntimeError(
f"--sidecar requires importable module {module_name!r} "
"with a main(argv) function."
) from e
if not callable(main):
raise RuntimeError(
f"--sidecar requires module {module_name!r} to expose "
"a callable main(argv)."
)
main(args)
class Sidecar:
def __init__(
self,
proc,
module_name: str,
shutdown_timeout: float,
):
self.proc = proc
self.module_name = module_name
self.shutdown_timeout = shutdown_timeout
self._watchdog = SubprocessWatchdog(
processes=[proc], process_names=[module_name]
)
def start(self) -> None:
self.proc.start()
self._watchdog.start()
logger.info(
"Sidecar module %s started pid=%s",
self.module_name,
self.proc.pid,
)
def stop(self) -> None:
self._watchdog.stop()
if self.proc.is_alive():
self.proc.terminate()
self.proc.join(timeout=self.shutdown_timeout)
else:
self.proc.join(timeout=0)
if self.proc.is_alive():
logger.warning("Sidecar module did not terminate; killing process tree")
kill_process_tree(self.proc.pid, wait_timeout=self.shutdown_timeout)
def start_sidecar(server_args) -> Sidecar:
module_name = server_args.sidecar
assert module_name is not None
sidecar_args, shutdown_timeout = _parse_sidecar_args(server_args.sidecar_args)
endpoint = build_sidecar_endpoint(server_args)
proc = mp.get_context("spawn").Process(
name=f"sglang_sidecar_{module_name}",
target=_run_sidecar,
args=(module_name, sidecar_args, endpoint),
)
sidecar = Sidecar(
proc,
module_name,
shutdown_timeout=shutdown_timeout,
)
sidecar.start()
return sidecar
python/sglang/srt/server_args.py
+35
View File
@@ -1181,6 +1181,24 @@ class ServerArgs:
"defaults to --port + 10000.",
NS("serving"),
] = None
sidecar: A[
Optional[str],
"Start a locally managed sidecar against the native gRPC server. "
"The selected module must expose main(argv) and read the resolved "
"native gRPC endpoint from SGLANG_GRPC_ENDPOINT. Requires --grpc-port "
"or SGLANG_GRPC_PORT.",
NS("serving"),
] = None
sidecar_args: A[
Optional[List[str]],
Arg(
help="JSON array passed to the selected sidecar module's "
"main(argv) function. --sidecar-shutdown-timeout SECONDS is "
"consumed by SGLang.",
type_parser=json_list_type,
),
NS("serving"),
] = None
skip_server_warmup: A[bool, "If set, skip warmup.", NS("serving")] = False
warmups: A[
Optional[str],
@@ -3730,6 +3748,23 @@ class ServerArgs:
# Native gRPC is incompatible with launch paths it doesn't wire into.
# Legacy takes precedence over grpc_port, keeping re-runs idempotent.
native_grpc = self.grpc_port is not None and not legacy_grpc
if self.sidecar_args is not None:
if self.sidecar is None:
raise ValueError("--sidecar-args requires --sidecar.")
if not isinstance(self.sidecar_args, list) or not all(
isinstance(arg, str) for arg in self.sidecar_args
):
raise ValueError("--sidecar-args must be a JSON array of strings.")
if self.sidecar is not None:
if not self.sidecar.strip():
raise ValueError("--sidecar must not be empty.")
if legacy_grpc:
raise ValueError(
"--sidecar requires SGLang's native gRPC server; "
"it cannot be combined with --smg-grpc-mode/--grpc-mode."
)
if self.grpc_port is None:
raise ValueError("--sidecar requires --grpc-port or SGLANG_GRPC_PORT.")
if native_grpc:
if self.use_ray:
raise ValueError(
test/registered/unit/server_args/test_server_args.py
+134
View File
@@ -9,6 +9,13 @@ from unittest.mock import MagicMock, patch
import sglang.srt.server_args as server_args_module
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
from sglang.srt.entrypoints.sidecar import (
SGLANG_GRPC_ENDPOINT_ENV,
Sidecar,
_run_sidecar,
build_sidecar_endpoint,
start_sidecar,
)
from sglang.srt.environ import envs
from sglang.srt.layers.cp.base import is_cp_enabled, is_interleave
from sglang.srt.model_executor.cuda_graph_config import (
@@ -1536,6 +1543,133 @@ class TestGrpcServerArgs(CustomTestCase):
sa._handle_deprecated_args()
self.assertEqual(sa.grpc_port, 45000)
@staticmethod
def _sidecar_parser():
parser = server_args_module.argparse.ArgumentParser()
ServerArgs.add_cli_args(parser)
return parser
def test_sidecar_builds_loopback_grpc_endpoints(self):
self.assertEqual(
build_sidecar_endpoint(SimpleNamespace(host="0.0.0.0", grpc_port=50051)),
"http://127.0.0.1:50051",
)
self.assertEqual(
build_sidecar_endpoint(SimpleNamespace(host="::", grpc_port=50051)),
"http://[::1]:50051",
)
self.assertEqual(
build_sidecar_endpoint(SimpleNamespace(host="[::]", grpc_port=50051)),
"http://[::1]:50051",
)
def test_sidecar_args_parse_as_exact_json_argv(self):
argv = ["--flag", "value"]
parsed = self._sidecar_parser().parse_args(
["--model-path", "dummy", "--sidecar-args", json.dumps(argv)]
)
self.assertEqual(parsed.sidecar_args, argv)
def test_start_sidecar_passes_endpoint_and_provider_argv_separately(self):
server_args = SimpleNamespace(
sidecar="example.sidecar",
sidecar_args=[
"--sidecar-shutdown-timeout",
"42",
"--grpc-connections",
"2",
],
host="127.0.0.1",
grpc_port=50051,
)
with (
patch("sglang.srt.entrypoints.sidecar.mp.get_context") as get_context,
patch("sglang.srt.entrypoints.sidecar.Sidecar") as sidecar_class,
):
start_sidecar(server_args)
process_kwargs = get_context.return_value.Process.call_args.kwargs
self.assertEqual(process_kwargs["name"], "sglang_sidecar_example.sidecar")
self.assertEqual(process_kwargs["target"], _run_sidecar)
self.assertEqual(
process_kwargs["args"],
(
"example.sidecar",
["--grpc-connections", "2"],
"http://127.0.0.1:50051",
),
)
sidecar_class.assert_called_once_with(
get_context.return_value.Process.return_value,
"example.sidecar",
shutdown_timeout=42.0,
)
def test_sidecar_requires_native_grpc(self):
sa = self._args(sidecar="example.sidecar")
with self.assertRaisesRegex(ValueError, "requires --grpc-port"):
sa._handle_deprecated_args()
def test_sidecar_rejects_legacy_grpc(self):
sa = self._args(sidecar="example.sidecar", smg_grpc_mode=True)
with self.assertRaisesRegex(ValueError, "native gRPC server"):
sa._handle_deprecated_args()
def test_sidecar_rejects_empty_value(self):
sa = self._args(sidecar="", grpc_port=50051)
with self.assertRaisesRegex(ValueError, "must not be empty"):
sa._handle_deprecated_args()
def test_sidecar_sets_endpoint_env_before_import_and_calls_main(self):
main = MagicMock()
def import_module(module_name):
self.assertEqual(module_name, "example.sidecar")
self.assertEqual(
os.environ[SGLANG_GRPC_ENDPOINT_ENV],
"http://127.0.0.1:50051",
)
self.assertEqual(os.environ["DYN_NAMESPACE"], "pluh")
return SimpleNamespace(main=main)
with (
patch.dict(
os.environ,
{
SGLANG_GRPC_ENDPOINT_ENV: "http://stale.example:1",
"DYN_NAMESPACE": "pluh",
},
),
patch("sglang.srt.entrypoints.sidecar.kill_itself_when_parent_died"),
patch(
"sglang.srt.entrypoints.sidecar.importlib.import_module",
side_effect=import_module,
),
):
_run_sidecar(
"example.sidecar",
["--provider-flag", "value"],
"http://127.0.0.1:50051",
)
main.assert_called_once_with(["--provider-flag", "value"])
def test_sidecar_stop_uses_configured_shutdown_timeout(self):
proc = MagicMock(pid=1234)
proc.is_alive.side_effect = [True, True]
sidecar = Sidecar(
proc,
"example.sidecar",
shutdown_timeout=42.0,
)
with patch("sglang.srt.entrypoints.sidecar.kill_process_tree") as kill_tree:
sidecar.stop()
proc.terminate.assert_called_once_with()
proc.join.assert_called_once_with(timeout=42.0)
kill_tree.assert_called_once_with(1234, wait_timeout=42.0)
def test_legacy_smg_derives_grpc_port_from_http_port(self):
sa = self._args(port=30000, smg_grpc_mode=True)
sa._handle_deprecated_args()