[gRPC] Native server: launcher + HTTP + server args wiring (3/4) (#23508)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alex Nails
2026-07-07 14:57:25 -07:00
committed by GitHub
co-authored by Claude Opus 4.7
parent 801571e949
commit 3d2e7cc601
6 changed files with 286 additions and 40 deletions
+74 -9
View File
@@ -260,6 +260,8 @@ async def init_multi_tokenizer() -> ServerArgs:
@asynccontextmanager
async def lifespan(fast_api_app: FastAPI):
grpc_handle = None
warmup_thread = None
if getattr(fast_api_app, "is_single_tokenizer_mode", False):
server_args = fast_api_app.server_args
warmup_thread_kwargs = fast_api_app.warmup_thread_kwargs
@@ -375,20 +377,38 @@ async def lifespan(fast_api_app: FastAPI):
)
logger.info("Warmup ended")
# Execute the general warmup
warmup_thread = threading.Thread(
target=_wait_and_warmup,
kwargs=warmup_thread_kwargs,
)
warmup_thread.start()
# Start the HTTP server
# Start the native gRPC server and warmup inside the try so a failure in
# either still runs the finally cleanup below. Native gRPC is enabled via
# --grpc-port / SGLANG_GRPC_PORT; only the single-tokenizer process is
# gRPC-capable (__post_init__ rejects --tokenizer-worker-num > 1).
try:
if (
getattr(fast_api_app, "is_single_tokenizer_mode", False)
and server_args.grpc_port is not None
and not (server_args.smg_grpc_mode or server_args.grpc_mode)
):
grpc_handle = _start_native_grpc_server_for_runtime(
server_args=server_args,
tokenizer_manager=_global_state.tokenizer_manager,
template_manager=_global_state.template_manager,
scheduler_info=_global_state.scheduler_info,
)
# Execute the general warmup
warmup_thread = threading.Thread(
target=_wait_and_warmup,
kwargs=warmup_thread_kwargs,
)
warmup_thread.start()
# Start the HTTP server
yield
finally:
_shutdown_native_grpc_server(grpc_handle)
if tool_server is not None and hasattr(tool_server, "aclose"):
await tool_server.aclose()
warmup_thread.join()
if warmup_thread is not None:
warmup_thread.join()
# Fast API
@@ -2476,6 +2496,51 @@ def _setup_and_run_http_server(
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
def _start_native_grpc_server_for_runtime(
server_args,
tokenizer_manager,
template_manager,
scheduler_info,
):
try:
from sglang.srt.entrypoints.grpc_bridge import RuntimeHandle
from sglang.srt.grpc import _core as grpc_native
except ImportError as e:
raise RuntimeError(
"Native gRPC extension (sglang.srt.grpc._core) not found in this wheel, "
"but --grpc-port was set. The extension is built from "
"rust/sglang-grpc/ via setuptools-rust during wheel build. Either "
"install a wheel that includes the extension or unset --grpc-port."
) from e
runtime_handle = RuntimeHandle(
tokenizer_manager=tokenizer_manager,
template_manager=template_manager,
server_args=server_args,
scheduler_info=scheduler_info or {},
)
grpc_handle = grpc_native.start_server(
host=server_args.host,
port=server_args.grpc_port,
runtime_handle=runtime_handle,
worker_threads=server_args.grpc_worker_threads,
)
logger.info(
f"Native gRPC server started on {server_args.host}:{server_args.grpc_port}"
)
return grpc_handle
def _shutdown_native_grpc_server(grpc_handle) -> None:
if grpc_handle is None:
return
try:
grpc_handle.shutdown()
except Exception as e:
logger.warning(f"Failed to shut down native gRPC server: {e}")
def launch_server(
server_args: ServerArgs,
init_tokenizer_manager_func: Callable = init_tokenizer_manager,