[gRPC] Native server: launcher + HTTP + server args wiring (3/4) (#23508)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
801571e949
commit
3d2e7cc601
@@ -260,6 +260,8 @@ async def init_multi_tokenizer() -> ServerArgs:
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(fast_api_app: FastAPI):
|
||||
grpc_handle = None
|
||||
warmup_thread = None
|
||||
if getattr(fast_api_app, "is_single_tokenizer_mode", False):
|
||||
server_args = fast_api_app.server_args
|
||||
warmup_thread_kwargs = fast_api_app.warmup_thread_kwargs
|
||||
@@ -375,20 +377,38 @@ async def lifespan(fast_api_app: FastAPI):
|
||||
)
|
||||
logger.info("Warmup ended")
|
||||
|
||||
# Execute the general warmup
|
||||
warmup_thread = threading.Thread(
|
||||
target=_wait_and_warmup,
|
||||
kwargs=warmup_thread_kwargs,
|
||||
)
|
||||
warmup_thread.start()
|
||||
|
||||
# Start the HTTP server
|
||||
# Start the native gRPC server and warmup inside the try so a failure in
|
||||
# either still runs the finally cleanup below. Native gRPC is enabled via
|
||||
# --grpc-port / SGLANG_GRPC_PORT; only the single-tokenizer process is
|
||||
# gRPC-capable (__post_init__ rejects --tokenizer-worker-num > 1).
|
||||
try:
|
||||
if (
|
||||
getattr(fast_api_app, "is_single_tokenizer_mode", False)
|
||||
and server_args.grpc_port is not None
|
||||
and not (server_args.smg_grpc_mode or server_args.grpc_mode)
|
||||
):
|
||||
grpc_handle = _start_native_grpc_server_for_runtime(
|
||||
server_args=server_args,
|
||||
tokenizer_manager=_global_state.tokenizer_manager,
|
||||
template_manager=_global_state.template_manager,
|
||||
scheduler_info=_global_state.scheduler_info,
|
||||
)
|
||||
|
||||
# Execute the general warmup
|
||||
warmup_thread = threading.Thread(
|
||||
target=_wait_and_warmup,
|
||||
kwargs=warmup_thread_kwargs,
|
||||
)
|
||||
warmup_thread.start()
|
||||
|
||||
# Start the HTTP server
|
||||
yield
|
||||
finally:
|
||||
_shutdown_native_grpc_server(grpc_handle)
|
||||
if tool_server is not None and hasattr(tool_server, "aclose"):
|
||||
await tool_server.aclose()
|
||||
warmup_thread.join()
|
||||
if warmup_thread is not None:
|
||||
warmup_thread.join()
|
||||
|
||||
|
||||
# Fast API
|
||||
@@ -2476,6 +2496,51 @@ def _setup_and_run_http_server(
|
||||
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
|
||||
|
||||
|
||||
def _start_native_grpc_server_for_runtime(
|
||||
server_args,
|
||||
tokenizer_manager,
|
||||
template_manager,
|
||||
scheduler_info,
|
||||
):
|
||||
try:
|
||||
from sglang.srt.entrypoints.grpc_bridge import RuntimeHandle
|
||||
from sglang.srt.grpc import _core as grpc_native
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"Native gRPC extension (sglang.srt.grpc._core) not found in this wheel, "
|
||||
"but --grpc-port was set. The extension is built from "
|
||||
"rust/sglang-grpc/ via setuptools-rust during wheel build. Either "
|
||||
"install a wheel that includes the extension or unset --grpc-port."
|
||||
) from e
|
||||
|
||||
runtime_handle = RuntimeHandle(
|
||||
tokenizer_manager=tokenizer_manager,
|
||||
template_manager=template_manager,
|
||||
server_args=server_args,
|
||||
scheduler_info=scheduler_info or {},
|
||||
)
|
||||
|
||||
grpc_handle = grpc_native.start_server(
|
||||
host=server_args.host,
|
||||
port=server_args.grpc_port,
|
||||
runtime_handle=runtime_handle,
|
||||
worker_threads=server_args.grpc_worker_threads,
|
||||
)
|
||||
logger.info(
|
||||
f"Native gRPC server started on {server_args.host}:{server_args.grpc_port}"
|
||||
)
|
||||
return grpc_handle
|
||||
|
||||
|
||||
def _shutdown_native_grpc_server(grpc_handle) -> None:
|
||||
if grpc_handle is None:
|
||||
return
|
||||
try:
|
||||
grpc_handle.shutdown()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to shut down native gRPC server: {e}")
|
||||
|
||||
|
||||
def launch_server(
|
||||
server_args: ServerArgs,
|
||||
init_tokenizer_manager_func: Callable = init_tokenizer_manager,
|
||||
|
||||
Reference in New Issue
Block a user