Fix /health regression from early prebound socket listen (#19805)
This commit is contained in:
@@ -1978,6 +1978,7 @@ def launch_server(
|
|||||||
# Reserve the HTTP port before launching subprocesses to fail fast if port is unavailable.
|
# Reserve the HTTP port before launching subprocesses to fail fast if port is unavailable.
|
||||||
# This prevents wasting time loading models only to discover port conflicts later.
|
# This prevents wasting time loading models only to discover port conflicts later.
|
||||||
reserved_socket = _prebind_listening_socket(server_args.host, server_args.port)
|
reserved_socket = _prebind_listening_socket(server_args.host, server_args.port)
|
||||||
|
multi_tokenizer_args_shm = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Launch subprocesses
|
# Launch subprocesses
|
||||||
@@ -2052,6 +2053,10 @@ def launch_server(
|
|||||||
# Update logging configs
|
# Update logging configs
|
||||||
set_uvicorn_logging_configs(server_args)
|
set_uvicorn_logging_configs(server_args)
|
||||||
|
|
||||||
|
# Delay listen() until uvicorn startup to avoid accepting probe traffic
|
||||||
|
# while model/subprocess initialization is still in progress.
|
||||||
|
reserved_socket.listen(128)
|
||||||
|
|
||||||
# Listen for HTTP requests
|
# Listen for HTTP requests
|
||||||
if server_args.tokenizer_worker_num == 1:
|
if server_args.tokenizer_worker_num == 1:
|
||||||
# Default case, one tokenizer process
|
# Default case, one tokenizer process
|
||||||
@@ -2089,5 +2094,7 @@ def launch_server(
|
|||||||
reserved_socket.close()
|
reserved_socket.close()
|
||||||
|
|
||||||
if server_args.tokenizer_worker_num > 1:
|
if server_args.tokenizer_worker_num > 1:
|
||||||
|
if multi_tokenizer_args_shm is not None:
|
||||||
multi_tokenizer_args_shm.unlink()
|
multi_tokenizer_args_shm.unlink()
|
||||||
|
if _global_state is not None:
|
||||||
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
|
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
|
||||||
|
|||||||
@@ -416,7 +416,9 @@ def _prebind_listening_socket(host: str, port: int) -> socket.socket:
|
|||||||
This prevents port conflicts that could occur if another process (e.g., a
|
This prevents port conflicts that could occur if another process (e.g., a
|
||||||
subprocess spawned during model loading) binds to the same port before the
|
subprocess spawned during model loading) binds to the same port before the
|
||||||
HTTP server starts. By binding early, we ensure the port is available and
|
HTTP server starts. By binding early, we ensure the port is available and
|
||||||
reserved for the HTTP server.
|
reserved for the HTTP server. The socket is intentionally not put into
|
||||||
|
listening mode here to avoid accepting probe connections before uvicorn is
|
||||||
|
ready to serve requests.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
host: The host address to bind to.
|
host: The host address to bind to.
|
||||||
@@ -444,14 +446,8 @@ def _prebind_listening_socket(host: str, port: int) -> socket.socket:
|
|||||||
# SO_REUSEADDR: Allows binding to a port in TIME_WAIT state
|
# SO_REUSEADDR: Allows binding to a port in TIME_WAIT state
|
||||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
|
||||||
# SO_REUSEPORT: Allows multiple processes to bind to the same port
|
|
||||||
# This is useful for multi-worker mode
|
|
||||||
if hasattr(socket, "SO_REUSEPORT"):
|
|
||||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
sock.bind((host or "", port))
|
sock.bind((host or "", port))
|
||||||
sock.listen(128) # Put socket in listening state for uvicorn
|
|
||||||
sock.set_inheritable(True)
|
sock.set_inheritable(True)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Successfully reserved port {port} on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'"
|
f"Successfully reserved port {port} on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'"
|
||||||
|
|||||||
Reference in New Issue
Block a user