Fix port overflow in DP attention path when base port is near 65535 (#20260)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
71a0b10462
commit
0d0254c9de
@@ -8033,13 +8033,28 @@ class PortArgs:
|
||||
else:
|
||||
# DP attention. Use TCP + port to handle both single-node and multi-node.
|
||||
if server_args.nnodes == 1 and server_args.dist_init_addr is None:
|
||||
na = NetworkAddress("127.0.0.1", server_args.port + ZMQ_TCP_PORT_DELTA)
|
||||
derived_port = server_args.port + ZMQ_TCP_PORT_DELTA
|
||||
if derived_port > 65535:
|
||||
derived_port = server_args.port - ZMQ_TCP_PORT_DELTA
|
||||
na = NetworkAddress("127.0.0.1", derived_port)
|
||||
else:
|
||||
na = NetworkAddress.parse(server_args.dist_init_addr)
|
||||
|
||||
dist_init_host = na.host
|
||||
dist_init_port = na.port
|
||||
port_base = dist_init_port + 1
|
||||
|
||||
# We need 5 consecutive ports from port_base for:
|
||||
# port_base, detokenizer, rpc, metrics, scheduler.
|
||||
# In multi-node, all nodes derive ports independently from
|
||||
# dist_init_port, so the derivation must be deterministic
|
||||
# (no availability-based search). If incrementing would
|
||||
# overflow the valid TCP range, decrement instead.
|
||||
NUM_DERIVED_PORTS = 5
|
||||
if dist_init_port + NUM_DERIVED_PORTS > 65535:
|
||||
port_base = dist_init_port - NUM_DERIVED_PORTS - 1
|
||||
else:
|
||||
port_base = dist_init_port + 1
|
||||
|
||||
detokenizer_port = port_base + 1
|
||||
rpc_port = port_base + 2
|
||||
metrics_port = port_base + 3
|
||||
|
||||
@@ -49,9 +49,19 @@ def find_process_using_port(port: int) -> Optional[psutil.Process]:
|
||||
return None
|
||||
|
||||
|
||||
MAX_VALID_PORT = 65535
|
||||
|
||||
|
||||
def wait_port_available(
|
||||
port: int, port_name: str, timeout_s: int = 30, raise_exception: bool = True
|
||||
) -> bool:
|
||||
if port < 0 or port > MAX_VALID_PORT:
|
||||
raise ValueError(
|
||||
f"{port_name} has invalid port number {port}. "
|
||||
f"Valid TCP port range is 0-{MAX_VALID_PORT}."
|
||||
)
|
||||
|
||||
error_message = f"{port_name} at {port} is not available"
|
||||
for i in range(timeout_s):
|
||||
if is_port_available(port):
|
||||
return True
|
||||
@@ -62,12 +72,12 @@ def wait_port_available(
|
||||
logger.warning(
|
||||
f"The port {port} is in use, but we could not find the process that uses it."
|
||||
)
|
||||
|
||||
pid = process.pid
|
||||
error_message = f"{port_name} is used by a process already. {process.name()=}' {process.cmdline()=} {process.status()=} {pid=}"
|
||||
logger.info(
|
||||
f"port {port} is in use. Waiting for {i} seconds for {port_name} to be available. {error_message}"
|
||||
)
|
||||
else:
|
||||
pid = process.pid
|
||||
error_message = f"{port_name} is used by a process already. {process.name()=}' {process.cmdline()=} {process.status()=} {pid=}"
|
||||
logger.info(
|
||||
f"port {port} is in use. Waiting for {i} seconds for {port_name} to be available. {error_message}"
|
||||
)
|
||||
time.sleep(0.1)
|
||||
|
||||
if raise_exception:
|
||||
|
||||
Reference in New Issue
Block a user