diff --git a/python/sglang/srt/utils/network.py b/python/sglang/srt/utils/network.py index be36df9b9..cec89c715 100644 --- a/python/sglang/srt/utils/network.py +++ b/python/sglang/srt/utils/network.py @@ -54,8 +54,17 @@ MAX_VALID_PORT = 65535 def wait_port_available( - port: int, port_name: str, timeout_s: int = 30, raise_exception: bool = True + port: int, + port_name: str, + timeout_s: Optional[int] = None, + raise_exception: bool = True, ) -> bool: + if timeout_s is None: + # A killed server can hold its ports well past kill_process_tree()'s + # return while GPU teardown completes (>30s observed on GB300), so CI + # raises this via SGLANG_WAIT_PORT_TIMEOUT before relaunching a server + # on the same port plan. + timeout_s = int(os.environ.get("SGLANG_WAIT_PORT_TIMEOUT", "30")) if port < 0 or port > MAX_VALID_PORT: raise ValueError( f"{port_name} has invalid port number {port}. " diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 773b1c888..6140b032f 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -927,6 +927,11 @@ def popen_launch_server( merged.update(env) env = merged + # A dying predecessor can hold the derived port plan past + # kill_process_tree() while GPU teardown completes; give CI launches + # teardown-sized patience (see wait_port_available). + env.setdefault("SGLANG_WAIT_PORT_TIMEOUT", "120") + # Store per-run marker path for potential invalidation per_run_marker_path = None try: