[Fix] diffusion/http: Allow bounded graceful shutdown (#46169)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-24 09:11:15 +08:00
committed by GitHub
co-authored by Mick Qian
parent e22a530859
commit 2d5ad3243f
5 changed files with 86 additions and 0 deletions
@@ -491,6 +491,7 @@ def launch_http_server_only(server_args):
port=server_args.port,
reload=False,
ws_per_message_deflate=False,
timeout_graceful_shutdown=server_args.http_graceful_shutdown_timeout,
)
@@ -561,6 +561,7 @@ class ServerArgs(DisaggServerArgsMixin):
# http server endpoint config
host: str | None = "127.0.0.1"
port: int | None = 30000
http_graceful_shutdown_timeout: int | None = None
enable_metrics: bool = False
# TODO: webui and their endpoint, check if webui_port is available.
@@ -699,6 +700,14 @@ class ServerArgs(DisaggServerArgsMixin):
def _validate_parameters(self):
"""check consistency and raise errors for invalid configs"""
self._validate_scheduler_rpc_timeout()
timeout = self.http_graceful_shutdown_timeout
if timeout is not None and (
not isinstance(timeout, int) or isinstance(timeout, bool) or timeout < 0
):
raise ValueError(
"http_graceful_shutdown_timeout must be None or a non-negative "
"integer in seconds"
)
self._validate_pipeline()
self._validate_offload()
self._validate_direct_gpu_weight_loading()
@@ -3041,6 +3050,14 @@ class ServerArgs(DisaggServerArgsMixin):
default=ServerArgs.port,
help="Port for the HTTP API server.",
)
parser.add_argument(
"--http-graceful-shutdown-timeout",
type=int,
default=ServerArgs.http_graceful_shutdown_timeout,
help="Seconds to wait for HTTP connections and tasks to finish on shutdown "
"before cancelling them. Unset waits indefinitely; 0 cancels immediately. "
"Does not bound GPU worker teardown or preserve asynchronous video jobs.",
)
parser.add_argument(
"--enable-metrics",
action=StoreBoolean,
@@ -34,6 +34,27 @@ class _FakeProcess:
class TestLaunchServerShutdown(unittest.TestCase):
def test_http_graceful_shutdown_timeout_is_forwarded(self):
for timeout in (None, 0, 30):
with self.subTest(timeout=timeout):
server_args = SimpleNamespace(
log_level="info",
host="127.0.0.1",
port=30000,
http_graceful_shutdown_timeout=timeout,
)
with (
patch.object(ls, "init_diffusion_tracing"),
patch.object(ls, "set_global_server_args"),
patch.object(ls, "create_app"),
patch.object(ls.uvicorn, "run") as run,
):
ls.launch_http_server_only(server_args)
self.assertEqual(
run.call_args.kwargs["timeout_graceful_shutdown"], timeout
)
def test_monolithic_shutdown_requests_scheduler_then_forces_worker(self):
process = _FakeProcess()
server_args = SimpleNamespace(disagg_role=RoleType.MONOLITHIC)
@@ -3596,6 +3596,39 @@ class TestSchedulerRpcTimeoutArgs(unittest.TestCase):
)
class TestHttpGracefulShutdownTimeoutArgs(unittest.TestCase):
def test_cli_default_and_explicit_timeout(self):
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
argv = ["--model-path", "/fake"]
self.assertIsNone(parser.parse_args(argv).http_graceful_shutdown_timeout)
for timeout in (0, 30):
with self.subTest(timeout=timeout):
args = parser.parse_args(
argv + ["--http-graceful-shutdown-timeout", str(timeout)]
)
self.assertEqual(args.http_graceful_shutdown_timeout, timeout)
def test_config_values(self):
for timeout in (None, 0, 30):
with self.subTest(timeout=timeout):
args = _from_dict_without_model_resolution(
{"model_path": "/fake", "http_graceful_shutdown_timeout": timeout}
)
self.assertEqual(args.http_graceful_shutdown_timeout, timeout)
for timeout in (-1, True, 1.5, "1"):
with self.subTest(timeout=timeout):
with self.assertRaisesRegex(
ValueError, "http_graceful_shutdown_timeout"
):
_from_dict_without_model_resolution(
{
"model_path": "/fake",
"http_graceful_shutdown_timeout": timeout,
}
)
class TestDisaggTransferBackendArgs(unittest.TestCase):
def test_transfer_backend_cli_arg_is_parsed(self):
parser = FlexibleArgumentParser()