[Fix] diffusion/http: Allow bounded graceful shutdown (#46169)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -491,6 +491,7 @@ def launch_http_server_only(server_args):
|
||||
port=server_args.port,
|
||||
reload=False,
|
||||
ws_per_message_deflate=False,
|
||||
timeout_graceful_shutdown=server_args.http_graceful_shutdown_timeout,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -561,6 +561,7 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
# http server endpoint config
|
||||
host: str | None = "127.0.0.1"
|
||||
port: int | None = 30000
|
||||
http_graceful_shutdown_timeout: int | None = None
|
||||
enable_metrics: bool = False
|
||||
|
||||
# TODO: webui and their endpoint, check if webui_port is available.
|
||||
@@ -699,6 +700,14 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
def _validate_parameters(self):
|
||||
"""check consistency and raise errors for invalid configs"""
|
||||
self._validate_scheduler_rpc_timeout()
|
||||
timeout = self.http_graceful_shutdown_timeout
|
||||
if timeout is not None and (
|
||||
not isinstance(timeout, int) or isinstance(timeout, bool) or timeout < 0
|
||||
):
|
||||
raise ValueError(
|
||||
"http_graceful_shutdown_timeout must be None or a non-negative "
|
||||
"integer in seconds"
|
||||
)
|
||||
self._validate_pipeline()
|
||||
self._validate_offload()
|
||||
self._validate_direct_gpu_weight_loading()
|
||||
@@ -3041,6 +3050,14 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
default=ServerArgs.port,
|
||||
help="Port for the HTTP API server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--http-graceful-shutdown-timeout",
|
||||
type=int,
|
||||
default=ServerArgs.http_graceful_shutdown_timeout,
|
||||
help="Seconds to wait for HTTP connections and tasks to finish on shutdown "
|
||||
"before cancelling them. Unset waits indefinitely; 0 cancels immediately. "
|
||||
"Does not bound GPU worker teardown or preserve asynchronous video jobs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-metrics",
|
||||
action=StoreBoolean,
|
||||
|
||||
@@ -34,6 +34,27 @@ class _FakeProcess:
|
||||
|
||||
|
||||
class TestLaunchServerShutdown(unittest.TestCase):
|
||||
def test_http_graceful_shutdown_timeout_is_forwarded(self):
|
||||
for timeout in (None, 0, 30):
|
||||
with self.subTest(timeout=timeout):
|
||||
server_args = SimpleNamespace(
|
||||
log_level="info",
|
||||
host="127.0.0.1",
|
||||
port=30000,
|
||||
http_graceful_shutdown_timeout=timeout,
|
||||
)
|
||||
with (
|
||||
patch.object(ls, "init_diffusion_tracing"),
|
||||
patch.object(ls, "set_global_server_args"),
|
||||
patch.object(ls, "create_app"),
|
||||
patch.object(ls.uvicorn, "run") as run,
|
||||
):
|
||||
ls.launch_http_server_only(server_args)
|
||||
|
||||
self.assertEqual(
|
||||
run.call_args.kwargs["timeout_graceful_shutdown"], timeout
|
||||
)
|
||||
|
||||
def test_monolithic_shutdown_requests_scheduler_then_forces_worker(self):
|
||||
process = _FakeProcess()
|
||||
server_args = SimpleNamespace(disagg_role=RoleType.MONOLITHIC)
|
||||
|
||||
@@ -3596,6 +3596,39 @@ class TestSchedulerRpcTimeoutArgs(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestHttpGracefulShutdownTimeoutArgs(unittest.TestCase):
|
||||
def test_cli_default_and_explicit_timeout(self):
|
||||
parser = FlexibleArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
argv = ["--model-path", "/fake"]
|
||||
self.assertIsNone(parser.parse_args(argv).http_graceful_shutdown_timeout)
|
||||
for timeout in (0, 30):
|
||||
with self.subTest(timeout=timeout):
|
||||
args = parser.parse_args(
|
||||
argv + ["--http-graceful-shutdown-timeout", str(timeout)]
|
||||
)
|
||||
self.assertEqual(args.http_graceful_shutdown_timeout, timeout)
|
||||
|
||||
def test_config_values(self):
|
||||
for timeout in (None, 0, 30):
|
||||
with self.subTest(timeout=timeout):
|
||||
args = _from_dict_without_model_resolution(
|
||||
{"model_path": "/fake", "http_graceful_shutdown_timeout": timeout}
|
||||
)
|
||||
self.assertEqual(args.http_graceful_shutdown_timeout, timeout)
|
||||
for timeout in (-1, True, 1.5, "1"):
|
||||
with self.subTest(timeout=timeout):
|
||||
with self.assertRaisesRegex(
|
||||
ValueError, "http_graceful_shutdown_timeout"
|
||||
):
|
||||
_from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/fake",
|
||||
"http_graceful_shutdown_timeout": timeout,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestDisaggTransferBackendArgs(unittest.TestCase):
|
||||
def test_transfer_backend_cli_arg_is_parsed(self):
|
||||
parser = FlexibleArgumentParser()
|
||||
|
||||
Reference in New Issue
Block a user