From 2d5ad3243f1d7ed9e16dd204170dc8852b89a04c Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 24 Oct 2026 09:11:15 +0800 Subject: [PATCH] [Fix] diffusion/http: Allow bounded graceful shutdown (#46169) Co-authored-by: Mick Qian --- docs/docs/sglang-diffusion/api/cli.mdx | 14 ++++++++ .../multimodal_gen/runtime/launch_server.py | 1 + .../runtime/server_args/server_args.py | 17 ++++++++++ .../test/unit/test_launch_server_shutdown.py | 21 ++++++++++++ .../test/unit/test_server_args.py | 33 +++++++++++++++++++ 5 files changed, 86 insertions(+) diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index 8df193fc6a..9de1e359ea 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -445,6 +445,20 @@ native NVIDIA CUDA serving. It is disabled by default. See [Asynchronous output saving](/docs/sglang-diffusion/performance-optimization#asynchronous-output-saving) for an example, eligible requests, and the response latency and memory tradeoffs. +### Graceful shutdown + +Set `--http-graceful-shutdown-timeout 30` to allow up to 30 seconds for open HTTP +connections and request tasks to finish after shutdown begins. At the deadline, +unfinished HTTP tasks are cancelled, including uploads that have not sent their +complete request body. The default is unset (wait indefinitely); `0` cancels +outstanding HTTP tasks without waiting. + +This controls HTTP draining only, not the full process shutdown time or an +inference deadline. Application cleanup and GPU worker teardown can take longer; +allow for them in your orchestrator's termination grace period. It does not make +asynchronous video jobs durable or guarantee that queued jobs finish before +shutdown. Use `--scheduler-rpc-timeout` separately to bound scheduler reply waits. + ### Health endpoints SGLang Diffusion separates process liveness from inference readiness: diff --git a/python/sglang/multimodal_gen/runtime/launch_server.py b/python/sglang/multimodal_gen/runtime/launch_server.py index 8e43c8f920..f612ff696c 100644 --- a/python/sglang/multimodal_gen/runtime/launch_server.py +++ b/python/sglang/multimodal_gen/runtime/launch_server.py @@ -491,6 +491,7 @@ def launch_http_server_only(server_args): port=server_args.port, reload=False, ws_per_message_deflate=False, + timeout_graceful_shutdown=server_args.http_graceful_shutdown_timeout, ) diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index c95e364d3b..928b128d37 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -561,6 +561,7 @@ class ServerArgs(DisaggServerArgsMixin): # http server endpoint config host: str | None = "127.0.0.1" port: int | None = 30000 + http_graceful_shutdown_timeout: int | None = None enable_metrics: bool = False # TODO: webui and their endpoint, check if webui_port is available. @@ -699,6 +700,14 @@ class ServerArgs(DisaggServerArgsMixin): def _validate_parameters(self): """check consistency and raise errors for invalid configs""" self._validate_scheduler_rpc_timeout() + timeout = self.http_graceful_shutdown_timeout + if timeout is not None and ( + not isinstance(timeout, int) or isinstance(timeout, bool) or timeout < 0 + ): + raise ValueError( + "http_graceful_shutdown_timeout must be None or a non-negative " + "integer in seconds" + ) self._validate_pipeline() self._validate_offload() self._validate_direct_gpu_weight_loading() @@ -3041,6 +3050,14 @@ class ServerArgs(DisaggServerArgsMixin): default=ServerArgs.port, help="Port for the HTTP API server.", ) + parser.add_argument( + "--http-graceful-shutdown-timeout", + type=int, + default=ServerArgs.http_graceful_shutdown_timeout, + help="Seconds to wait for HTTP connections and tasks to finish on shutdown " + "before cancelling them. Unset waits indefinitely; 0 cancels immediately. " + "Does not bound GPU worker teardown or preserve asynchronous video jobs.", + ) parser.add_argument( "--enable-metrics", action=StoreBoolean, diff --git a/python/sglang/multimodal_gen/test/unit/test_launch_server_shutdown.py b/python/sglang/multimodal_gen/test/unit/test_launch_server_shutdown.py index 0a3949acad..40c7f410c6 100644 --- a/python/sglang/multimodal_gen/test/unit/test_launch_server_shutdown.py +++ b/python/sglang/multimodal_gen/test/unit/test_launch_server_shutdown.py @@ -34,6 +34,27 @@ class _FakeProcess: class TestLaunchServerShutdown(unittest.TestCase): + def test_http_graceful_shutdown_timeout_is_forwarded(self): + for timeout in (None, 0, 30): + with self.subTest(timeout=timeout): + server_args = SimpleNamespace( + log_level="info", + host="127.0.0.1", + port=30000, + http_graceful_shutdown_timeout=timeout, + ) + with ( + patch.object(ls, "init_diffusion_tracing"), + patch.object(ls, "set_global_server_args"), + patch.object(ls, "create_app"), + patch.object(ls.uvicorn, "run") as run, + ): + ls.launch_http_server_only(server_args) + + self.assertEqual( + run.call_args.kwargs["timeout_graceful_shutdown"], timeout + ) + def test_monolithic_shutdown_requests_scheduler_then_forces_worker(self): process = _FakeProcess() server_args = SimpleNamespace(disagg_role=RoleType.MONOLITHIC) diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index efb5e59275..1f7f9e6344 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -3596,6 +3596,39 @@ class TestSchedulerRpcTimeoutArgs(unittest.TestCase): ) +class TestHttpGracefulShutdownTimeoutArgs(unittest.TestCase): + def test_cli_default_and_explicit_timeout(self): + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + argv = ["--model-path", "/fake"] + self.assertIsNone(parser.parse_args(argv).http_graceful_shutdown_timeout) + for timeout in (0, 30): + with self.subTest(timeout=timeout): + args = parser.parse_args( + argv + ["--http-graceful-shutdown-timeout", str(timeout)] + ) + self.assertEqual(args.http_graceful_shutdown_timeout, timeout) + + def test_config_values(self): + for timeout in (None, 0, 30): + with self.subTest(timeout=timeout): + args = _from_dict_without_model_resolution( + {"model_path": "/fake", "http_graceful_shutdown_timeout": timeout} + ) + self.assertEqual(args.http_graceful_shutdown_timeout, timeout) + for timeout in (-1, True, 1.5, "1"): + with self.subTest(timeout=timeout): + with self.assertRaisesRegex( + ValueError, "http_graceful_shutdown_timeout" + ): + _from_dict_without_model_resolution( + { + "model_path": "/fake", + "http_graceful_shutdown_timeout": timeout, + } + ) + + class TestDisaggTransferBackendArgs(unittest.TestCase): def test_transfer_backend_cli_arg_is_parsed(self): parser = FlexibleArgumentParser()