From 7cefa607657b9e63abe6946cb795e77d4d31c9c0 Mon Sep 17 00:00:00 2001 From: Richard Wang Date: Fri, 23 Oct 2026 20:02:24 -0700 Subject: [PATCH] [Perf] diffusion: Wait on the socket when the scheduler is idle (#46007) Co-authored-by: Mick Qian --- docs/docs/sglang-diffusion/api/cli.mdx | 1 + .../runtime/managers/gpu_worker.py | 13 ++-- .../runtime/managers/scheduler.py | 20 +++++- .../runtime/server_args/server_args.py | 9 +++ .../test/unit/test_idle_cache_release.py | 69 +++++++++++++++++++ 5 files changed, 107 insertions(+), 5 deletions(-) diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index 148e01997b..570833db8e 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -111,6 +111,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis - `--srt-encoder-timeout {SECONDS}`: Timeout in seconds for HTTP requests to the SGLang encoder server - `--srt-encoder-connection-timeout {SECONDS}`: TCP connection timeout in seconds for SGLang encoder server - `--scheduler-rpc-timeout {SECONDS}`: optional end-to-end deadline for an internal scheduler RPC, including scheduler queue time. It is unset by default so valid long-running and queued video jobs are not failed by the transport layer. Set it only when the deployment requires a bounded request deadline; caller cancellation and server shutdown remain effective without it. +- `--sleep-on-idle`: reduce idle scheduler CPU usage by waiting for socket events. Disabled by default, matching SRT, to minimize idle-to-active wake-up latency. Applies only to monolithic serving with one rank per DP replica; TP/SP/CFG multi-rank replicas keep polling. Each wait ends when a message arrives, a pending allocator-cache release is due, or one second elapses. Active generation is unchanged; benchmark low-QPS tail latency before enabling it in latency-sensitive deployments. - `--enable-trace`: export OpenTelemetry traces for the diffusion scheduler and worker stages. - `--otlp-traces-endpoint {HOST:PORT}`: OTLP collector endpoint used with `--enable-trace` (default: `localhost:4317`). - `--otlp-service-name {NAME}`: `service.name` attached to exported traces. Resolution order is this flag, `OTEL_SERVICE_NAME`, then `sglang-diffusion`. diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index 08db747ebc..5c1e365a6f 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -924,17 +924,22 @@ class GPUWorker(GPUWorkerPostTrainingMixin): ) stream.synchronize() - def release_cache_if_idle(self) -> None: + def release_cache_if_idle(self) -> float | None: """Release the allocator cache once no request has finished for a while. - The scheduler calls this only with nothing queued or in flight. + The scheduler calls this only with nothing queued or in flight. Returns + the seconds until a pending release is due, or None when none is pending. """ due = self._cache_release_due - if due is None or time.monotonic() < due: - return + if due is None: + return None + wait_s = due - time.monotonic() + if wait_s > 0: + return wait_s self._cache_release_due = None with maybe_record_function("EMPTY_CACHE"): torch.get_device_module().empty_cache() + return None def take_deferred_finalize(self) -> Callable[[], None] | None: deferred = self._deferred_finalize diff --git a/python/sglang/multimodal_gen/runtime/managers/scheduler.py b/python/sglang/multimodal_gen/runtime/managers/scheduler.py index 7e6fc056b6..9091056508 100644 --- a/python/sglang/multimodal_gen/runtime/managers/scheduler.py +++ b/python/sglang/multimodal_gen/runtime/managers/scheduler.py @@ -1459,12 +1459,13 @@ class Scheduler( # 2: execute, make sure a reply is always sent items = self.get_next_batch_to_run() if not items: + release_wait_s = None if ( release_cache_if_idle is not None and not self.waiting_queue and not self._inflight_finalizes ): - release_cache_if_idle() + release_wait_s = release_cache_if_idle() if self.waiting_queue and self._dynamic_batching_enabled(): oldest_ts = self.waiting_queue[0][2] elapsed_ms = (time.monotonic() - oldest_ts) * 1000.0 @@ -1483,6 +1484,23 @@ class Scheduler( self._poller.poll(timeout=1) else: time.sleep(0.001) + elif ( + not self.waiting_queue + and self.receiver is not None + and not self._is_multi_rank() + and self.server_args.sleep_on_idle + ): + # Wait instead of spinning a core. Multi rank replicas keep + # spinning because their other ranks time their idle cache + # release by this loop's iterations. + with self._socket_lock: + self._poller.poll( + timeout=( + 1000 + if release_wait_s is None + else min(release_wait_s * 1000.0, 1000) + ) + ) continue if self.metrics is not None: diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index dede3cdb72..74b98be951 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -577,6 +577,7 @@ class ServerArgs(DisaggServerArgsMixin): batching_config: str | None = None enable_batching_metrics: bool = False async_output_save: bool = False + sleep_on_idle: bool = False # Strict port mode: fail if requested port is unavailable instead of auto-selecting strict_ports: bool = False @@ -3084,6 +3085,14 @@ class ServerArgs(DisaggServerArgsMixin): default=ServerArgs.enable_batching_metrics, help="Log periodic batch efficiency metrics such as realized batch size and queue wait time.", ) + parser.add_argument( + "--sleep-on-idle", + action="store_true", + default=ServerArgs.sleep_on_idle, + help="Reduce idle CPU usage by waiting for requests instead of busy polling. " + "Disabled by default to minimize wake-up latency. Applies only to " + "single-rank monolithic replicas; TP/SP/CFG multi-rank replicas keep polling.", + ) parser.add_argument( "--async-output-save", action="store_true", diff --git a/python/sglang/multimodal_gen/test/unit/test_idle_cache_release.py b/python/sglang/multimodal_gen/test/unit/test_idle_cache_release.py index 4eff152a16..c8eabc69d3 100644 --- a/python/sglang/multimodal_gen/test/unit/test_idle_cache_release.py +++ b/python/sglang/multimodal_gen/test/unit/test_idle_cache_release.py @@ -1,12 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 +import threading +from collections import deque from types import SimpleNamespace +from unittest.mock import Mock import pytest +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.managers import gpu_worker as gpu_worker_module from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker +from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser class _Clock: @@ -87,3 +94,65 @@ def test_without_an_idle_hook_each_request_still_releases(worker): assert releases == [clock.now] worker.release_cache_if_idle() assert releases == [clock.now] + + +@pytest.mark.parametrize( + "cli_args,tp_size,should_sleep", + [([], 1, False), (["--sleep-on-idle"], 1, True), (["--sleep-on-idle"], 2, False)], +) +def test_idle_scheduler_wait_policy(worker, cli_args, tp_size, should_sleep): + worker, releases, clock = worker + worker.defer_cache_release = True + _finish_request(worker) + due = clock.now + gpu_worker_module._IDLE_CACHE_RELEASE_S + clock.now += gpu_worker_module._IDLE_CACHE_RELEASE_S / 2 + + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + parsed = parser.parse_args(["--model-path", "/fake", *cli_args]) + scheduler = Scheduler.__new__(Scheduler) + scheduler.server_args = SimpleNamespace( + sp_degree=1, + tp_size=tp_size, + enable_cfg_parallel=False, + sleep_on_idle=parsed.sleep_on_idle, + ) + scheduler.worker = worker + scheduler._disagg_role = RoleType.MONOLITHIC + scheduler._disagg_metrics = None + scheduler.metrics = None + scheduler.dp_replica = 0 + scheduler.receiver = Mock() + scheduler.context = Mock() + scheduler.waiting_queue = deque() + scheduler._async_output_save = False + scheduler._inflight_finalizes = deque() + scheduler._socket_lock = threading.Lock() + scheduler._running = True + scheduler._log_batch_metrics_summary = Mock() + scheduler._cleanup_disagg = Mock() + scheduler.process_received_reqs_with_req_based_warmup = lambda reqs: reqs + scheduler.get_next_batch_to_run = lambda: None + loops = [] + + def recv_reqs(): + loops.append(None) + if not should_sleep: + clock.now += gpu_worker_module._IDLE_CACHE_RELEASE_S / 2 + scheduler._running = len(loops) < 4 + return [] + + timeouts = [] + + def poll(timeout): + timeouts.append(timeout) + clock.now += timeout / 1000.0 + if len(timeouts) == 2: + scheduler._running = False + + scheduler.recv_reqs = recv_reqs + scheduler._poller = SimpleNamespace(poll=poll) + scheduler.event_loop() + expected = [gpu_worker_module._IDLE_CACHE_RELEASE_S * 500.0, 1000] + assert timeouts == (expected if should_sleep else []) + assert releases == [due]