[Perf] diffusion: Wait on the socket when the scheduler is idle (#46007)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Richard Wang
2026-10-24 11:02:24 +08:00
committed by GitHub
co-authored by Mick Qian
parent 25ba24e9a7
commit 7cefa60765
5 changed files with 107 additions and 5 deletions
@@ -924,17 +924,22 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
)
stream.synchronize()
def release_cache_if_idle(self) -> None:
def release_cache_if_idle(self) -> float | None:
"""Release the allocator cache once no request has finished for a while.
The scheduler calls this only with nothing queued or in flight.
The scheduler calls this only with nothing queued or in flight. Returns
the seconds until a pending release is due, or None when none is pending.
"""
due = self._cache_release_due
if due is None or time.monotonic() < due:
return
if due is None:
return None
wait_s = due - time.monotonic()
if wait_s > 0:
return wait_s
self._cache_release_due = None
with maybe_record_function("EMPTY_CACHE"):
torch.get_device_module().empty_cache()
return None
def take_deferred_finalize(self) -> Callable[[], None] | None:
deferred = self._deferred_finalize
@@ -1459,12 +1459,13 @@ class Scheduler(
# 2: execute, make sure a reply is always sent
items = self.get_next_batch_to_run()
if not items:
release_wait_s = None
if (
release_cache_if_idle is not None
and not self.waiting_queue
and not self._inflight_finalizes
):
release_cache_if_idle()
release_wait_s = release_cache_if_idle()
if self.waiting_queue and self._dynamic_batching_enabled():
oldest_ts = self.waiting_queue[0][2]
elapsed_ms = (time.monotonic() - oldest_ts) * 1000.0
@@ -1483,6 +1484,23 @@ class Scheduler(
self._poller.poll(timeout=1)
else:
time.sleep(0.001)
elif (
not self.waiting_queue
and self.receiver is not None
and not self._is_multi_rank()
and self.server_args.sleep_on_idle
):
# Wait instead of spinning a core. Multi rank replicas keep
# spinning because their other ranks time their idle cache
# release by this loop's iterations.
with self._socket_lock:
self._poller.poll(
timeout=(
1000
if release_wait_s is None
else min(release_wait_s * 1000.0, 1000)
)
)
continue
if self.metrics is not None:
@@ -577,6 +577,7 @@ class ServerArgs(DisaggServerArgsMixin):
batching_config: str | None = None
enable_batching_metrics: bool = False
async_output_save: bool = False
sleep_on_idle: bool = False
# Strict port mode: fail if requested port is unavailable instead of auto-selecting
strict_ports: bool = False
@@ -3084,6 +3085,14 @@ class ServerArgs(DisaggServerArgsMixin):
default=ServerArgs.enable_batching_metrics,
help="Log periodic batch efficiency metrics such as realized batch size and queue wait time.",
)
parser.add_argument(
"--sleep-on-idle",
action="store_true",
default=ServerArgs.sleep_on_idle,
help="Reduce idle CPU usage by waiting for requests instead of busy polling. "
"Disabled by default to minimize wake-up latency. Applies only to "
"single-rank monolithic replicas; TP/SP/CFG multi-rank replicas keep polling.",
)
parser.add_argument(
"--async-output-save",
action="store_true",
@@ -1,12 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
import threading
from collections import deque
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.managers import gpu_worker as gpu_worker_module
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser
class _Clock:
@@ -87,3 +94,65 @@ def test_without_an_idle_hook_each_request_still_releases(worker):
assert releases == [clock.now]
worker.release_cache_if_idle()
assert releases == [clock.now]
@pytest.mark.parametrize(
"cli_args,tp_size,should_sleep",
[([], 1, False), (["--sleep-on-idle"], 1, True), (["--sleep-on-idle"], 2, False)],
)
def test_idle_scheduler_wait_policy(worker, cli_args, tp_size, should_sleep):
worker, releases, clock = worker
worker.defer_cache_release = True
_finish_request(worker)
due = clock.now + gpu_worker_module._IDLE_CACHE_RELEASE_S
clock.now += gpu_worker_module._IDLE_CACHE_RELEASE_S / 2
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
parsed = parser.parse_args(["--model-path", "/fake", *cli_args])
scheduler = Scheduler.__new__(Scheduler)
scheduler.server_args = SimpleNamespace(
sp_degree=1,
tp_size=tp_size,
enable_cfg_parallel=False,
sleep_on_idle=parsed.sleep_on_idle,
)
scheduler.worker = worker
scheduler._disagg_role = RoleType.MONOLITHIC
scheduler._disagg_metrics = None
scheduler.metrics = None
scheduler.dp_replica = 0
scheduler.receiver = Mock()
scheduler.context = Mock()
scheduler.waiting_queue = deque()
scheduler._async_output_save = False
scheduler._inflight_finalizes = deque()
scheduler._socket_lock = threading.Lock()
scheduler._running = True
scheduler._log_batch_metrics_summary = Mock()
scheduler._cleanup_disagg = Mock()
scheduler.process_received_reqs_with_req_based_warmup = lambda reqs: reqs
scheduler.get_next_batch_to_run = lambda: None
loops = []
def recv_reqs():
loops.append(None)
if not should_sleep:
clock.now += gpu_worker_module._IDLE_CACHE_RELEASE_S / 2
scheduler._running = len(loops) < 4
return []
timeouts = []
def poll(timeout):
timeouts.append(timeout)
clock.now += timeout / 1000.0
if len(timeouts) == 2:
scheduler._running = False
scheduler.recv_reqs = recv_reqs
scheduler._poller = SimpleNamespace(poll=poll)
scheduler.event_loop()
expected = [gpu_worker_module._IDLE_CACHE_RELEASE_S * 500.0, 1000]
assert timeouts == (expected if should_sleep else [])
assert releases == [due]