[Perf] diffusion: Wait on the socket when the scheduler is idle (#46007)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Mick Qian
parent
25ba24e9a7
commit
7cefa60765
@@ -924,17 +924,22 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
)
|
||||
stream.synchronize()
|
||||
|
||||
def release_cache_if_idle(self) -> None:
|
||||
def release_cache_if_idle(self) -> float | None:
|
||||
"""Release the allocator cache once no request has finished for a while.
|
||||
|
||||
The scheduler calls this only with nothing queued or in flight.
|
||||
The scheduler calls this only with nothing queued or in flight. Returns
|
||||
the seconds until a pending release is due, or None when none is pending.
|
||||
"""
|
||||
due = self._cache_release_due
|
||||
if due is None or time.monotonic() < due:
|
||||
return
|
||||
if due is None:
|
||||
return None
|
||||
wait_s = due - time.monotonic()
|
||||
if wait_s > 0:
|
||||
return wait_s
|
||||
self._cache_release_due = None
|
||||
with maybe_record_function("EMPTY_CACHE"):
|
||||
torch.get_device_module().empty_cache()
|
||||
return None
|
||||
|
||||
def take_deferred_finalize(self) -> Callable[[], None] | None:
|
||||
deferred = self._deferred_finalize
|
||||
|
||||
@@ -1459,12 +1459,13 @@ class Scheduler(
|
||||
# 2: execute, make sure a reply is always sent
|
||||
items = self.get_next_batch_to_run()
|
||||
if not items:
|
||||
release_wait_s = None
|
||||
if (
|
||||
release_cache_if_idle is not None
|
||||
and not self.waiting_queue
|
||||
and not self._inflight_finalizes
|
||||
):
|
||||
release_cache_if_idle()
|
||||
release_wait_s = release_cache_if_idle()
|
||||
if self.waiting_queue and self._dynamic_batching_enabled():
|
||||
oldest_ts = self.waiting_queue[0][2]
|
||||
elapsed_ms = (time.monotonic() - oldest_ts) * 1000.0
|
||||
@@ -1483,6 +1484,23 @@ class Scheduler(
|
||||
self._poller.poll(timeout=1)
|
||||
else:
|
||||
time.sleep(0.001)
|
||||
elif (
|
||||
not self.waiting_queue
|
||||
and self.receiver is not None
|
||||
and not self._is_multi_rank()
|
||||
and self.server_args.sleep_on_idle
|
||||
):
|
||||
# Wait instead of spinning a core. Multi rank replicas keep
|
||||
# spinning because their other ranks time their idle cache
|
||||
# release by this loop's iterations.
|
||||
with self._socket_lock:
|
||||
self._poller.poll(
|
||||
timeout=(
|
||||
1000
|
||||
if release_wait_s is None
|
||||
else min(release_wait_s * 1000.0, 1000)
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if self.metrics is not None:
|
||||
|
||||
@@ -577,6 +577,7 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
batching_config: str | None = None
|
||||
enable_batching_metrics: bool = False
|
||||
async_output_save: bool = False
|
||||
sleep_on_idle: bool = False
|
||||
|
||||
# Strict port mode: fail if requested port is unavailable instead of auto-selecting
|
||||
strict_ports: bool = False
|
||||
@@ -3084,6 +3085,14 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
default=ServerArgs.enable_batching_metrics,
|
||||
help="Log periodic batch efficiency metrics such as realized batch size and queue wait time.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sleep-on-idle",
|
||||
action="store_true",
|
||||
default=ServerArgs.sleep_on_idle,
|
||||
help="Reduce idle CPU usage by waiting for requests instead of busy polling. "
|
||||
"Disabled by default to minimize wake-up latency. Applies only to "
|
||||
"single-rank monolithic replicas; TP/SP/CFG multi-rank replicas keep polling.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--async-output-save",
|
||||
action="store_true",
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import threading
|
||||
from collections import deque
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.managers import gpu_worker as gpu_worker_module
|
||||
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
|
||||
from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser
|
||||
|
||||
|
||||
class _Clock:
|
||||
@@ -87,3 +94,65 @@ def test_without_an_idle_hook_each_request_still_releases(worker):
|
||||
assert releases == [clock.now]
|
||||
worker.release_cache_if_idle()
|
||||
assert releases == [clock.now]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cli_args,tp_size,should_sleep",
|
||||
[([], 1, False), (["--sleep-on-idle"], 1, True), (["--sleep-on-idle"], 2, False)],
|
||||
)
|
||||
def test_idle_scheduler_wait_policy(worker, cli_args, tp_size, should_sleep):
|
||||
worker, releases, clock = worker
|
||||
worker.defer_cache_release = True
|
||||
_finish_request(worker)
|
||||
due = clock.now + gpu_worker_module._IDLE_CACHE_RELEASE_S
|
||||
clock.now += gpu_worker_module._IDLE_CACHE_RELEASE_S / 2
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
parsed = parser.parse_args(["--model-path", "/fake", *cli_args])
|
||||
scheduler = Scheduler.__new__(Scheduler)
|
||||
scheduler.server_args = SimpleNamespace(
|
||||
sp_degree=1,
|
||||
tp_size=tp_size,
|
||||
enable_cfg_parallel=False,
|
||||
sleep_on_idle=parsed.sleep_on_idle,
|
||||
)
|
||||
scheduler.worker = worker
|
||||
scheduler._disagg_role = RoleType.MONOLITHIC
|
||||
scheduler._disagg_metrics = None
|
||||
scheduler.metrics = None
|
||||
scheduler.dp_replica = 0
|
||||
scheduler.receiver = Mock()
|
||||
scheduler.context = Mock()
|
||||
scheduler.waiting_queue = deque()
|
||||
scheduler._async_output_save = False
|
||||
scheduler._inflight_finalizes = deque()
|
||||
scheduler._socket_lock = threading.Lock()
|
||||
scheduler._running = True
|
||||
scheduler._log_batch_metrics_summary = Mock()
|
||||
scheduler._cleanup_disagg = Mock()
|
||||
scheduler.process_received_reqs_with_req_based_warmup = lambda reqs: reqs
|
||||
scheduler.get_next_batch_to_run = lambda: None
|
||||
loops = []
|
||||
|
||||
def recv_reqs():
|
||||
loops.append(None)
|
||||
if not should_sleep:
|
||||
clock.now += gpu_worker_module._IDLE_CACHE_RELEASE_S / 2
|
||||
scheduler._running = len(loops) < 4
|
||||
return []
|
||||
|
||||
timeouts = []
|
||||
|
||||
def poll(timeout):
|
||||
timeouts.append(timeout)
|
||||
clock.now += timeout / 1000.0
|
||||
if len(timeouts) == 2:
|
||||
scheduler._running = False
|
||||
|
||||
scheduler.recv_reqs = recv_reqs
|
||||
scheduler._poller = SimpleNamespace(poll=poll)
|
||||
scheduler.event_loop()
|
||||
expected = [gpu_worker_module._IDLE_CACHE_RELEASE_S * 500.0, 1000]
|
||||
assert timeouts == (expected if should_sleep else [])
|
||||
assert releases == [due]
|
||||
|
||||
Reference in New Issue
Block a user