[RL] Keep pause_generation and weight updates from deadlocking each other (#40779)

Co-authored-by: maocheng23 <maocheng@berkeley.edu>
Co-authored-by: Zhichenzzz <zczeng@uw.edu>
This commit is contained in:
Yueming Yuan
2026-09-23 15:13:42 -07:00
committed by GitHub
co-authored by maocheng23 Zhichenzzz
parent 79fec592a8
commit 6fe4b66f5c
4 changed files with 130 additions and 12 deletions
+8 -4
View File
@@ -829,6 +829,7 @@ class Scheduler(
self.ipc_channels.recv_from_tokenizer,
self.ipc_channels.recv_from_rpc,
],
can_empty_cache=lambda: not self._engine_paused,
)
else:
self.idle_sleeper = None
@@ -2269,7 +2270,9 @@ class Scheduler(
# drains its request ring (rust_server_mode) instead of a zmq socket.
self.recv_from_tokenizer = rust_server
# Park the idle loop on the request ring within the rank-0 rust-server
self.idle_sleeper = RustServerIdleSleeper(rust_server)
self.idle_sleeper = RustServerIdleSleeper(
rust_server, can_empty_cache=lambda: not self._engine_paused
)
def get_rust_server_class(self) -> type[RustServer]:
return RustServer
@@ -4958,7 +4961,7 @@ class Scheduler(
else:
self.metrics_reporter.record_scheduler_active()
def is_fully_idle(self, for_health_check=False) -> bool:
def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool:
# Health check piggybacks on running requests in process_output.
# Only running_batch + waiting_queue guarantee active GPU processing;
# disagg queues (bootstrap/prealloc/transfer) may have items without
@@ -4975,7 +4978,8 @@ class Scheduler(
)
# Waiting queues: waiting + bootstrapping + preallocation + kv transfer (decode)
idle &= len(self.waiting_queue) == 0
if not ignore_waiting:
idle &= len(self.waiting_queue) == 0
if (
for_health_check
@@ -5133,7 +5137,7 @@ class Scheduler(
def flush_cache(self, empty_cache: bool = True):
"""Flush memory pools (e.g., KV cache, Mamba cache) and optionally empty device allocator cache."""
if self.is_fully_idle():
if self.is_fully_idle(ignore_waiting=self._engine_paused):
self.cur_batch_for_debug = None
self.last_batch = None
self.tree_cache.reset()
@@ -24,9 +24,10 @@ class IdleSleeper:
data that needs handling immediately.
"""
def __init__(self, sockets):
def __init__(self, sockets, can_empty_cache=None):
self.poller = zmq.Poller()
self.last_empty_time = real_time()
self.can_empty_cache = can_empty_cache
for s in sockets:
self.poller.register(s, zmq.POLLIN)
@@ -39,7 +40,8 @@ class IdleSleeper:
and real_time() - self.last_empty_time > self.empty_cache_interval
):
self.last_empty_time = real_time()
current_platform.empty_cache()
if self.can_empty_cache is None or self.can_empty_cache():
current_platform.empty_cache()
class RustServerIdleSleeper:
@@ -52,10 +54,13 @@ class RustServerIdleSleeper:
requests — or the timeout elapses.
"""
def __init__(self, rust_server: RustServer, timeout_ms: int = 1000):
def __init__(
self, rust_server: RustServer, timeout_ms: int = 1000, can_empty_cache=None
):
self.rust_server = rust_server
self.timeout_ms = timeout_ms
self.last_empty_time = real_time()
self.can_empty_cache = can_empty_cache
self.empty_cache_interval = envs.SGLANG_EMPTY_CACHE_INTERVAL.get()
def maybe_sleep(self):
@@ -65,4 +70,5 @@ class RustServerIdleSleeper:
and real_time() - self.last_empty_time > self.empty_cache_interval
):
self.last_empty_time = real_time()
current_platform.empty_cache()
if self.can_empty_cache is None or self.can_empty_cache():
current_platform.empty_cache()
@@ -210,9 +210,10 @@ class SchedulerWeightUpdaterManager:
)
def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
assert self.is_fully_idle(), (
"release_memory_occupation should be called only when server is idle."
)
scheduler = self.scheduler
assert self.is_fully_idle(
ignore_waiting=scheduler is not None and scheduler._engine_paused
), "release_memory_occupation should be called only when server is idle."
tags = recv_req.tags
@@ -223,7 +224,6 @@ class SchedulerWeightUpdaterManager:
self.offload_tags.add(tag)
if GPU_MEMORY_TYPE_KV_CACHE in tags:
scheduler = self.scheduler
if scheduler is not None:
if scheduler.disaggregation_mode == DisaggregationMode.DECODE:
for queue_name in (