[RL] Keep pause_generation and weight updates from deadlocking each other (#40779)
Co-authored-by: maocheng23 <maocheng@berkeley.edu> Co-authored-by: Zhichenzzz <zczeng@uw.edu>
This commit is contained in:
co-authored by
maocheng23
Zhichenzzz
parent
79fec592a8
commit
6fe4b66f5c
@@ -829,6 +829,7 @@ class Scheduler(
|
||||
self.ipc_channels.recv_from_tokenizer,
|
||||
self.ipc_channels.recv_from_rpc,
|
||||
],
|
||||
can_empty_cache=lambda: not self._engine_paused,
|
||||
)
|
||||
else:
|
||||
self.idle_sleeper = None
|
||||
@@ -2269,7 +2270,9 @@ class Scheduler(
|
||||
# drains its request ring (rust_server_mode) instead of a zmq socket.
|
||||
self.recv_from_tokenizer = rust_server
|
||||
# Park the idle loop on the request ring within the rank-0 rust-server
|
||||
self.idle_sleeper = RustServerIdleSleeper(rust_server)
|
||||
self.idle_sleeper = RustServerIdleSleeper(
|
||||
rust_server, can_empty_cache=lambda: not self._engine_paused
|
||||
)
|
||||
|
||||
def get_rust_server_class(self) -> type[RustServer]:
|
||||
return RustServer
|
||||
@@ -4958,7 +4961,7 @@ class Scheduler(
|
||||
else:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
|
||||
def is_fully_idle(self, for_health_check=False) -> bool:
|
||||
def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool:
|
||||
# Health check piggybacks on running requests in process_output.
|
||||
# Only running_batch + waiting_queue guarantee active GPU processing;
|
||||
# disagg queues (bootstrap/prealloc/transfer) may have items without
|
||||
@@ -4975,7 +4978,8 @@ class Scheduler(
|
||||
)
|
||||
|
||||
# Waiting queues: waiting + bootstrapping + preallocation + kv transfer (decode)
|
||||
idle &= len(self.waiting_queue) == 0
|
||||
if not ignore_waiting:
|
||||
idle &= len(self.waiting_queue) == 0
|
||||
|
||||
if (
|
||||
for_health_check
|
||||
@@ -5133,7 +5137,7 @@ class Scheduler(
|
||||
|
||||
def flush_cache(self, empty_cache: bool = True):
|
||||
"""Flush memory pools (e.g., KV cache, Mamba cache) and optionally empty device allocator cache."""
|
||||
if self.is_fully_idle():
|
||||
if self.is_fully_idle(ignore_waiting=self._engine_paused):
|
||||
self.cur_batch_for_debug = None
|
||||
self.last_batch = None
|
||||
self.tree_cache.reset()
|
||||
|
||||
@@ -24,9 +24,10 @@ class IdleSleeper:
|
||||
data that needs handling immediately.
|
||||
"""
|
||||
|
||||
def __init__(self, sockets):
|
||||
def __init__(self, sockets, can_empty_cache=None):
|
||||
self.poller = zmq.Poller()
|
||||
self.last_empty_time = real_time()
|
||||
self.can_empty_cache = can_empty_cache
|
||||
for s in sockets:
|
||||
self.poller.register(s, zmq.POLLIN)
|
||||
|
||||
@@ -39,7 +40,8 @@ class IdleSleeper:
|
||||
and real_time() - self.last_empty_time > self.empty_cache_interval
|
||||
):
|
||||
self.last_empty_time = real_time()
|
||||
current_platform.empty_cache()
|
||||
if self.can_empty_cache is None or self.can_empty_cache():
|
||||
current_platform.empty_cache()
|
||||
|
||||
|
||||
class RustServerIdleSleeper:
|
||||
@@ -52,10 +54,13 @@ class RustServerIdleSleeper:
|
||||
requests — or the timeout elapses.
|
||||
"""
|
||||
|
||||
def __init__(self, rust_server: RustServer, timeout_ms: int = 1000):
|
||||
def __init__(
|
||||
self, rust_server: RustServer, timeout_ms: int = 1000, can_empty_cache=None
|
||||
):
|
||||
self.rust_server = rust_server
|
||||
self.timeout_ms = timeout_ms
|
||||
self.last_empty_time = real_time()
|
||||
self.can_empty_cache = can_empty_cache
|
||||
self.empty_cache_interval = envs.SGLANG_EMPTY_CACHE_INTERVAL.get()
|
||||
|
||||
def maybe_sleep(self):
|
||||
@@ -65,4 +70,5 @@ class RustServerIdleSleeper:
|
||||
and real_time() - self.last_empty_time > self.empty_cache_interval
|
||||
):
|
||||
self.last_empty_time = real_time()
|
||||
current_platform.empty_cache()
|
||||
if self.can_empty_cache is None or self.can_empty_cache():
|
||||
current_platform.empty_cache()
|
||||
|
||||
@@ -210,9 +210,10 @@ class SchedulerWeightUpdaterManager:
|
||||
)
|
||||
|
||||
def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
|
||||
assert self.is_fully_idle(), (
|
||||
"release_memory_occupation should be called only when server is idle."
|
||||
)
|
||||
scheduler = self.scheduler
|
||||
assert self.is_fully_idle(
|
||||
ignore_waiting=scheduler is not None and scheduler._engine_paused
|
||||
), "release_memory_occupation should be called only when server is idle."
|
||||
|
||||
tags = recv_req.tags
|
||||
|
||||
@@ -223,7 +224,6 @@ class SchedulerWeightUpdaterManager:
|
||||
self.offload_tags.add(tag)
|
||||
|
||||
if GPU_MEMORY_TYPE_KV_CACHE in tags:
|
||||
scheduler = self.scheduler
|
||||
if scheduler is not None:
|
||||
if scheduler.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
for queue_name in (
|
||||
|
||||
Reference in New Issue
Block a user