diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 8dc1616171..f0d8533e03 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -829,6 +829,7 @@ class Scheduler( self.ipc_channels.recv_from_tokenizer, self.ipc_channels.recv_from_rpc, ], + can_empty_cache=lambda: not self._engine_paused, ) else: self.idle_sleeper = None @@ -2269,7 +2270,9 @@ class Scheduler( # drains its request ring (rust_server_mode) instead of a zmq socket. self.recv_from_tokenizer = rust_server # Park the idle loop on the request ring within the rank-0 rust-server - self.idle_sleeper = RustServerIdleSleeper(rust_server) + self.idle_sleeper = RustServerIdleSleeper( + rust_server, can_empty_cache=lambda: not self._engine_paused + ) def get_rust_server_class(self) -> type[RustServer]: return RustServer @@ -4958,7 +4961,7 @@ class Scheduler( else: self.metrics_reporter.record_scheduler_active() - def is_fully_idle(self, for_health_check=False) -> bool: + def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool: # Health check piggybacks on running requests in process_output. # Only running_batch + waiting_queue guarantee active GPU processing; # disagg queues (bootstrap/prealloc/transfer) may have items without @@ -4975,7 +4978,8 @@ class Scheduler( ) # Waiting queues: waiting + bootstrapping + preallocation + kv transfer (decode) - idle &= len(self.waiting_queue) == 0 + if not ignore_waiting: + idle &= len(self.waiting_queue) == 0 if ( for_health_check @@ -5133,7 +5137,7 @@ class Scheduler( def flush_cache(self, empty_cache: bool = True): """Flush memory pools (e.g., KV cache, Mamba cache) and optionally empty device allocator cache.""" - if self.is_fully_idle(): + if self.is_fully_idle(ignore_waiting=self._engine_paused): self.cur_batch_for_debug = None self.last_batch = None self.tree_cache.reset() diff --git a/python/sglang/srt/managers/scheduler_components/idle_sleeper.py b/python/sglang/srt/managers/scheduler_components/idle_sleeper.py index 54099ae3ab..83127e8247 100644 --- a/python/sglang/srt/managers/scheduler_components/idle_sleeper.py +++ b/python/sglang/srt/managers/scheduler_components/idle_sleeper.py @@ -24,9 +24,10 @@ class IdleSleeper: data that needs handling immediately. """ - def __init__(self, sockets): + def __init__(self, sockets, can_empty_cache=None): self.poller = zmq.Poller() self.last_empty_time = real_time() + self.can_empty_cache = can_empty_cache for s in sockets: self.poller.register(s, zmq.POLLIN) @@ -39,7 +40,8 @@ class IdleSleeper: and real_time() - self.last_empty_time > self.empty_cache_interval ): self.last_empty_time = real_time() - current_platform.empty_cache() + if self.can_empty_cache is None or self.can_empty_cache(): + current_platform.empty_cache() class RustServerIdleSleeper: @@ -52,10 +54,13 @@ class RustServerIdleSleeper: requests — or the timeout elapses. """ - def __init__(self, rust_server: RustServer, timeout_ms: int = 1000): + def __init__( + self, rust_server: RustServer, timeout_ms: int = 1000, can_empty_cache=None + ): self.rust_server = rust_server self.timeout_ms = timeout_ms self.last_empty_time = real_time() + self.can_empty_cache = can_empty_cache self.empty_cache_interval = envs.SGLANG_EMPTY_CACHE_INTERVAL.get() def maybe_sleep(self): @@ -65,4 +70,5 @@ class RustServerIdleSleeper: and real_time() - self.last_empty_time > self.empty_cache_interval ): self.last_empty_time = real_time() - current_platform.empty_cache() + if self.can_empty_cache is None or self.can_empty_cache(): + current_platform.empty_cache() diff --git a/python/sglang/srt/managers/scheduler_components/weight_updater.py b/python/sglang/srt/managers/scheduler_components/weight_updater.py index 9c838e59ff..adc971baee 100644 --- a/python/sglang/srt/managers/scheduler_components/weight_updater.py +++ b/python/sglang/srt/managers/scheduler_components/weight_updater.py @@ -210,9 +210,10 @@ class SchedulerWeightUpdaterManager: ) def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput): - assert self.is_fully_idle(), ( - "release_memory_occupation should be called only when server is idle." - ) + scheduler = self.scheduler + assert self.is_fully_idle( + ignore_waiting=scheduler is not None and scheduler._engine_paused + ), "release_memory_occupation should be called only when server is idle." tags = recv_req.tags @@ -223,7 +224,6 @@ class SchedulerWeightUpdaterManager: self.offload_tags.add(tag) if GPU_MEMORY_TYPE_KV_CACHE in tags: - scheduler = self.scheduler if scheduler is not None: if scheduler.disaggregation_mode == DisaggregationMode.DECODE: for queue_name in ( diff --git a/test/registered/unit/managers/test_scheduler_flush_cache_after_retract.py b/test/registered/unit/managers/test_scheduler_flush_cache_after_retract.py new file mode 100644 index 0000000000..2cc85d60f9 --- /dev/null +++ b/test/registered/unit/managers/test_scheduler_flush_cache_after_retract.py @@ -0,0 +1,108 @@ +import unittest +from collections import deque +from unittest.mock import MagicMock + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() + +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.managers.scheduler import Scheduler + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def _make_req(is_retracted: bool) -> MagicMock: + req = MagicMock() + req.is_retracted = is_retracted + return req + + +class TestSchedulerFlushCacheAfterRetract(unittest.TestCase): + """Regression coverage for flush_cache() no-oping while the engine is paused. + + With generation paused and running_batch drained, nothing in waiting_queue + holds KV/pool state: neither requests pause_generation(mode="retract") just + moved there, nor requests that were never scheduled at all. The latter are + always present under high concurrency (in-flight targets exceed engine + parallelism), where gating the flush on them starves weight updates until + the caller's retry budget times out (reproduced in RL training at 512 + in-flight requests).""" + + def _new_scheduler(self) -> Scheduler: + scheduler = Scheduler.__new__(Scheduler) + scheduler._engine_paused = True + scheduler.enable_overlap = False + scheduler.last_batch = None + scheduler.cur_batch = None + scheduler.chunked_req = None + scheduler.disaggregation_mode = DisaggregationMode.NULL + scheduler.running_batch = MagicMock() + scheduler.running_batch.is_empty.return_value = True + scheduler.result_queue = deque() + scheduler.waiting_queue = [] + scheduler.dllm_manager = MagicMock() + scheduler.dllm_manager.any_staging_reqs.return_value = False + scheduler._pp_microbatches_drained = MagicMock(return_value=True) + scheduler.grammar_manager = MagicMock() + scheduler.grammar_manager.grammar_queue = [] + scheduler.enable_hierarchical_cache = False + scheduler.enable_hisparse = False + scheduler.tree_cache = MagicMock() + scheduler.req_to_token_pool = MagicMock() + scheduler.token_to_kv_pool_allocator = MagicMock() + scheduler.draft_worker = None + scheduler.metrics_reporter = MagicMock() + return scheduler + + def test_flush_cache_succeeds_when_only_retracted_reqs_are_waiting(self): + scheduler = self._new_scheduler() + scheduler.waiting_queue = [_make_req(is_retracted=True)] + + self.assertTrue(scheduler.flush_cache()) + scheduler.tree_cache.reset.assert_called_once() + scheduler.req_to_token_pool.clear.assert_called_once() + scheduler.token_to_kv_pool_allocator.clear.assert_called_once() + + def test_flush_cache_succeeds_with_never_scheduled_reqs_while_paused(self): + """A request that was queued but never scheduled holds no KV/pool state + either; a paused-engine flush must not starve on it.""" + scheduler = self._new_scheduler() + scheduler.waiting_queue = [_make_req(is_retracted=False)] + + self.assertTrue(scheduler.flush_cache()) + scheduler.tree_cache.reset.assert_called_once() + + def test_flush_cache_succeeds_on_mixed_queue_while_paused(self): + scheduler = self._new_scheduler() + scheduler.waiting_queue = [ + _make_req(is_retracted=True), + _make_req(is_retracted=False), + ] + + self.assertTrue(scheduler.flush_cache()) + scheduler.tree_cache.reset.assert_called_once() + + def test_is_fully_idle_default_ignores_nothing(self): + """Regression guard: every other is_fully_idle() caller passes no args, + so a non-empty waiting_queue must still report not-idle by default.""" + scheduler = self._new_scheduler() + scheduler.waiting_queue = [_make_req(is_retracted=True)] + + self.assertFalse(scheduler.is_fully_idle()) + self.assertTrue(scheduler.is_fully_idle(ignore_waiting=True)) + + def test_flush_cache_does_not_ignore_waiting_reqs_while_unpaused(self): + """ignore_waiting is gated on _engine_paused inside flush_cache — a + flush issued without pausing first must not be silently permissive.""" + scheduler = self._new_scheduler() + scheduler._engine_paused = False + scheduler.waiting_queue = [_make_req(is_retracted=True)] + + self.assertFalse(scheduler.flush_cache()) + scheduler.tree_cache.reset.assert_not_called() + + +if __name__ == "__main__": + unittest.main()