diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 908dfbd9b0..732a6a51b0 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2189,6 +2189,10 @@ class Scheduler( # Count successful receive/broadcast time as active; empty polls stay idle. self.metrics_reporter.record_scheduler_active(recv_start_ns) self.process_input_requests(recv_reqs) + if recv_reqs: + # The last snapshot may predate these requests (it is published after + # each forward, before the requests sent during it are read). + self.publish_load_snapshot(force=True) return recv_reqs @scheduler_stage_method(SCHEDULER_STAGE_PROCESS_REQUESTS) @@ -2574,6 +2578,7 @@ class Scheduler( ), get_stats=lambda: self.metrics_reporter.stats, get_chunked_req=lambda: self.chunked_req, + get_grammar_queue=lambda: self.grammar_manager.grammar_queue, get_disagg_prefill_bootstrap_queue=lambda: ( self.disagg_prefill_bootstrap_queue ), diff --git a/python/sglang/srt/managers/scheduler_components/load_inquirer.py b/python/sglang/srt/managers/scheduler_components/load_inquirer.py index 5d1dfaed8d..b719b5de5b 100644 --- a/python/sglang/srt/managers/scheduler_components/load_inquirer.py +++ b/python/sglang/srt/managers/scheduler_components/load_inquirer.py @@ -46,6 +46,7 @@ class SchedulerLoadInquirer: get_recent_cache_hit_rate: Callable get_stats: Callable get_chunked_req: Callable + get_grammar_queue: Callable get_disagg_prefill_bootstrap_queue: Callable get_disagg_prefill_inflight_queue: Callable get_disagg_decode_prealloc_queue: Callable @@ -138,6 +139,12 @@ class SchedulerLoadInquirer: for req in queue ) + # Requests waiting for their grammar to compile reach the queues above + # only once it is ready, but they are already this rank's work. + grammar_queue = self.get_grammar_queue() + waiting_queues.append(grammar_queue) + pending_token_queues.append(grammar_queue) + num_waiting_reqs = sum(len(queue) for queue in waiting_queues) num_used_tokens, kv_token_usage = ( self.pool_stats_observer.get_pool_stats().get_kv_token_stats() @@ -145,6 +152,10 @@ class SchedulerLoadInquirer: num_total_tokens = num_used_tokens + sum( req.seqlen for queue in pending_token_queues for req in queue ) + chunked_req = self.get_chunked_req() + if chunked_req is not None and chunked_req.extend_end is not None: + # KV covers only the chunks run so far; the rest is still pending. + num_total_tokens += max(0, chunked_req.seqlen - chunked_req.extend_end) num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens) memory = None diff --git a/test/registered/unit/managers/test_disagg_idle_step_counters.py b/test/registered/unit/managers/test_disagg_idle_step_counters.py index 2719c6dd8e..e24a71b73f 100644 --- a/test/registered/unit/managers/test_disagg_idle_step_counters.py +++ b/test/registered/unit/managers/test_disagg_idle_step_counters.py @@ -510,6 +510,7 @@ class TestSchedulerIdleStepCounters(CustomTestCase): recv_requests=Mock(side_effect=[[]] * len(schedule) + [StopIteration]) ) scheduler.process_input_requests = Mock() + scheduler.load_snapshot_writer = None scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace( pop_bootstrapped=Mock(return_value=[]) ) diff --git a/test/registered/unit/managers/test_scheduler_on_idle_load.py b/test/registered/unit/managers/test_scheduler_on_idle_load.py index 94aa382733..1bbcb112a5 100644 --- a/test/registered/unit/managers/test_scheduler_on_idle_load.py +++ b/test/registered/unit/managers/test_scheduler_on_idle_load.py @@ -1,16 +1,21 @@ -"""on_idle's stalled-path load publish is wall-clock bounded. +"""When the scheduler publishes its load snapshot. -A non-storage no-batch stall spins on_idle without sleeping, so the gate must -cap the O(queue) get_loads for both the DP-balancing writer and the load -socket. CPU-only: builds a bare Scheduler with mocked collaborators, like -test_scheduler_flush_cache. +on_idle's stalled-path publish is wall-clock bounded: a non-storage no-batch +stall spins on_idle without sleeping, so the gate must cap the O(queue) +get_loads for both the DP-balancing writer and the load socket. Ingesting new +requests publishes once they are queued. CPU-only: builds a bare Scheduler with +mocked collaborators, like test_scheduler_flush_cache. """ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel +from sglang.test.test_utils import ( + CustomTestCase, + maybe_stub_sgl_kernel, + published_topology, +) maybe_stub_sgl_kernel() @@ -52,5 +57,26 @@ class TestOnIdleStallPublish(CustomTestCase): self.assertEqual(s.publish_load_snapshot.call_count, 2) +class TestIngestPublish(CustomTestCase): + def test_a_snapshot_follows_newly_queued_requests(self): + """A rank publishes its load after each forward, before it reads the + requests sent meanwhile; without a publish once they are queued, DP + balancing resets its estimate to that stale load and sends it more.""" + s = Scheduler.__new__(Scheduler) + s._deferred_input_requests = [] + s._poll_timeout_aborts = MagicMock(return_value=[]) + s.request_receiver = MagicMock() + s.request_receiver.recv_requests.return_value = ["req"] + s.metrics_reporter = MagicMock() + steps = MagicMock() + s.process_input_requests = steps.queue + s.publish_load_snapshot = steps.publish + with published_topology(): + s.ingest_requests() + self.assertEqual( + steps.mock_calls, [call.queue(["req"]), call.publish(force=True)] + ) + + if __name__ == "__main__": unittest.main()