[Fix] scheduler: Count all pending work in DP load snapshots

total_tokens DP balancing replaces its per-rank estimate with the rank's
latest load snapshot, so a snapshot that misses work already sent to the
rank lets the controller keep sending to it. Close three gaps:

- count grammar-queue requests in num_waiting_reqs and num_total_tokens
- count the in-progress chunked request's unprefilled tokens
- publish a snapshot once newly received requests are queued, since the
  one published after a forward predates the requests sent during it
This commit is contained in:
QNan0513
2026-11-10 23:58:23 +08:00
committed by minke.yu
parent 15218b2f65
commit fef023b64f
4 changed files with 50 additions and 7 deletions
+5
View File
@@ -2189,6 +2189,10 @@ class Scheduler(
# Count successful receive/broadcast time as active; empty polls stay idle.
self.metrics_reporter.record_scheduler_active(recv_start_ns)
self.process_input_requests(recv_reqs)
if recv_reqs:
# The last snapshot may predate these requests (it is published after
# each forward, before the requests sent during it are read).
self.publish_load_snapshot(force=True)
return recv_reqs
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_REQUESTS)
@@ -2574,6 +2578,7 @@ class Scheduler(
),
get_stats=lambda: self.metrics_reporter.stats,
get_chunked_req=lambda: self.chunked_req,
get_grammar_queue=lambda: self.grammar_manager.grammar_queue,
get_disagg_prefill_bootstrap_queue=lambda: (
self.disagg_prefill_bootstrap_queue
),
@@ -46,6 +46,7 @@ class SchedulerLoadInquirer:
get_recent_cache_hit_rate: Callable
get_stats: Callable
get_chunked_req: Callable
get_grammar_queue: Callable
get_disagg_prefill_bootstrap_queue: Callable
get_disagg_prefill_inflight_queue: Callable
get_disagg_decode_prealloc_queue: Callable
@@ -138,6 +139,12 @@ class SchedulerLoadInquirer:
for req in queue
)
# Requests waiting for their grammar to compile reach the queues above
# only once it is ready, but they are already this rank's work.
grammar_queue = self.get_grammar_queue()
waiting_queues.append(grammar_queue)
pending_token_queues.append(grammar_queue)
num_waiting_reqs = sum(len(queue) for queue in waiting_queues)
num_used_tokens, kv_token_usage = (
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
@@ -145,6 +152,10 @@ class SchedulerLoadInquirer:
num_total_tokens = num_used_tokens + sum(
req.seqlen for queue in pending_token_queues for req in queue
)
chunked_req = self.get_chunked_req()
if chunked_req is not None and chunked_req.extend_end is not None:
# KV covers only the chunks run so far; the rest is still pending.
num_total_tokens += max(0, chunked_req.seqlen - chunked_req.extend_end)
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
memory = None
@@ -510,6 +510,7 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
recv_requests=Mock(side_effect=[[]] * len(schedule) + [StopIteration])
)
scheduler.process_input_requests = Mock()
scheduler.load_snapshot_writer = None
scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(
pop_bootstrapped=Mock(return_value=[])
)
@@ -1,16 +1,21 @@
"""on_idle's stalled-path load publish is wall-clock bounded.
"""When the scheduler publishes its load snapshot.
A non-storage no-batch stall spins on_idle without sleeping, so the gate must
cap the O(queue) get_loads for both the DP-balancing writer and the load
socket. CPU-only: builds a bare Scheduler with mocked collaborators, like
test_scheduler_flush_cache.
on_idle's stalled-path publish is wall-clock bounded: a non-storage no-batch
stall spins on_idle without sleeping, so the gate must cap the O(queue)
get_loads for both the DP-balancing writer and the load socket. Ingesting new
requests publishes once they are queued. CPU-only: builds a bare Scheduler with
mocked collaborators, like test_scheduler_flush_cache.
"""
import unittest
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock, call, patch
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
from sglang.test.test_utils import (
CustomTestCase,
maybe_stub_sgl_kernel,
published_topology,
)
maybe_stub_sgl_kernel()
@@ -52,5 +57,26 @@ class TestOnIdleStallPublish(CustomTestCase):
self.assertEqual(s.publish_load_snapshot.call_count, 2)
class TestIngestPublish(CustomTestCase):
def test_a_snapshot_follows_newly_queued_requests(self):
"""A rank publishes its load after each forward, before it reads the
requests sent meanwhile; without a publish once they are queued, DP
balancing resets its estimate to that stale load and sends it more."""
s = Scheduler.__new__(Scheduler)
s._deferred_input_requests = []
s._poll_timeout_aborts = MagicMock(return_value=[])
s.request_receiver = MagicMock()
s.request_receiver.recv_requests.return_value = ["req"]
s.metrics_reporter = MagicMock()
steps = MagicMock()
s.process_input_requests = steps.queue
s.publish_load_snapshot = steps.publish
with published_topology():
s.ingest_requests()
self.assertEqual(
steps.mock_calls, [call.queue(["req"]), call.publish(force=True)]
)
if __name__ == "__main__":
unittest.main()