[Fix] scheduler: Count all pending work in DP load snapshots
total_tokens DP balancing replaces its per-rank estimate with the rank's latest load snapshot, so a snapshot that misses work already sent to the rank lets the controller keep sending to it. Close three gaps: - count grammar-queue requests in num_waiting_reqs and num_total_tokens - count the in-progress chunked request's unprefilled tokens - publish a snapshot once newly received requests are queued, since the one published after a forward predates the requests sent during it
This commit is contained in:
@@ -2189,6 +2189,10 @@ class Scheduler(
|
||||
# Count successful receive/broadcast time as active; empty polls stay idle.
|
||||
self.metrics_reporter.record_scheduler_active(recv_start_ns)
|
||||
self.process_input_requests(recv_reqs)
|
||||
if recv_reqs:
|
||||
# The last snapshot may predate these requests (it is published after
|
||||
# each forward, before the requests sent during it are read).
|
||||
self.publish_load_snapshot(force=True)
|
||||
return recv_reqs
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_REQUESTS)
|
||||
@@ -2574,6 +2578,7 @@ class Scheduler(
|
||||
),
|
||||
get_stats=lambda: self.metrics_reporter.stats,
|
||||
get_chunked_req=lambda: self.chunked_req,
|
||||
get_grammar_queue=lambda: self.grammar_manager.grammar_queue,
|
||||
get_disagg_prefill_bootstrap_queue=lambda: (
|
||||
self.disagg_prefill_bootstrap_queue
|
||||
),
|
||||
|
||||
@@ -46,6 +46,7 @@ class SchedulerLoadInquirer:
|
||||
get_recent_cache_hit_rate: Callable
|
||||
get_stats: Callable
|
||||
get_chunked_req: Callable
|
||||
get_grammar_queue: Callable
|
||||
get_disagg_prefill_bootstrap_queue: Callable
|
||||
get_disagg_prefill_inflight_queue: Callable
|
||||
get_disagg_decode_prealloc_queue: Callable
|
||||
@@ -138,6 +139,12 @@ class SchedulerLoadInquirer:
|
||||
for req in queue
|
||||
)
|
||||
|
||||
# Requests waiting for their grammar to compile reach the queues above
|
||||
# only once it is ready, but they are already this rank's work.
|
||||
grammar_queue = self.get_grammar_queue()
|
||||
waiting_queues.append(grammar_queue)
|
||||
pending_token_queues.append(grammar_queue)
|
||||
|
||||
num_waiting_reqs = sum(len(queue) for queue in waiting_queues)
|
||||
num_used_tokens, kv_token_usage = (
|
||||
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
|
||||
@@ -145,6 +152,10 @@ class SchedulerLoadInquirer:
|
||||
num_total_tokens = num_used_tokens + sum(
|
||||
req.seqlen for queue in pending_token_queues for req in queue
|
||||
)
|
||||
chunked_req = self.get_chunked_req()
|
||||
if chunked_req is not None and chunked_req.extend_end is not None:
|
||||
# KV covers only the chunks run so far; the rest is still pending.
|
||||
num_total_tokens += max(0, chunked_req.seqlen - chunked_req.extend_end)
|
||||
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
|
||||
|
||||
memory = None
|
||||
|
||||
@@ -510,6 +510,7 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
recv_requests=Mock(side_effect=[[]] * len(schedule) + [StopIteration])
|
||||
)
|
||||
scheduler.process_input_requests = Mock()
|
||||
scheduler.load_snapshot_writer = None
|
||||
scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(
|
||||
pop_bootstrapped=Mock(return_value=[])
|
||||
)
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
"""on_idle's stalled-path load publish is wall-clock bounded.
|
||||
"""When the scheduler publishes its load snapshot.
|
||||
|
||||
A non-storage no-batch stall spins on_idle without sleeping, so the gate must
|
||||
cap the O(queue) get_loads for both the DP-balancing writer and the load
|
||||
socket. CPU-only: builds a bare Scheduler with mocked collaborators, like
|
||||
test_scheduler_flush_cache.
|
||||
on_idle's stalled-path publish is wall-clock bounded: a non-storage no-batch
|
||||
stall spins on_idle without sleeping, so the gate must cap the O(queue)
|
||||
get_loads for both the DP-balancing writer and the load socket. Ingesting new
|
||||
requests publishes once they are queued. CPU-only: builds a bare Scheduler with
|
||||
mocked collaborators, like test_scheduler_flush_cache.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
from sglang.test.test_utils import (
|
||||
CustomTestCase,
|
||||
maybe_stub_sgl_kernel,
|
||||
published_topology,
|
||||
)
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
@@ -52,5 +57,26 @@ class TestOnIdleStallPublish(CustomTestCase):
|
||||
self.assertEqual(s.publish_load_snapshot.call_count, 2)
|
||||
|
||||
|
||||
class TestIngestPublish(CustomTestCase):
|
||||
def test_a_snapshot_follows_newly_queued_requests(self):
|
||||
"""A rank publishes its load after each forward, before it reads the
|
||||
requests sent meanwhile; without a publish once they are queued, DP
|
||||
balancing resets its estimate to that stale load and sends it more."""
|
||||
s = Scheduler.__new__(Scheduler)
|
||||
s._deferred_input_requests = []
|
||||
s._poll_timeout_aborts = MagicMock(return_value=[])
|
||||
s.request_receiver = MagicMock()
|
||||
s.request_receiver.recv_requests.return_value = ["req"]
|
||||
s.metrics_reporter = MagicMock()
|
||||
steps = MagicMock()
|
||||
s.process_input_requests = steps.queue
|
||||
s.publish_load_snapshot = steps.publish
|
||||
with published_topology():
|
||||
s.ingest_requests()
|
||||
self.assertEqual(
|
||||
steps.mock_calls, [call.queue(["req"]), call.publish(force=True)]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user