[Fix] scheduler: Count all pending work in DP load snapshots

total_tokens DP balancing replaces its per-rank estimate with the rank's
latest load snapshot, so a snapshot that misses work already sent to the
rank lets the controller keep sending to it. Close three gaps:

- count grammar-queue requests in num_waiting_reqs and num_total_tokens
- count the in-progress chunked request's unprefilled tokens
- publish a snapshot once newly received requests are queued, since the
  one published after a forward predates the requests sent during it
This commit is contained in:
QNan0513
2026-11-10 23:58:23 +08:00
committed by minke.yu
parent 15218b2f65
commit fef023b64f
4 changed files with 50 additions and 7 deletions
+5
View File
@@ -2189,6 +2189,10 @@ class Scheduler(
# Count successful receive/broadcast time as active; empty polls stay idle.
self.metrics_reporter.record_scheduler_active(recv_start_ns)
self.process_input_requests(recv_reqs)
if recv_reqs:
# The last snapshot may predate these requests (it is published after
# each forward, before the requests sent during it are read).
self.publish_load_snapshot(force=True)
return recv_reqs
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_REQUESTS)
@@ -2574,6 +2578,7 @@ class Scheduler(
),
get_stats=lambda: self.metrics_reporter.stats,
get_chunked_req=lambda: self.chunked_req,
get_grammar_queue=lambda: self.grammar_manager.grammar_queue,
get_disagg_prefill_bootstrap_queue=lambda: (
self.disagg_prefill_bootstrap_queue
),
@@ -46,6 +46,7 @@ class SchedulerLoadInquirer:
get_recent_cache_hit_rate: Callable
get_stats: Callable
get_chunked_req: Callable
get_grammar_queue: Callable
get_disagg_prefill_bootstrap_queue: Callable
get_disagg_prefill_inflight_queue: Callable
get_disagg_decode_prealloc_queue: Callable
@@ -138,6 +139,12 @@ class SchedulerLoadInquirer:
for req in queue
)
# Requests waiting for their grammar to compile reach the queues above
# only once it is ready, but they are already this rank's work.
grammar_queue = self.get_grammar_queue()
waiting_queues.append(grammar_queue)
pending_token_queues.append(grammar_queue)
num_waiting_reqs = sum(len(queue) for queue in waiting_queues)
num_used_tokens, kv_token_usage = (
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
@@ -145,6 +152,10 @@ class SchedulerLoadInquirer:
num_total_tokens = num_used_tokens + sum(
req.seqlen for queue in pending_token_queues for req in queue
)
chunked_req = self.get_chunked_req()
if chunked_req is not None and chunked_req.extend_end is not None:
# KV covers only the chunks run so far; the rest is still pending.
num_total_tokens += max(0, chunked_req.seqlen - chunked_req.extend_end)
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
memory = None