[Fix] scheduler: Count all pending work in DP load snapshots
total_tokens DP balancing replaces its per-rank estimate with the rank's latest load snapshot, so a snapshot that misses work already sent to the rank lets the controller keep sending to it. Close three gaps: - count grammar-queue requests in num_waiting_reqs and num_total_tokens - count the in-progress chunked request's unprefilled tokens - publish a snapshot once newly received requests are queued, since the one published after a forward predates the requests sent during it
This commit is contained in:
@@ -2189,6 +2189,10 @@ class Scheduler(
|
||||
# Count successful receive/broadcast time as active; empty polls stay idle.
|
||||
self.metrics_reporter.record_scheduler_active(recv_start_ns)
|
||||
self.process_input_requests(recv_reqs)
|
||||
if recv_reqs:
|
||||
# The last snapshot may predate these requests (it is published after
|
||||
# each forward, before the requests sent during it are read).
|
||||
self.publish_load_snapshot(force=True)
|
||||
return recv_reqs
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_REQUESTS)
|
||||
@@ -2574,6 +2578,7 @@ class Scheduler(
|
||||
),
|
||||
get_stats=lambda: self.metrics_reporter.stats,
|
||||
get_chunked_req=lambda: self.chunked_req,
|
||||
get_grammar_queue=lambda: self.grammar_manager.grammar_queue,
|
||||
get_disagg_prefill_bootstrap_queue=lambda: (
|
||||
self.disagg_prefill_bootstrap_queue
|
||||
),
|
||||
|
||||
@@ -46,6 +46,7 @@ class SchedulerLoadInquirer:
|
||||
get_recent_cache_hit_rate: Callable
|
||||
get_stats: Callable
|
||||
get_chunked_req: Callable
|
||||
get_grammar_queue: Callable
|
||||
get_disagg_prefill_bootstrap_queue: Callable
|
||||
get_disagg_prefill_inflight_queue: Callable
|
||||
get_disagg_decode_prealloc_queue: Callable
|
||||
@@ -138,6 +139,12 @@ class SchedulerLoadInquirer:
|
||||
for req in queue
|
||||
)
|
||||
|
||||
# Requests waiting for their grammar to compile reach the queues above
|
||||
# only once it is ready, but they are already this rank's work.
|
||||
grammar_queue = self.get_grammar_queue()
|
||||
waiting_queues.append(grammar_queue)
|
||||
pending_token_queues.append(grammar_queue)
|
||||
|
||||
num_waiting_reqs = sum(len(queue) for queue in waiting_queues)
|
||||
num_used_tokens, kv_token_usage = (
|
||||
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
|
||||
@@ -145,6 +152,10 @@ class SchedulerLoadInquirer:
|
||||
num_total_tokens = num_used_tokens + sum(
|
||||
req.seqlen for queue in pending_token_queues for req in queue
|
||||
)
|
||||
chunked_req = self.get_chunked_req()
|
||||
if chunked_req is not None and chunked_req.extend_end is not None:
|
||||
# KV covers only the chunks run so far; the rest is still pending.
|
||||
num_total_tokens += max(0, chunked_req.seqlen - chunked_req.extend_end)
|
||||
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
|
||||
|
||||
memory = None
|
||||
|
||||
Reference in New Issue
Block a user