[scheduler] Zero gen_throughput and flush KV events on pause (#24003)

Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
Bruce Changlong Xu
2026-06-02 16:43:04 +08:00
committed by GitHub
co-authored by Ke Bao
parent b55570d38e
commit 172bd8e6b9
2 changed files with 17 additions and 0 deletions
+13
View File
@@ -3723,6 +3723,19 @@ class Scheduler(
self.running_batch.batch_is_full = False
self.chunked_req = None
# Surface the paused state to dashboards immediately. The scheduler
# event loop short-circuits before reaching ``on_idle`` while paused,
# so without this hop ``gen_throughput`` retains its last non-zero
# value and KV events are not flushed for the entire pause window
# (e.g. across a weight update). Zero the gauge, force a one-shot
# idle log by resetting the rate-limit timestamp, and flush pending
# KV events.
self.metrics_reporter.last_gen_throughput = 0.0
if self.metrics_reporter.current_scheduler_metrics_enabled:
self.metrics_reporter.metrics_collector.last_log_time = 0.0
self.metrics_reporter._maybe_log_idle_metrics()
self.kv_events_publisher.publish_kv_events()
def continue_generation(self, recv_req: ContinueGenerationReqInput):
if recv_req.torch_empty_cache:
before_mb = torch.cuda.memory_reserved() / (1024 * 1024)