[scheduler] Zero gen_throughput and flush KV events on pause (#24003)
Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
co-authored by
Ke Bao
parent
b55570d38e
commit
172bd8e6b9
@@ -3723,6 +3723,19 @@ class Scheduler(
|
||||
self.running_batch.batch_is_full = False
|
||||
self.chunked_req = None
|
||||
|
||||
# Surface the paused state to dashboards immediately. The scheduler
|
||||
# event loop short-circuits before reaching ``on_idle`` while paused,
|
||||
# so without this hop ``gen_throughput`` retains its last non-zero
|
||||
# value and KV events are not flushed for the entire pause window
|
||||
# (e.g. across a weight update). Zero the gauge, force a one-shot
|
||||
# idle log by resetting the rate-limit timestamp, and flush pending
|
||||
# KV events.
|
||||
self.metrics_reporter.last_gen_throughput = 0.0
|
||||
if self.metrics_reporter.current_scheduler_metrics_enabled:
|
||||
self.metrics_reporter.metrics_collector.last_log_time = 0.0
|
||||
self.metrics_reporter._maybe_log_idle_metrics()
|
||||
self.kv_events_publisher.publish_kv_events()
|
||||
|
||||
def continue_generation(self, recv_req: ContinueGenerationReqInput):
|
||||
if recv_req.torch_empty_cache:
|
||||
before_mb = torch.cuda.memory_reserved() / (1024 * 1024)
|
||||
|
||||
Reference in New Issue
Block a user