minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add realtime token counter metrics (#15198)

Browse Source
This commit is contained in:
fzyzcjy
2025-12-17 21:20:24 +08:00
committed by GitHub
parent b12b40dea4
commit 3e690cce53
2 changed files with 37 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/scheduler_metrics_mixin.py
+10
View File
@@ -216,6 +216,11 @@ class SchedulerMetricsMixin:
self.disagg_decode_transfer_queue.queue self.disagg_decode_transfer_queue.queue
) )
self.metrics_collector.increment_realtime_tokens(
prefill_compute_tokens=adder.log_input_tokens,
prefill_cache_tokens=adder.log_hit_tokens,
)
# Others # Others
self.calculate_utilization() self.calculate_utilization()
self.metrics_collector.log_stats(self.stats) self.metrics_collector.log_stats(self.stats)
@@ -227,6 +232,7 @@ class SchedulerMetricsMixin:
): ):
batch = running_batch or self.running_batch batch = running_batch or self.running_batch
last_num_generated_tokens = self.num_generated_tokens
gap_latency = time.perf_counter() - self.last_decode_stats_tic gap_latency = time.perf_counter() - self.last_decode_stats_tic
self.last_decode_stats_tic = time.perf_counter() self.last_decode_stats_tic = time.perf_counter()
self.last_gen_throughput = self.num_generated_tokens / gap_latency self.last_gen_throughput = self.num_generated_tokens / gap_latency
@@ -367,6 +373,10 @@ class SchedulerMetricsMixin:
self.disagg_decode_transfer_queue.queue self.disagg_decode_transfer_queue.queue
) )
self.metrics_collector.increment_realtime_tokens(
decode_tokens=last_num_generated_tokens
)
# Others # Others
self.calculate_utilization() self.calculate_utilization()
self.metrics_collector.log_stats(self.stats) self.metrics_collector.log_stats(self.stats)
python/sglang/srt/metrics/collector.py
+27
View File
@@ -632,6 +632,22 @@ class SchedulerMetricsCollector:
multiprocess_mode="mostrecent", multiprocess_mode="mostrecent",
) )
self.realtime_prefill_compute_tokens_total = Counter(
name="sglang:realtime_prefill_compute_tokens_total",
documentation="Total number of prefill compute tokens processed (updated on each log interval).",
labelnames=labels.keys(),
)
self.realtime_prefill_cache_tokens_total = Counter(
name="sglang:realtime_prefill_cache_tokens_total",
documentation="Total number of prefill cache tokens processed (updated on each log interval).",
labelnames=labels.keys(),
)
self.realtime_decode_tokens_total = Counter(
name="sglang:realtime_decode_tokens_total",
documentation="Total number of decode tokens processed (updated on each log interval).",
labelnames=labels.keys(),
)
def _log_gauge(self, gauge, data: Union[int, float]) -> None: def _log_gauge(self, gauge, data: Union[int, float]) -> None:
# Convenience function for logging to gauge. # Convenience function for logging to gauge.
gauge.labels(**self.labels).set(data) gauge.labels(**self.labels).set(data)
@@ -660,6 +676,17 @@ class SchedulerMetricsCollector:
mode = "decode_cuda_graph" if value else "decode_none" mode = "decode_cuda_graph" if value else "decode_none"
self.cuda_graph_passes_total.labels(**self.labels, mode=mode).inc(1) self.cuda_graph_passes_total.labels(**self.labels, mode=mode).inc(1)
def increment_realtime_tokens(
self, prefill_compute_tokens=0, prefill_cache_tokens=0, decode_tokens=0
):
self.realtime_prefill_compute_tokens_total.labels(**self.labels).inc(
prefill_compute_tokens
)
self.realtime_prefill_cache_tokens_total.labels(**self.labels).inc(
prefill_cache_tokens
)
self.realtime_decode_tokens_total.labels(**self.labels).inc(decode_tokens)
def log_stats(self, stats: SchedulerStats) -> None: def log_stats(self, stats: SchedulerStats) -> None:
self._log_gauge(self.num_running_reqs, stats.num_running_reqs) self._log_gauge(self.num_running_reqs, stats.num_running_reqs)
self._log_gauge(self.num_used_tokens, stats.num_used_tokens) self._log_gauge(self.num_used_tokens, stats.num_used_tokens)