minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Auto Sync] Add max_total_num_tokens metric: Update scheduler_metrics_mixin.py, collector.py (20251202) (#14256)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Dan Zheng <dzheng@x.ai>
This commit is contained in:
Lianmin Zheng
2025-12-01 16:34:34 -08:00
committed by GitHub
co-authored by github-actions[bot] Dan Zheng
parent 1da59e8304
commit 796d82b107
2 changed files with 15 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/scheduler_metrics_mixin.py
+4
View File
@@ -172,6 +172,8 @@ class SchedulerMetricsMixin:
self.stats.num_grammar_queue_reqs = len(self.grammar_queue) self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
self.stats.cache_hit_rate = cache_hit_rate self.stats.cache_hit_rate = cache_hit_rate
self.stats.max_total_num_tokens = self.max_total_num_tokens
# Retract # Retract
self.stats.num_retracted_reqs = self.num_retracted_reqs self.stats.num_retracted_reqs = self.num_retracted_reqs
self.stats.num_paused_reqs = self.num_paused_reqs self.stats.num_paused_reqs = self.num_paused_reqs
@@ -320,6 +322,8 @@ class SchedulerMetricsMixin:
self.stats.num_grammar_queue_reqs = len(self.grammar_queue) self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
self.stats.cache_hit_rate = cache_hit_rate self.stats.cache_hit_rate = cache_hit_rate
self.stats.max_total_num_tokens = self.max_total_num_tokens
# Speculative decoding # Speculative decoding
self.stats.spec_accept_rate = spec_accept_rate self.stats.spec_accept_rate = spec_accept_rate
self.stats.spec_accept_length = spec_accept_length self.stats.spec_accept_length = spec_accept_length
python/sglang/srt/metrics/collector.py
+11
View File
@@ -184,6 +184,8 @@ class SchedulerStats:
num_running_reqs_offline_batch: int = 0 num_running_reqs_offline_batch: int = 0
cache_hit_rate: float = 0.0 cache_hit_rate: float = 0.0
max_total_num_tokens: int = 0
# Speculative decoding # Speculative decoding
spec_accept_length: float = 0.0 spec_accept_length: float = 0.0
spec_accept_rate: float = 0.0 spec_accept_rate: float = 0.0
@@ -294,6 +296,13 @@ class SchedulerMetricsCollector:
multiprocess_mode="mostrecent", multiprocess_mode="mostrecent",
) )
self.max_total_num_tokens = Gauge(
name="sglang:max_total_num_tokens",
documentation="Maximum total number of tokens in the KV cache pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
# Speculative decoding # Speculative decoding
self.spec_accept_length = Gauge( self.spec_accept_length = Gauge(
name="sglang:spec_accept_length", name="sglang:spec_accept_length",
@@ -640,6 +649,8 @@ class SchedulerMetricsCollector:
) )
self._log_gauge(self.cache_hit_rate, stats.cache_hit_rate) self._log_gauge(self.cache_hit_rate, stats.cache_hit_rate)
self._log_gauge(self.max_total_num_tokens, stats.max_total_num_tokens)
# Speculative decoding # Speculative decoding
self._log_gauge(self.spec_accept_length, stats.spec_accept_length) self._log_gauge(self.spec_accept_length, stats.spec_accept_length)
self._log_gauge(self.spec_accept_rate, stats.spec_accept_rate) self._log_gauge(self.spec_accept_rate, stats.spec_accept_rate)