minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[sgl] expose swa and mamba cache metrics (#24396)

Browse Source
This commit is contained in:
Bi Xue
2026-05-05 20:19:50 +08:00
committed by GitHub
parent 67e8bd7a80
commit 9fb9a1cca6
2 changed files with 55 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/scheduler_runtime_checker_mixin.py
+6
View File
@@ -124,8 +124,14 @@ class PoolStats:
stats.full_token_usage = self.full_token_usage stats.full_token_usage = self.full_token_usage
if self.is_hybrid_swa: if self.is_hybrid_swa:
stats.swa_token_usage = self.swa_token_usage stats.swa_token_usage = self.swa_token_usage
stats.swa_available_tokens = self.swa_available_size
stats.swa_evictable_tokens = self.swa_evictable_size
stats.swa_used_tokens = self.swa_num_used
if self.is_hybrid_ssm: if self.is_hybrid_ssm:
stats.mamba_usage = self.mamba_usage stats.mamba_usage = self.mamba_usage
stats.mamba_available_tokens = self.mamba_available_size
stats.mamba_evictable_tokens = self.mamba_evictable_size
stats.mamba_used_tokens = self.mamba_num_used
stats.kv_available_tokens = self.full_available_size stats.kv_available_tokens = self.full_available_size
stats.kv_evictable_tokens = self.full_evictable_size stats.kv_evictable_tokens = self.full_evictable_size
stats.kv_used_tokens = self.full_num_used stats.kv_used_tokens = self.full_num_used
python/sglang/srt/observability/metrics_collector.py
+49
View File
@@ -98,6 +98,13 @@ class SchedulerStats:
kv_evictable_tokens: int = 0 kv_evictable_tokens: int = 0
kv_used_tokens: int = 0 kv_used_tokens: int = 0
swa_available_tokens: int = 0
swa_evictable_tokens: int = 0
swa_used_tokens: int = 0
mamba_available_tokens: int = 0
mamba_evictable_tokens: int = 0
mamba_used_tokens: int = 0
# Speculative decoding # Speculative decoding
spec_accept_length: float = 0.0 spec_accept_length: float = 0.0
spec_accept_rate: float = 0.0 spec_accept_rate: float = 0.0
@@ -289,6 +296,42 @@ class SchedulerMetricsCollector:
labelnames=labels.keys(), labelnames=labels.keys(),
multiprocess_mode="mostrecent", multiprocess_mode="mostrecent",
) )
self.swa_available_tokens = Gauge(
name="sglang:swa_available_tokens",
documentation="Number of free token slots in the SWA pool (hybrid-SWA only).",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.swa_evictable_tokens = Gauge(
name="sglang:swa_evictable_tokens",
documentation="Number of evictable (radix-cached) token slots in the SWA pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.swa_used_tokens = Gauge(
name="sglang:swa_used_tokens",
documentation="Number of actively used token slots in the SWA pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.mamba_available_tokens = Gauge(
name="sglang:mamba_available_tokens",
documentation="Number of free state slots in the mamba SSM pool (hybrid-SSM only).",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.mamba_evictable_tokens = Gauge(
name="sglang:mamba_evictable_tokens",
documentation="Number of evictable (radix-cached) state slots in the mamba SSM pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.mamba_used_tokens = Gauge(
name="sglang:mamba_used_tokens",
documentation="Number of actively used state slots in the mamba SSM pool.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
# ================================================================= # =================================================================
# Speculative decoding # Speculative decoding
@@ -1069,6 +1112,12 @@ class SchedulerMetricsCollector:
self._log_gauge(self.kv_available_tokens, stats.kv_available_tokens) self._log_gauge(self.kv_available_tokens, stats.kv_available_tokens)
self._log_gauge(self.kv_evictable_tokens, stats.kv_evictable_tokens) self._log_gauge(self.kv_evictable_tokens, stats.kv_evictable_tokens)
self._log_gauge(self.kv_used_tokens, stats.kv_used_tokens) self._log_gauge(self.kv_used_tokens, stats.kv_used_tokens)
self._log_gauge(self.swa_available_tokens, stats.swa_available_tokens)
self._log_gauge(self.swa_evictable_tokens, stats.swa_evictable_tokens)
self._log_gauge(self.swa_used_tokens, stats.swa_used_tokens)
self._log_gauge(self.mamba_available_tokens, stats.mamba_available_tokens)
self._log_gauge(self.mamba_evictable_tokens, stats.mamba_evictable_tokens)
self._log_gauge(self.mamba_used_tokens, stats.mamba_used_tokens)
# Speculative decoding # Speculative decoding
self._log_gauge(self.spec_accept_length, stats.spec_accept_length) self._log_gauge(self.spec_accept_length, stats.spec_accept_length)