Add fwd_occupancy metric to SchedulerStats and Prometheus collector (#24458)

python/sglang/srt/observability/metrics_collector.py

@@ -124,6 +124,7 @@ class SchedulerStats:
 
     # Utilization
     utilization: float = 0.0
+    fwd_occupancy: float = float("nan")
 
     # Scheduler policy
     new_token_ratio: float = 0.0
@@ -468,6 +469,12 @@ class SchedulerMetricsCollector:
             labelnames=labels.keys(),
             multiprocess_mode="mostrecent",
         )
+        self.fwd_occupancy = Gauge(
+            name="sglang:fwd_occupancy",
+            documentation="Forward pass GPU occupancy percentage.",
+            labelnames=labels.keys(),
+            multiprocess_mode="mostrecent",
+        )
 
         # =================================================================
         # Scheduler policy
@@ -1147,6 +1154,7 @@ class SchedulerMetricsCollector:
 
         # Utilization
         self._log_gauge(self.utilization, stats.utilization)
+        self._log_gauge(self.fwd_occupancy, stats.fwd_occupancy)
 
         # Scheduler policy
         self._log_gauge(self.new_token_ratio, stats.new_token_ratio)

python/sglang/srt/observability/scheduler_metrics_mixin.py

@@ -157,11 +157,12 @@ class SchedulerMetricsMixin:
                 self._mfu_log_read_bytes = 0.0
                 self._mfu_log_write_bytes = 0.0
 
+        self.fwd_occupancy = float("nan")
+
         if ENABLE_METRICS_DEVICE_TIMER:
             self._device_timer_window_batch_count = 0
             self._device_timer_window_gpu_time = 0.0
             self._device_timer_window_start = None
-            self.fwd_occupancy = float("nan")
 
             def _wrap_execution_reporter(**kwargs):
                 self._device_timer_window_gpu_time += kwargs["t"]
@@ -469,6 +470,7 @@ class SchedulerMetricsMixin:
 
             # Utilization / LoRA / HiCache
             self.calculate_utilization()
+            self.stats.fwd_occupancy = self.fwd_occupancy
             self.update_lora_metrics()
             self._log_hicache_stats()
             self.metrics_collector.log_stats(self.stats)
@@ -661,6 +663,7 @@ class SchedulerMetricsMixin:
 
             # Utilization / LoRA / HiCache
             self.calculate_utilization()
+            self.stats.fwd_occupancy = self.fwd_occupancy
             self.update_lora_metrics()
             self._log_hicache_stats()
             self.metrics_collector.log_stats(self.stats)