[Metrics] Add rolling scheduler utilization counters (#37461)

Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>
Co-authored-by: pranjalssh <pranjalssh@fb.com>

test/registered/observability/test_metrics.py

@@ -193,6 +193,8 @@ class TestEnableMetrics(CustomTestCase):
             "sglang:startup_available_gpu_memory_gb",
             "sglang:startup_time_seconds",
             "sglang:startup_cuda_graph_time_seconds",
+            "sglang:scheduler_idle_seconds_total",
+            "sglang:scheduler_process_cpu_seconds_total",
         ]
         mfu_metrics = [
             "sglang:estimated_flops_per_gpu_total",
@@ -229,6 +231,7 @@ class TestEnableMetrics(CustomTestCase):
             ("sglang:realtime_tokens_total", {"mode": "decode"}),
             ("sglang:forward_execution_seconds_total", {"category": "extend"}),
             ("sglang:forward_execution_seconds_total", {"category": "decode"}),
+            ("sglang:scheduler_process_cpu_seconds_total", {}),
             ("sglang:process_cpu_seconds_total", {"component": "tokenizer"}),
             ("sglang:weight_memory_usage_gb", {"model_name": _MODEL_NAME}),
             ("sglang:kv_cache_memory_usage_gb", {"model_name": _MODEL_NAME}),

test/registered/unit/managers/test_generation_auxiliary_output.py

@@ -388,6 +388,7 @@ def test_pdmux_split_prefill_schedules_auxiliary_output_copy():
     )
     copy_done = CopyDone()
     scheduler = object.__new__(Scheduler)
+    scheduler.metrics_reporter = Mock()
     scheduler.forward_ct = 0
     scheduler._sched_idled = False
     scheduler.scripted_scheduler_hook = None

test/registered/unit/managers/test_scheduler_on_idle_load.py

@@ -27,6 +27,7 @@ class TestOnIdleStallPublish(CustomTestCase):
         s.publish_load_snapshot = MagicMock(return_value=None)
         s.load_publisher = MagicMock()
         s.load_inquirer = MagicMock()
+        s.metrics_reporter = MagicMock()
         s._last_stall_publish_ts = float("-inf")
         return s
 

test/registered/unit/managers/test_scheduler_pause_generation.py

@@ -143,6 +143,24 @@ class TestSchedulerPauseGeneration(unittest.TestCase):
         self.assertIs(scheduler.cur_batch_for_debug, original_cur_batch)
         self.assertIs(scheduler.chunked_req, original_chunked_req)
 
+    def test_paused_engine_accounting_uses_current_scheduler_state(self):
+        scheduler = self._new_scheduler()
+        scheduler.is_fully_idle = MagicMock()
+
+        for is_idle in (True, False):
+            with self.subTest(is_idle=is_idle):
+                scheduler.is_fully_idle.return_value = is_idle
+                scheduler.metrics_reporter.reset_mock()
+
+                scheduler._record_scheduler_state_for_paused_engine()
+
+                if is_idle:
+                    scheduler.metrics_reporter.record_scheduler_idle.assert_called_once_with()
+                    scheduler.metrics_reporter.record_scheduler_active.assert_not_called()
+                else:
+                    scheduler.metrics_reporter.record_scheduler_active.assert_called_once_with()
+                    scheduler.metrics_reporter.record_scheduler_idle.assert_not_called()
+
     def test_inplace_does_not_drain_overlap_queue(self):
         """in_place should not process the overlap result_queue."""
         scheduler = self._new_scheduler()

test/registered/unit/observability/test_forward_pass_metrics.py

@@ -407,6 +407,100 @@ class TestIdleMetrics(unittest.TestCase):
         self.assertEqual(self.published_occupancies, [])
 
 
+class TestSchedulerTimeAccounting(CustomTestCase):
+    def setUp(self):
+        self.reporter = _make_reporter(self, types.SimpleNamespace())
+        self.idle_seconds = []
+        self.process_cpu_seconds = []
+        self.reporter.enable_metrics = True
+        self.reporter.metrics_collector = types.SimpleNamespace(
+            increment_scheduler_idle_seconds=self.idle_seconds.append,
+            increment_scheduler_process_cpu_seconds=self.process_cpu_seconds.append,
+        )
+
+    def test_counts_idle_wall_time_and_process_cpu_time(self):
+        wall_timestamps = [
+            0,
+            1_200_000_000,
+            1_500_000_000,
+            2_700_000_000,
+            3_000_000_000,
+            4_100_000_000,
+        ]
+        process_cpu_timestamps = [
+            0,
+            400_000_000,
+            1_200_000_000,
+            1_900_000_000,
+        ]
+        with (
+            patch(
+                "sglang.srt.managers.scheduler_components.metrics_reporter.time.monotonic_ns",
+                side_effect=wall_timestamps,
+            ),
+            patch(
+                "sglang.srt.managers.scheduler_components.metrics_reporter.time.process_time_ns",
+                side_effect=process_cpu_timestamps,
+            ),
+        ):
+            self.reporter.start_scheduler_time_accounting()
+            self.reporter.record_scheduler_idle()
+            self.reporter.record_scheduler_active()
+            self.reporter.record_scheduler_active()
+            self.reporter.record_scheduler_idle()
+            self.reporter.record_scheduler_idle()
+
+        self.assertAlmostEqual(sum(self.idle_seconds), 2.6)
+        self.assertAlmostEqual(sum(self.process_cpu_seconds), 1.9)
+
+    def test_state_transitions_accumulate_until_periodic_update(self):
+        with (
+            patch(
+                "sglang.srt.managers.scheduler_components.metrics_reporter.time.monotonic_ns",
+                side_effect=[0, 200_000_000, 400_000_000, 700_000_000, 1_100_000_000],
+            ),
+            patch(
+                "sglang.srt.managers.scheduler_components.metrics_reporter.time.process_time_ns",
+                side_effect=[0, 300_000_000],
+            ) as process_time,
+        ):
+            self.reporter.start_scheduler_time_accounting()
+            accounting = self.reporter._scheduler_time_accounting
+            self.reporter.record_scheduler_active()
+            self.reporter.record_scheduler_idle()
+            self.reporter.record_scheduler_active()
+            self.assertEqual(self.idle_seconds, [])
+            self.assertEqual(self.process_cpu_seconds, [])
+            self.assertEqual(
+                self.reporter._scheduler_time_accounting.accumulate_idle_ns,
+                500_000_000,
+            )
+            self.reporter.record_scheduler_active()
+
+        self.assertIs(self.reporter._scheduler_time_accounting, accounting)
+        self.assertEqual(process_time.call_count, 2)
+        self.assertEqual(self.idle_seconds, [0.5])
+        self.assertAlmostEqual(self.process_cpu_seconds[0], 0.3)
+
+    def test_periodic_update_skips_zero_idle_but_records_cpu_sample(self):
+        with (
+            patch(
+                "sglang.srt.managers.scheduler_components.metrics_reporter.time.monotonic_ns",
+                side_effect=[0, 0, 1_000_000_000],
+            ),
+            patch(
+                "sglang.srt.managers.scheduler_components.metrics_reporter.time.process_time_ns",
+                side_effect=[0, 0],
+            ),
+        ):
+            self.reporter.start_scheduler_time_accounting()
+            self.reporter.record_scheduler_active()
+            self.reporter.record_scheduler_active()
+
+        self.assertEqual(self.idle_seconds, [])
+        self.assertEqual(self.process_cpu_seconds, [0.0])
+
+
 class TestEstimatedPrefillPerf(CustomTestCase):
     """Causal pair count behind ``est. prefill TFLOPS/s`` and ``estimated_flops``."""