[Metrics] Fix fwd_occupancy reading NaN on every decode log line; probe-free base-a (#27836)

.github/workflows/_pr-test-stage.yml

@@ -60,7 +60,9 @@ on:
 # defines must be redeclared here for the called job to see the same context.
 env:
   SGLANG_IS_IN_CI: true
-  SGLANG_ENABLE_ASYNC_ASSERT: true
+  # base-a runs the fwd-occupancy sanity kit; async-assert probes add GPU work
+  # and skew the occupancy measurement, so keep them off in that stage.
+  SGLANG_ENABLE_ASYNC_ASSERT: ${{ startsWith(inputs.self_name, 'base-a') && 'false' || 'true' }}
   SGLANG_CUDA_COREDUMP: "1"
   SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
   SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }}

python/sglang/srt/managers/scheduler_components/metrics_reporter.py

@@ -990,17 +990,19 @@ class SchedulerMetricsReporter:
         self.forward_pass_device_timer._report()
         now = time.perf_counter()
         if self._device_timer_window_batch_count == 0:
+            # Window start: keep the last published value instead of NaN-ing
+            # the gauge. Readers sample it asynchronously, and the window
+            # boundary can phase-lock with the decode-log cadence, turning a
+            # one-tick NaN into NaN on every log line. NaN is published only
+            # when truly stale (reset_device_timer_window after idle).
             self._device_timer_window_start = now
             self._device_timer_window_gpu_time = 0.0
-            cpu_time = 0
-            self.fwd_occupancy = float("nan")
         else:
             cpu_time = now - self._device_timer_window_start
-            self.fwd_occupancy = min(
-                self._device_timer_window_gpu_time / cpu_time * 100, 100
-            )
-        # ratio = self._device_timer_window_gpu_time / cpu_time if cpu_time > 0 else float("nan")
-        # print(f"{self._device_timer_window_batch_count=} {self.fwd_occupancy=}, {self._device_timer_window_gpu_time=}, {cpu_time=}, {ratio=}")
+            if cpu_time > 0:
+                self.fwd_occupancy = min(
+                    self._device_timer_window_gpu_time / cpu_time * 100, 100
+                )
         self._device_timer_window_batch_count += 1
         if (
             self._device_timer_window_batch_count

test/registered/core/test_basic_sanity.py

@@ -34,8 +34,9 @@ class TestBasicSanity(
 ):
     served_model_name = DEFAULT_MODEL_NAME_FOR_TEST
     # 5090 + Llama-3.1-8B single-batch decode with overlap scheduler +
-    # cuda graph measured ~99 median in CI; keep ~2pp headroom.
-    fwd_occupancy_threshold = 97.0
+    # cuda graph measured ~99 median in CI; async-assert probes are off in
+    # base-a, so the threshold can sit right under the measured median.
+    fwd_occupancy_threshold = 99.0
 
     @classmethod
     def setUpClass(cls):

test/registered/core/test_basic_sanity_eagle3.py

@@ -33,10 +33,11 @@ class TestBasicSanityEagle3(
     CustomTestCase,
 ):
     served_model_name = DEFAULT_TARGET_MODEL_EAGLE3
-    # CUDA 5090 + Llama-3.1-8B measured ~99 median in CI. AMD EAGLE3
-    # currently sustains lower single-batch occupancy and needs a longer
-    # measurement window to avoid too few non-NaN samples.
-    fwd_occupancy_threshold = 80.0 if is_in_amd_ci() else 97.0
+    # CUDA 5090 + Llama-3.1-8B measured ~99 median in CI with async-assert
+    # probes off in base-a. AMD EAGLE3 currently sustains lower single-batch
+    # occupancy and needs a longer measurement window to avoid too few
+    # non-NaN samples.
+    fwd_occupancy_threshold = 80.0 if is_in_amd_ci() else 98.0
     fwd_occupancy_max_new_tokens = 4096 if is_in_amd_ci() else 2048
     fwd_occupancy_acc_length_threshold: float = 1.6