From 16124fc9b21b0988ecc2eba4e810b1893e906cba Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 10 Jun 2026 15:42:19 -0700 Subject: [PATCH] [Metrics] Fix `fwd_occupancy` reading NaN on every decode log line; probe-free `base-a` (#27836) --- .github/workflows/_pr-test-stage.yml | 4 +++- .../scheduler_components/metrics_reporter.py | 16 +++++++++------- test/registered/core/test_basic_sanity.py | 5 +++-- test/registered/core/test_basic_sanity_eagle3.py | 9 +++++---- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 89d4068a1..bdf7f70f2 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -60,7 +60,9 @@ on: # defines must be redeclared here for the called job to see the same context. env: SGLANG_IS_IN_CI: true - SGLANG_ENABLE_ASYNC_ASSERT: true + # base-a runs the fwd-occupancy sanity kit; async-assert probes add GPU work + # and skew the occupancy measurement, so keep them off in that stage. + SGLANG_ENABLE_ASYNC_ASSERT: ${{ startsWith(inputs.self_name, 'base-a') && 'false' || 'true' }} SGLANG_CUDA_COREDUMP: "1" SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }} diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index 50f739223..566461116 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -990,17 +990,19 @@ class SchedulerMetricsReporter: self.forward_pass_device_timer._report() now = time.perf_counter() if self._device_timer_window_batch_count == 0: + # Window start: keep the last published value instead of NaN-ing + # the gauge. Readers sample it asynchronously, and the window + # boundary can phase-lock with the decode-log cadence, turning a + # one-tick NaN into NaN on every log line. NaN is published only + # when truly stale (reset_device_timer_window after idle). self._device_timer_window_start = now self._device_timer_window_gpu_time = 0.0 - cpu_time = 0 - self.fwd_occupancy = float("nan") else: cpu_time = now - self._device_timer_window_start - self.fwd_occupancy = min( - self._device_timer_window_gpu_time / cpu_time * 100, 100 - ) - # ratio = self._device_timer_window_gpu_time / cpu_time if cpu_time > 0 else float("nan") - # print(f"{self._device_timer_window_batch_count=} {self.fwd_occupancy=}, {self._device_timer_window_gpu_time=}, {cpu_time=}, {ratio=}") + if cpu_time > 0: + self.fwd_occupancy = min( + self._device_timer_window_gpu_time / cpu_time * 100, 100 + ) self._device_timer_window_batch_count += 1 if ( self._device_timer_window_batch_count diff --git a/test/registered/core/test_basic_sanity.py b/test/registered/core/test_basic_sanity.py index 1b58ed833..33ac9e709 100644 --- a/test/registered/core/test_basic_sanity.py +++ b/test/registered/core/test_basic_sanity.py @@ -34,8 +34,9 @@ class TestBasicSanity( ): served_model_name = DEFAULT_MODEL_NAME_FOR_TEST # 5090 + Llama-3.1-8B single-batch decode with overlap scheduler + - # cuda graph measured ~99 median in CI; keep ~2pp headroom. - fwd_occupancy_threshold = 97.0 + # cuda graph measured ~99 median in CI; async-assert probes are off in + # base-a, so the threshold can sit right under the measured median. + fwd_occupancy_threshold = 99.0 @classmethod def setUpClass(cls): diff --git a/test/registered/core/test_basic_sanity_eagle3.py b/test/registered/core/test_basic_sanity_eagle3.py index ce3c4d192..d435cd0c2 100644 --- a/test/registered/core/test_basic_sanity_eagle3.py +++ b/test/registered/core/test_basic_sanity_eagle3.py @@ -33,10 +33,11 @@ class TestBasicSanityEagle3( CustomTestCase, ): served_model_name = DEFAULT_TARGET_MODEL_EAGLE3 - # CUDA 5090 + Llama-3.1-8B measured ~99 median in CI. AMD EAGLE3 - # currently sustains lower single-batch occupancy and needs a longer - # measurement window to avoid too few non-NaN samples. - fwd_occupancy_threshold = 80.0 if is_in_amd_ci() else 97.0 + # CUDA 5090 + Llama-3.1-8B measured ~99 median in CI with async-assert + # probes off in base-a. AMD EAGLE3 currently sustains lower single-batch + # occupancy and needs a longer measurement window to avoid too few + # non-NaN samples. + fwd_occupancy_threshold = 80.0 if is_in_amd_ci() else 98.0 fwd_occupancy_max_new_tokens = 4096 if is_in_amd_ci() else 2048 fwd_occupancy_acc_length_threshold: float = 1.6