diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index 7a733605f6..61df34f93d 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -2477,8 +2477,11 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None if cpu_count == count: non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6) attrs[prefix + "_non_cpu_ms"] = non_cpu_ms - if total_ns < cpu_ns: - attrs[prefix + "_non_cpu_raw_ms"] = (total_ns - cpu_ns) / 1e6 + # Only a deficit beyond clock granularity is an accounting + # anomaly; a few hundred ns of skew on a 2 ms phase is noise. + deficit_ns = cpu_ns - total_ns + if deficit_ns > max(100_000, total_ns // 100): + attrs[prefix + "_non_cpu_raw_ms"] = -deficit_ns / 1e6 attrs[prefix + "_accounting_anomaly"] = True accounting_anomalies += 1 if count > 1: diff --git a/test/registered/unit/observability/test_prefill_batch_init_detail.py b/test/registered/unit/observability/test_prefill_batch_init_detail.py index 444052de8a..745a4585f3 100644 --- a/test/registered/unit/observability/test_prefill_batch_init_detail.py +++ b/test/registered/unit/observability/test_prefill_batch_init_detail.py @@ -403,6 +403,25 @@ class PrefillBatchInitDetailTest(unittest.TestCase): self.assertTrue(attrs["sglang.prepare_for_extend_accounting_anomaly"]) self.assertEqual(attrs["sglang.accounting_anomaly_count"], 1) + def test_clock_skew_within_tolerance_is_not_an_anomaly(self): + # 0.1 us of CPU > wall on a 2 ms phase is clock granularity, not an + # accounting bug: keep non_cpu clamped at 0 without flagging. + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add( + "prefill_batch_init", t0, t0 + 2_000_000, thread_cpu_ns=2_000_100 + ) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertEqual(attrs["sglang.prefill_batch_init_non_cpu_ms"], 0.0) + self.assertNotIn("sglang.prefill_batch_init_non_cpu_raw_ms", attrs) + self.assertNotIn("sglang.prefill_batch_init_accounting_anomaly", attrs) + self.assertNotIn("sglang.accounting_anomaly_count", attrs) + def test_child_span_carries_cpu_attributes(self): mod = _load() mod._ensure_loop_metrics = lambda: _fake_loop_metrics()