observability: ignore sub-tolerance CPU>wall skew in accounting anomaly

A 2 ms phase whose CPU clock reads 0.1 us over its wall clock is clock
granularity, not an accounting bug; the anomaly flag fired on nearly every
short phase in the live verification run.

Only flag (and record non_cpu_raw_ms) when the deficit exceeds both 100 us and
1% of the phase wall. Larger real anomalies keep the previous behaviour.
This commit is contained in:
2026-11-12 12:30:58 +08:00
parent 7f42a0806a
commit f4de9f2141
2 changed files with 24 additions and 2 deletions
@@ -2477,8 +2477,11 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None
if cpu_count == count:
non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6)
attrs[prefix + "_non_cpu_ms"] = non_cpu_ms
if total_ns < cpu_ns:
attrs[prefix + "_non_cpu_raw_ms"] = (total_ns - cpu_ns) / 1e6
# Only a deficit beyond clock granularity is an accounting
# anomaly; a few hundred ns of skew on a 2 ms phase is noise.
deficit_ns = cpu_ns - total_ns
if deficit_ns > max(100_000, total_ns // 100):
attrs[prefix + "_non_cpu_raw_ms"] = -deficit_ns / 1e6
attrs[prefix + "_accounting_anomaly"] = True
accounting_anomalies += 1
if count > 1:
@@ -403,6 +403,25 @@ class PrefillBatchInitDetailTest(unittest.TestCase):
self.assertTrue(attrs["sglang.prepare_for_extend_accounting_anomaly"])
self.assertEqual(attrs["sglang.accounting_anomaly_count"], 1)
def test_clock_skew_within_tolerance_is_not_an_anomaly(self):
# 0.1 us of CPU > wall on a 2 ms phase is clock granularity, not an
# accounting bug: keep non_cpu clamped at 0 without flagging.
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add(
"prefill_batch_init", t0, t0 + 2_000_000, thread_cpu_ns=2_000_100
)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertEqual(attrs["sglang.prefill_batch_init_non_cpu_ms"], 0.0)
self.assertNotIn("sglang.prefill_batch_init_non_cpu_raw_ms", attrs)
self.assertNotIn("sglang.prefill_batch_init_accounting_anomaly", attrs)
self.assertNotIn("sglang.accounting_anomaly_count", attrs)
def test_child_span_carries_cpu_attributes(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()