test: prefill_batch_init detail and eviction semantics

Covers the design's unit-level acceptance points:
* disabled/unsampled loops never read the thread clock;
* wall/CPU/non-CPU semantics, unmeasured != zero, partial measurement omits
  non-CPU, busy vs sleeping windows are distinguishable;
* CPU > wall flags an accounting anomaly with a raw negative delta;
* child spans carry the CPU split;
* repeated phases (prefix indices) stay attribute-only, inclusive phases are
  not summed, and eviction demand/shortfall/plan plus Mamba plan/zero results
  are recorded as separate counters.
This commit is contained in:
2026-11-12 10:30:47 +08:00
parent 5ad9ff6a5f
commit 17b922c84f
2 changed files with 579 additions and 0 deletions
@@ -0,0 +1,425 @@
"""Unit tests for the `prefill_batch_init` observability design (2026-11-11).
Covers the design's §9.1 acceptance points that are verifiable without a live
scheduler/GPU:
* disabled detail never calls ``time.thread_time_ns()``;
* unsampled loops never call it either, and record nothing;
* wall / thread-CPU / non-CPU semantics, and "not measured" != 0;
* busy vs sleeping windows are distinguishable by the clocks;
* repeated phases (prefix indices) stay attribute-only;
* inclusive parent/child phases are reported separately, not summed;
* attribute sums and setdefault keep low-cardinality eviction counters.
Scheduler-level cases (early return, mixed-chunk branch, decorator coverage)
are listed as integration checks in the design and exercised on a live engine.
"""
import importlib.util
import os
import pathlib
import sys
import time
import types
import unittest
from unittest import mock
def _module_path():
p = (
pathlib.Path(__file__).resolve().parents[4]
/ "python/sglang/srt/observability/batch_telemetry.py"
)
if p.exists():
return p
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
return pathlib.Path(spec.origin)
def _load(**env):
base = {
"SGLANG_BATCH_STAGE_TIMING": "1",
"SGLANG_TRACE_LOOP_SPAN": "1",
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
"SGLANG_TRACE_LOOP_DETAIL": "1",
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
"SGLANG_TRACE_FORWARD_SPAN": "0",
}
base.update(env)
with mock.patch.dict(os.environ, base):
spec = importlib.util.spec_from_file_location(
"batch_telemetry_batch_init_under_test", _module_path()
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
class _FakeCounter:
def __init__(self):
self.data = {}
def labels(self, **kw):
self._key = tuple(sorted(kw.items()))
return self
def inc(self, value=1.0):
self.data[self._key] = self.data.get(self._key, 0.0) + value
def _fake_loop_metrics():
return {
"no_batch_seconds": _FakeCounter(),
"no_batch_total": _FakeCounter(),
"lockstep_seconds": _FakeCounter(),
}
class _FakeSpan:
def __init__(self, name, **kwargs):
self.name = name
self.kwargs = kwargs
self.attributes = dict(kwargs.get("attributes") or {})
self.end_time = None
def set_attributes(self, attrs):
self.attributes.update(attrs)
def get_span_context(self):
return types.SimpleNamespace(is_valid=True, trace_id=1, span_id=2)
def end(self, end_time=None):
self.end_time = end_time
class _FakeTracer:
def __init__(self):
self.spans = []
def start_span(self, name, **kwargs):
span = _FakeSpan(name, **kwargs)
self.spans.append(span)
return span
def _fake_otel():
tracer = _FakeTracer()
trace = types.ModuleType("opentelemetry.trace")
trace.get_tracer = lambda _name: tracer
trace.set_span_in_context = lambda span: ("ctx", span)
trace.Link = lambda ctx: ("link", ctx)
package = types.ModuleType("opentelemetry")
package.trace = trace
patcher = mock.patch.dict(
sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace}
)
return tracer, patcher
def _state():
return {
"waiting": 0,
"result_queue": 0,
"result_limit": 2,
"inflight": 0,
"bootstrap": 0,
"deferred": 0,
"paused": False,
"waiting_for_result": False,
"skip_batch_creation": False,
"forward_active": False,
}
def _loop_attrs(tracer):
spans = [s for s in tracer.spans if s.name == "scheduler.loop"]
return spans[-1].attributes
def _no_thread_clock():
return mock.patch.object(
time, "thread_time_ns", side_effect=AssertionError("must not be called")
)
class PrefillBatchInitDetailTest(unittest.TestCase):
def test_disabled_never_reads_thread_cpu(self):
mod = _load(SGLANG_TRACE_LOOP_DETAIL="0")
with _no_thread_clock():
self.assertIsNone(mod.detail_begin("prefill_batch_init"))
mod.detail_add("prefill_batch_init", 0, 1_000)
mod.detail_attr_sum("evict_evicted_tokens", 3)
mod.detail_attr_setdefault("evict_free_tokens_before", 1)
with mod.detail_phase("prepare_for_extend"):
pass
def test_unsampled_loop_never_reads_thread_cpu(self):
mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0")
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
self.assertFalse(mod.loop_detail_enabled())
with _no_thread_clock():
self.assertIsNone(mod.detail_begin("prefill_batch_init"))
mod.detail_add("prefill_batch_init", 0, 1_000, thread_cpu_ns=None)
with mod.detail_phase("prepare_for_extend"):
pass
self.assertIsNone(lt.detail)
def test_wall_cpu_and_non_cpu_attributes(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add(
"prefill_batch_init",
t0,
t0 + 100_000_000,
thread_cpu_ns=60_000_000,
)
mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertAlmostEqual(attrs["sglang.prefill_batch_init_ms"], 100.0)
self.assertAlmostEqual(
attrs["sglang.prefill_batch_init_thread_cpu_ms"], 60.0
)
self.assertAlmostEqual(attrs["sglang.prefill_batch_init_non_cpu_ms"], 40.0)
def test_thread_cpu_omitted_when_not_measured(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add("prefill_stats", t0, t0 + 5_000_000)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertAlmostEqual(attrs["sglang.prefill_stats_ms"], 5.0)
self.assertNotIn("sglang.prefill_stats_thread_cpu_ms", attrs)
self.assertNotIn("sglang.prefill_stats_non_cpu_ms", attrs)
def test_detail_phase_records_cpu_when_enabled(self):
mod = _load()
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
with mod.detail_phase("prefill_stats"):
pass
rec = lt.detail["phases"]["prefill_stats"]
self.assertEqual(rec[1], 1)
self.assertEqual(rec[5], 1)
self.assertGreaterEqual(rec[4], 0)
def test_busy_and_sleep_are_distinguishable(self):
mod = _load()
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0, c0 = time.monotonic_ns(), time.thread_time_ns()
deadline = t0 + 30_000_000
while time.monotonic_ns() < deadline:
pass
mod.detail_add(
"busy_window", t0, time.monotonic_ns(), thread_cpu_ns=time.thread_time_ns() - c0
)
t1, c1 = time.monotonic_ns(), time.thread_time_ns()
time.sleep(0.03)
mod.detail_add(
"sleep_window",
t1,
time.monotonic_ns(),
thread_cpu_ns=time.thread_time_ns() - c1,
)
phases = lt.detail["phases"]
busy_wall, busy_cpu = phases["busy_window"][0], phases["busy_window"][4]
sleep_wall, sleep_cpu = phases["sleep_window"][0], phases["sleep_window"][4]
self.assertGreater(busy_cpu / busy_wall, 0.7)
self.assertLess(sleep_cpu / sleep_wall, 0.5)
def test_repeated_prefix_phase_is_attribute_only(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add(
"prefix_device_indices", t0, t0 + 2_000_000, thread_cpu_ns=1_500_000
)
mod.detail_add(
"prefix_device_indices", t0, t0 + 3_000_000, thread_cpu_ns=1_000_000
)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertAlmostEqual(attrs["sglang.prefix_device_indices_ms"], 5.0)
self.assertAlmostEqual(
attrs["sglang.prefix_device_indices_thread_cpu_ms"], 2.5
)
self.assertEqual(attrs["sglang.prefix_device_indices_count"], 2)
names = [s.name for s in tracer.spans]
self.assertNotIn("scheduler.loop.prefix_device_indices", names)
def test_inclusive_parent_and_children_are_not_summed(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add("prepare_for_extend", t0, t0 + 100_000_000)
mod.detail_add("alloc_for_extend", t0, t0 + 60_000_000)
mod.detail_add("evict_to_free_tokens", t0, t0 + 20_000_000)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertAlmostEqual(attrs["sglang.prepare_for_extend_ms"], 100.0)
self.assertAlmostEqual(attrs["sglang.alloc_for_extend_ms"], 60.0)
self.assertAlmostEqual(attrs["sglang.evict_to_free_tokens_ms"], 20.0)
# The event attrs are raw per-phase values, never a rolled-up sum.
self.assertNotAlmostEqual(
attrs["sglang.prepare_for_extend_ms"], 180.0
)
def test_eviction_sums_and_setdefault(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
mod.detail_attr_setdefault("evict_free_tokens_before", 100)
mod.detail_attr_setdefault("evict_free_tokens_before", 999)
mod.detail_attr("evict_free_tokens_after", 40)
mod.detail_attr_sum("evict_requested_tokens", 60)
mod.detail_attr_sum("evict_requested_tokens", 15)
mod.detail_attr_sum("evict_evicted_tokens", 50)
mod.detail_count("evict_calls", 2)
mod.detail_attr("evict_rank_consensus_checker", False)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertEqual(attrs["sglang.evict_free_tokens_before"], 100)
self.assertEqual(attrs["sglang.evict_free_tokens_after"], 40)
self.assertEqual(attrs["sglang.evict_requested_tokens"], 75)
self.assertEqual(attrs["sglang.evict_evicted_tokens"], 50)
self.assertEqual(attrs["sglang.evict_calls_count"], 2)
self.assertFalse(attrs["sglang.evict_rank_consensus_checker"])
def test_batch_shape_attributes_are_scalar(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
mod.detail_attr("prefill_batch_n_reqs", 7)
mod.detail_attr("prefill_batch_n_new", 4096)
mod.detail_attr("prefill_batch_n_cached", 120000)
mod.detail_attr("prefill_batch_chunked", True)
mod.detail_attr("prefill_batch_mixed_chunk", False)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertEqual(attrs["sglang.prefill_batch_n_reqs"], 7)
self.assertEqual(attrs["sglang.prefill_batch_n_new"], 4096)
self.assertEqual(attrs["sglang.prefill_batch_n_cached"], 120000)
self.assertTrue(attrs["sglang.prefill_batch_chunked"])
self.assertFalse(attrs["sglang.prefill_batch_mixed_chunk"])
for key in attrs:
self.assertNotIn("rid", key)
self.assertNotIn("hash", key)
def test_partial_cpu_measurement_omits_non_cpu(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add(
"prepare_for_extend", t0, t0 + 60_000_000, thread_cpu_ns=60_000_000
)
mod.detail_add("prepare_for_extend", t0, t0 + 40_000_000)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertAlmostEqual(attrs["sglang.prepare_for_extend_ms"], 100.0)
self.assertAlmostEqual(
attrs["sglang.prepare_for_extend_thread_cpu_ms"], 60.0
)
self.assertEqual(
attrs["sglang.prepare_for_extend_thread_cpu_measured_count"], 1
)
self.assertEqual(attrs["sglang.prepare_for_extend_count"], 2)
self.assertNotIn("sglang.prepare_for_extend_non_cpu_ms", attrs)
self.assertNotIn("sglang.prepare_for_extend_accounting_anomaly", attrs)
def test_full_cpu_measurement_emits_non_cpu(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add(
"prepare_for_extend", t0, t0 + 60_000_000, thread_cpu_ns=55_000_000
)
mod.detail_add(
"prepare_for_extend", t0, t0 + 40_000_000, thread_cpu_ns=35_000_000
)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertAlmostEqual(
attrs["sglang.prepare_for_extend_thread_cpu_ms"], 90.0
)
self.assertEqual(
attrs["sglang.prepare_for_extend_thread_cpu_measured_count"], 2
)
self.assertAlmostEqual(attrs["sglang.prepare_for_extend_non_cpu_ms"], 10.0)
def test_unmeasured_phase_has_no_cpu_fields(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add("prepare_for_extend", t0, t0 + 10_000_000)
mod.detail_add("prepare_for_extend", t0, t0 + 20_000_000)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertNotIn("sglang.prepare_for_extend_thread_cpu_ms", attrs)
self.assertNotIn("sglang.prepare_for_extend_non_cpu_ms", attrs)
self.assertNotIn(
"sglang.prepare_for_extend_thread_cpu_measured_count", attrs
)
def test_cpu_greater_than_wall_flags_anomaly(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add(
"prepare_for_extend", t0, t0 + 10_000_000, thread_cpu_ns=12_000_000
)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = _loop_attrs(tracer)
self.assertEqual(attrs["sglang.prepare_for_extend_non_cpu_ms"], 0.0)
self.assertAlmostEqual(
attrs["sglang.prepare_for_extend_non_cpu_raw_ms"], -2.0
)
self.assertTrue(attrs["sglang.prepare_for_extend_accounting_anomaly"])
self.assertEqual(attrs["sglang.accounting_anomaly_count"], 1)
def test_child_span_carries_cpu_attributes(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add(
"prefill_batch_init", t0, t0 + 50_000_000, thread_cpu_ns=20_000_000
)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
span = next(s for s in tracer.spans if s.name == "scheduler.loop.prefill_batch_init")
self.assertAlmostEqual(span.attributes["sglang.phase_ms"], 50.0)
self.assertAlmostEqual(span.attributes["sglang.phase_thread_cpu_ms"], 20.0)
self.assertAlmostEqual(span.attributes["sglang.phase_non_cpu_ms"], 30.0)
self.assertEqual(span.attributes["sglang.phase_cpu_measured_count"], 1)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,154 @@
"""Eviction accounting semantics for the prefill_batch_init instrumentation.
Verifies the review's P1-1: demand (what the batch wants allocatable),
shortfall (what eviction must actually recover) and the plan handed to
``tree_cache.evict_for_alloc`` are three distinct counters, and that a batch
that already fits records a zero shortfall instead of a negative one.
Requires an importable ``sglang`` + ``torch``; skipped on bare interpreters.
"""
import os
import types
import unittest
from unittest import mock
_DETAIL_ENV = {
"SGLANG_BATCH_STAGE_TIMING": "1",
"SGLANG_TRACE_LOOP_SPAN": "1",
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
"SGLANG_TRACE_LOOP_DETAIL": "1",
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
"SGLANG_TRACE_FORWARD_SPAN": "0",
}
_AVAILABLE = False
_common = None
_batch_telemetry = None
try: # pragma: no cover - environment dependent
import torch # noqa: F401
from sglang.srt.mem_cache import common as _common
from sglang.srt.observability import batch_telemetry as _batch_telemetry
_AVAILABLE = True
except Exception: # pragma: no cover
pass
class _FakeAllocator:
def __init__(self, available):
self.available = available
self.calls = []
def available_size(self):
return self.available
def evict_to_free_tokens(self, tree_cache, num_tokens):
self.calls.append(num_tokens)
self.available += num_tokens
return True
class _FakeTreeCache:
def __init__(self, allocator):
self.token_to_kv_pool_allocator = allocator
def supports_prefix_sharing(self):
return True
class _ZeroEvictTree:
def evict_for_alloc(self, params, **kwargs):
return types.SimpleNamespace(
num_tokens_evicted=0, swa_num_tokens_evicted=0, mamba_num_evicted=0
)
class _FakeTreeCacheForAlloc:
def __init__(self):
self.calls = []
def evict_for_alloc(self, params, **kwargs):
self.calls.append((params, kwargs))
return types.SimpleNamespace(
num_tokens_evicted=int(getattr(params, "num_tokens", 0) or 0),
swa_num_tokens_evicted=int(getattr(params, "swa_num_tokens", 0) or 0),
mamba_num_evicted=int(getattr(params, "mamba_num", 0) or 0),
)
@unittest.skipUnless(_AVAILABLE, "requires importable sglang + torch")
class EvictionSemanticsTest(unittest.TestCase):
def setUp(self):
self._env = mock.patch.dict(os.environ, _DETAIL_ENV)
self._env.start()
# Force the module-level gate on for this process.
_batch_telemetry._DETAIL_ENABLED = True
def tearDown(self):
self._env.stop()
def _begin(self):
return _batch_telemetry.begin_loop(
"prefill", pre_has_work=True, forward_active=False
)
def test_demand_shortfall_and_plan_are_distinct(self):
allocator = _FakeAllocator(available=31_744)
tree = _FakeTreeCache(allocator)
lt = self._begin()
_common.evict_from_tree_cache(tree, 32_768)
sums = lt.detail["sums"]
attrs = lt.detail["attrs"]
self.assertEqual(sums["evict_demand_tokens"], 32_768)
self.assertEqual(sums["evict_shortfall_tokens"], 1_024)
self.assertEqual(attrs["evict_free_tokens_before"], 31_744)
# The allocator itself computes the plan from the live shortfall.
self.assertEqual(allocator.calls, [32_768])
def test_no_shortfall_is_zero_not_negative(self):
allocator = _FakeAllocator(available=40_000)
tree = _FakeTreeCache(allocator)
lt = self._begin()
_common.evict_from_tree_cache(tree, 32_768)
sums = lt.detail["sums"]
self.assertEqual(sums["evict_demand_tokens"], 32_768)
self.assertEqual(sums["evict_shortfall_tokens"], 0)
def test_plan_and_realized_counts(self):
lt = self._begin()
_common.evict_for_alloc_observed(
_FakeTreeCacheForAlloc(),
types.SimpleNamespace(num_tokens=1_024, swa_num_tokens=256, mamba_num=0),
)
sums = lt.detail["sums"]
self.assertEqual(sums["evict_plan_requested_tokens"], 1_024)
self.assertEqual(sums["evict_plan_requested_swa_tokens"], 256)
self.assertEqual(sums["evict_evicted_tokens"], 1_024)
self.assertEqual(sums["evict_evicted_swa_tokens"], 256)
self.assertEqual(lt.detail["counts"]["evict_for_alloc_calls"], 1)
def test_mamba_plan_and_zero_realized_are_visible(self):
lt = self._begin()
_common.evict_for_alloc_observed(
_ZeroEvictTree(),
types.SimpleNamespace(num_tokens=0, swa_num_tokens=0, mamba_num=5),
)
sums = lt.detail["sums"]
self.assertEqual(sums["evict_plan_requested_mamba"], 5)
# Zero realized eviction must be recorded, not omitted.
self.assertIn("evict_evicted_mamba", sums)
self.assertEqual(sums["evict_evicted_mamba"], 0)
def test_shortfall_is_per_call_not_aggregated(self):
allocator = _FakeAllocator(available=10_000)
tree = _FakeTreeCache(allocator)
lt = self._begin()
_common.evict_from_tree_cache(tree, 12_000) # shortfall 2_000, frees 12k
_common.evict_from_tree_cache(tree, 12_000) # now fits: shortfall 0
sums = lt.detail["sums"]
self.assertEqual(sums["evict_demand_tokens"], 24_000)
self.assertEqual(sums["evict_shortfall_tokens"], 2_000)
if __name__ == "__main__":
unittest.main()