diff --git a/test/registered/unit/observability/test_prefill_batch_init_detail.py b/test/registered/unit/observability/test_prefill_batch_init_detail.py new file mode 100644 index 0000000000..444052de8a --- /dev/null +++ b/test/registered/unit/observability/test_prefill_batch_init_detail.py @@ -0,0 +1,425 @@ +"""Unit tests for the `prefill_batch_init` observability design (2026-11-11). + +Covers the design's ยง9.1 acceptance points that are verifiable without a live +scheduler/GPU: + + * disabled detail never calls ``time.thread_time_ns()``; + * unsampled loops never call it either, and record nothing; + * wall / thread-CPU / non-CPU semantics, and "not measured" != 0; + * busy vs sleeping windows are distinguishable by the clocks; + * repeated phases (prefix indices) stay attribute-only; + * inclusive parent/child phases are reported separately, not summed; + * attribute sums and setdefault keep low-cardinality eviction counters. + +Scheduler-level cases (early return, mixed-chunk branch, decorator coverage) +are listed as integration checks in the design and exercised on a live engine. +""" +import importlib.util +import os +import pathlib +import sys +import time +import types +import unittest +from unittest import mock + + +def _module_path(): + p = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/observability/batch_telemetry.py" + ) + if p.exists(): + return p + spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry") + return pathlib.Path(spec.origin) + + +def _load(**env): + base = { + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_TRACE_LOOP_SPAN": "1", + "SGLANG_TRACE_LOOP_SAMPLE": "1.0", + "SGLANG_TRACE_LOOP_DETAIL": "1", + "SGLANG_TRACE_MOONCAKE_DETAIL": "0", + "SGLANG_TRACE_FORWARD_SPAN": "0", + } + base.update(env) + with mock.patch.dict(os.environ, base): + spec = importlib.util.spec_from_file_location( + "batch_telemetry_batch_init_under_test", _module_path() + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class _FakeCounter: + def __init__(self): + self.data = {} + + def labels(self, **kw): + self._key = tuple(sorted(kw.items())) + return self + + def inc(self, value=1.0): + self.data[self._key] = self.data.get(self._key, 0.0) + value + + +def _fake_loop_metrics(): + return { + "no_batch_seconds": _FakeCounter(), + "no_batch_total": _FakeCounter(), + "lockstep_seconds": _FakeCounter(), + } + + +class _FakeSpan: + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs + self.attributes = dict(kwargs.get("attributes") or {}) + self.end_time = None + + def set_attributes(self, attrs): + self.attributes.update(attrs) + + def get_span_context(self): + return types.SimpleNamespace(is_valid=True, trace_id=1, span_id=2) + + def end(self, end_time=None): + self.end_time = end_time + + +class _FakeTracer: + def __init__(self): + self.spans = [] + + def start_span(self, name, **kwargs): + span = _FakeSpan(name, **kwargs) + self.spans.append(span) + return span + + +def _fake_otel(): + tracer = _FakeTracer() + trace = types.ModuleType("opentelemetry.trace") + trace.get_tracer = lambda _name: tracer + trace.set_span_in_context = lambda span: ("ctx", span) + trace.Link = lambda ctx: ("link", ctx) + package = types.ModuleType("opentelemetry") + package.trace = trace + patcher = mock.patch.dict( + sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace} + ) + return tracer, patcher + + +def _state(): + return { + "waiting": 0, + "result_queue": 0, + "result_limit": 2, + "inflight": 0, + "bootstrap": 0, + "deferred": 0, + "paused": False, + "waiting_for_result": False, + "skip_batch_creation": False, + "forward_active": False, + } + + +def _loop_attrs(tracer): + spans = [s for s in tracer.spans if s.name == "scheduler.loop"] + return spans[-1].attributes + + +def _no_thread_clock(): + return mock.patch.object( + time, "thread_time_ns", side_effect=AssertionError("must not be called") + ) + + +class PrefillBatchInitDetailTest(unittest.TestCase): + def test_disabled_never_reads_thread_cpu(self): + mod = _load(SGLANG_TRACE_LOOP_DETAIL="0") + with _no_thread_clock(): + self.assertIsNone(mod.detail_begin("prefill_batch_init")) + mod.detail_add("prefill_batch_init", 0, 1_000) + mod.detail_attr_sum("evict_evicted_tokens", 3) + mod.detail_attr_setdefault("evict_free_tokens_before", 1) + with mod.detail_phase("prepare_for_extend"): + pass + + def test_unsampled_loop_never_reads_thread_cpu(self): + mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0") + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + self.assertFalse(mod.loop_detail_enabled()) + with _no_thread_clock(): + self.assertIsNone(mod.detail_begin("prefill_batch_init")) + mod.detail_add("prefill_batch_init", 0, 1_000, thread_cpu_ns=None) + with mod.detail_phase("prepare_for_extend"): + pass + self.assertIsNone(lt.detail) + + def test_wall_cpu_and_non_cpu_attributes(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add( + "prefill_batch_init", + t0, + t0 + 100_000_000, + thread_cpu_ns=60_000_000, + ) + mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertAlmostEqual(attrs["sglang.prefill_batch_init_ms"], 100.0) + self.assertAlmostEqual( + attrs["sglang.prefill_batch_init_thread_cpu_ms"], 60.0 + ) + self.assertAlmostEqual(attrs["sglang.prefill_batch_init_non_cpu_ms"], 40.0) + + def test_thread_cpu_omitted_when_not_measured(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add("prefill_stats", t0, t0 + 5_000_000) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertAlmostEqual(attrs["sglang.prefill_stats_ms"], 5.0) + self.assertNotIn("sglang.prefill_stats_thread_cpu_ms", attrs) + self.assertNotIn("sglang.prefill_stats_non_cpu_ms", attrs) + + def test_detail_phase_records_cpu_when_enabled(self): + mod = _load() + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + with mod.detail_phase("prefill_stats"): + pass + rec = lt.detail["phases"]["prefill_stats"] + self.assertEqual(rec[1], 1) + self.assertEqual(rec[5], 1) + self.assertGreaterEqual(rec[4], 0) + + def test_busy_and_sleep_are_distinguishable(self): + mod = _load() + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0, c0 = time.monotonic_ns(), time.thread_time_ns() + deadline = t0 + 30_000_000 + while time.monotonic_ns() < deadline: + pass + mod.detail_add( + "busy_window", t0, time.monotonic_ns(), thread_cpu_ns=time.thread_time_ns() - c0 + ) + t1, c1 = time.monotonic_ns(), time.thread_time_ns() + time.sleep(0.03) + mod.detail_add( + "sleep_window", + t1, + time.monotonic_ns(), + thread_cpu_ns=time.thread_time_ns() - c1, + ) + phases = lt.detail["phases"] + busy_wall, busy_cpu = phases["busy_window"][0], phases["busy_window"][4] + sleep_wall, sleep_cpu = phases["sleep_window"][0], phases["sleep_window"][4] + self.assertGreater(busy_cpu / busy_wall, 0.7) + self.assertLess(sleep_cpu / sleep_wall, 0.5) + + def test_repeated_prefix_phase_is_attribute_only(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add( + "prefix_device_indices", t0, t0 + 2_000_000, thread_cpu_ns=1_500_000 + ) + mod.detail_add( + "prefix_device_indices", t0, t0 + 3_000_000, thread_cpu_ns=1_000_000 + ) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertAlmostEqual(attrs["sglang.prefix_device_indices_ms"], 5.0) + self.assertAlmostEqual( + attrs["sglang.prefix_device_indices_thread_cpu_ms"], 2.5 + ) + self.assertEqual(attrs["sglang.prefix_device_indices_count"], 2) + names = [s.name for s in tracer.spans] + self.assertNotIn("scheduler.loop.prefix_device_indices", names) + + def test_inclusive_parent_and_children_are_not_summed(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add("prepare_for_extend", t0, t0 + 100_000_000) + mod.detail_add("alloc_for_extend", t0, t0 + 60_000_000) + mod.detail_add("evict_to_free_tokens", t0, t0 + 20_000_000) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertAlmostEqual(attrs["sglang.prepare_for_extend_ms"], 100.0) + self.assertAlmostEqual(attrs["sglang.alloc_for_extend_ms"], 60.0) + self.assertAlmostEqual(attrs["sglang.evict_to_free_tokens_ms"], 20.0) + # The event attrs are raw per-phase values, never a rolled-up sum. + self.assertNotAlmostEqual( + attrs["sglang.prepare_for_extend_ms"], 180.0 + ) + + def test_eviction_sums_and_setdefault(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.detail_attr_setdefault("evict_free_tokens_before", 100) + mod.detail_attr_setdefault("evict_free_tokens_before", 999) + mod.detail_attr("evict_free_tokens_after", 40) + mod.detail_attr_sum("evict_requested_tokens", 60) + mod.detail_attr_sum("evict_requested_tokens", 15) + mod.detail_attr_sum("evict_evicted_tokens", 50) + mod.detail_count("evict_calls", 2) + mod.detail_attr("evict_rank_consensus_checker", False) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertEqual(attrs["sglang.evict_free_tokens_before"], 100) + self.assertEqual(attrs["sglang.evict_free_tokens_after"], 40) + self.assertEqual(attrs["sglang.evict_requested_tokens"], 75) + self.assertEqual(attrs["sglang.evict_evicted_tokens"], 50) + self.assertEqual(attrs["sglang.evict_calls_count"], 2) + self.assertFalse(attrs["sglang.evict_rank_consensus_checker"]) + + def test_batch_shape_attributes_are_scalar(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.detail_attr("prefill_batch_n_reqs", 7) + mod.detail_attr("prefill_batch_n_new", 4096) + mod.detail_attr("prefill_batch_n_cached", 120000) + mod.detail_attr("prefill_batch_chunked", True) + mod.detail_attr("prefill_batch_mixed_chunk", False) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertEqual(attrs["sglang.prefill_batch_n_reqs"], 7) + self.assertEqual(attrs["sglang.prefill_batch_n_new"], 4096) + self.assertEqual(attrs["sglang.prefill_batch_n_cached"], 120000) + self.assertTrue(attrs["sglang.prefill_batch_chunked"]) + self.assertFalse(attrs["sglang.prefill_batch_mixed_chunk"]) + for key in attrs: + self.assertNotIn("rid", key) + self.assertNotIn("hash", key) + + def test_partial_cpu_measurement_omits_non_cpu(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add( + "prepare_for_extend", t0, t0 + 60_000_000, thread_cpu_ns=60_000_000 + ) + mod.detail_add("prepare_for_extend", t0, t0 + 40_000_000) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertAlmostEqual(attrs["sglang.prepare_for_extend_ms"], 100.0) + self.assertAlmostEqual( + attrs["sglang.prepare_for_extend_thread_cpu_ms"], 60.0 + ) + self.assertEqual( + attrs["sglang.prepare_for_extend_thread_cpu_measured_count"], 1 + ) + self.assertEqual(attrs["sglang.prepare_for_extend_count"], 2) + self.assertNotIn("sglang.prepare_for_extend_non_cpu_ms", attrs) + self.assertNotIn("sglang.prepare_for_extend_accounting_anomaly", attrs) + + def test_full_cpu_measurement_emits_non_cpu(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add( + "prepare_for_extend", t0, t0 + 60_000_000, thread_cpu_ns=55_000_000 + ) + mod.detail_add( + "prepare_for_extend", t0, t0 + 40_000_000, thread_cpu_ns=35_000_000 + ) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertAlmostEqual( + attrs["sglang.prepare_for_extend_thread_cpu_ms"], 90.0 + ) + self.assertEqual( + attrs["sglang.prepare_for_extend_thread_cpu_measured_count"], 2 + ) + self.assertAlmostEqual(attrs["sglang.prepare_for_extend_non_cpu_ms"], 10.0) + + def test_unmeasured_phase_has_no_cpu_fields(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add("prepare_for_extend", t0, t0 + 10_000_000) + mod.detail_add("prepare_for_extend", t0, t0 + 20_000_000) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertNotIn("sglang.prepare_for_extend_thread_cpu_ms", attrs) + self.assertNotIn("sglang.prepare_for_extend_non_cpu_ms", attrs) + self.assertNotIn( + "sglang.prepare_for_extend_thread_cpu_measured_count", attrs + ) + + def test_cpu_greater_than_wall_flags_anomaly(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add( + "prepare_for_extend", t0, t0 + 10_000_000, thread_cpu_ns=12_000_000 + ) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _loop_attrs(tracer) + self.assertEqual(attrs["sglang.prepare_for_extend_non_cpu_ms"], 0.0) + self.assertAlmostEqual( + attrs["sglang.prepare_for_extend_non_cpu_raw_ms"], -2.0 + ) + self.assertTrue(attrs["sglang.prepare_for_extend_accounting_anomaly"]) + self.assertEqual(attrs["sglang.accounting_anomaly_count"], 1) + + def test_child_span_carries_cpu_attributes(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add( + "prefill_batch_init", t0, t0 + 50_000_000, thread_cpu_ns=20_000_000 + ) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + span = next(s for s in tracer.spans if s.name == "scheduler.loop.prefill_batch_init") + self.assertAlmostEqual(span.attributes["sglang.phase_ms"], 50.0) + self.assertAlmostEqual(span.attributes["sglang.phase_thread_cpu_ms"], 20.0) + self.assertAlmostEqual(span.attributes["sglang.phase_non_cpu_ms"], 30.0) + self.assertEqual(span.attributes["sglang.phase_cpu_measured_count"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/observability/test_prefill_batch_init_eviction.py b/test/registered/unit/observability/test_prefill_batch_init_eviction.py new file mode 100644 index 0000000000..e986a2c12a --- /dev/null +++ b/test/registered/unit/observability/test_prefill_batch_init_eviction.py @@ -0,0 +1,154 @@ +"""Eviction accounting semantics for the prefill_batch_init instrumentation. + +Verifies the review's P1-1: demand (what the batch wants allocatable), +shortfall (what eviction must actually recover) and the plan handed to +``tree_cache.evict_for_alloc`` are three distinct counters, and that a batch +that already fits records a zero shortfall instead of a negative one. + +Requires an importable ``sglang`` + ``torch``; skipped on bare interpreters. +""" +import os +import types +import unittest +from unittest import mock + +_DETAIL_ENV = { + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_TRACE_LOOP_SPAN": "1", + "SGLANG_TRACE_LOOP_SAMPLE": "1.0", + "SGLANG_TRACE_LOOP_DETAIL": "1", + "SGLANG_TRACE_MOONCAKE_DETAIL": "0", + "SGLANG_TRACE_FORWARD_SPAN": "0", +} + +_AVAILABLE = False +_common = None +_batch_telemetry = None +try: # pragma: no cover - environment dependent + import torch # noqa: F401 + from sglang.srt.mem_cache import common as _common + from sglang.srt.observability import batch_telemetry as _batch_telemetry + + _AVAILABLE = True +except Exception: # pragma: no cover + pass + + +class _FakeAllocator: + def __init__(self, available): + self.available = available + self.calls = [] + + def available_size(self): + return self.available + + def evict_to_free_tokens(self, tree_cache, num_tokens): + self.calls.append(num_tokens) + self.available += num_tokens + return True + + +class _FakeTreeCache: + def __init__(self, allocator): + self.token_to_kv_pool_allocator = allocator + + def supports_prefix_sharing(self): + return True + + +class _ZeroEvictTree: + def evict_for_alloc(self, params, **kwargs): + return types.SimpleNamespace( + num_tokens_evicted=0, swa_num_tokens_evicted=0, mamba_num_evicted=0 + ) + + +class _FakeTreeCacheForAlloc: + def __init__(self): + self.calls = [] + + def evict_for_alloc(self, params, **kwargs): + self.calls.append((params, kwargs)) + return types.SimpleNamespace( + num_tokens_evicted=int(getattr(params, "num_tokens", 0) or 0), + swa_num_tokens_evicted=int(getattr(params, "swa_num_tokens", 0) or 0), + mamba_num_evicted=int(getattr(params, "mamba_num", 0) or 0), + ) + + +@unittest.skipUnless(_AVAILABLE, "requires importable sglang + torch") +class EvictionSemanticsTest(unittest.TestCase): + def setUp(self): + self._env = mock.patch.dict(os.environ, _DETAIL_ENV) + self._env.start() + # Force the module-level gate on for this process. + _batch_telemetry._DETAIL_ENABLED = True + + def tearDown(self): + self._env.stop() + + def _begin(self): + return _batch_telemetry.begin_loop( + "prefill", pre_has_work=True, forward_active=False + ) + + def test_demand_shortfall_and_plan_are_distinct(self): + allocator = _FakeAllocator(available=31_744) + tree = _FakeTreeCache(allocator) + lt = self._begin() + _common.evict_from_tree_cache(tree, 32_768) + sums = lt.detail["sums"] + attrs = lt.detail["attrs"] + self.assertEqual(sums["evict_demand_tokens"], 32_768) + self.assertEqual(sums["evict_shortfall_tokens"], 1_024) + self.assertEqual(attrs["evict_free_tokens_before"], 31_744) + # The allocator itself computes the plan from the live shortfall. + self.assertEqual(allocator.calls, [32_768]) + + def test_no_shortfall_is_zero_not_negative(self): + allocator = _FakeAllocator(available=40_000) + tree = _FakeTreeCache(allocator) + lt = self._begin() + _common.evict_from_tree_cache(tree, 32_768) + sums = lt.detail["sums"] + self.assertEqual(sums["evict_demand_tokens"], 32_768) + self.assertEqual(sums["evict_shortfall_tokens"], 0) + + def test_plan_and_realized_counts(self): + lt = self._begin() + _common.evict_for_alloc_observed( + _FakeTreeCacheForAlloc(), + types.SimpleNamespace(num_tokens=1_024, swa_num_tokens=256, mamba_num=0), + ) + sums = lt.detail["sums"] + self.assertEqual(sums["evict_plan_requested_tokens"], 1_024) + self.assertEqual(sums["evict_plan_requested_swa_tokens"], 256) + self.assertEqual(sums["evict_evicted_tokens"], 1_024) + self.assertEqual(sums["evict_evicted_swa_tokens"], 256) + self.assertEqual(lt.detail["counts"]["evict_for_alloc_calls"], 1) + + def test_mamba_plan_and_zero_realized_are_visible(self): + lt = self._begin() + _common.evict_for_alloc_observed( + _ZeroEvictTree(), + types.SimpleNamespace(num_tokens=0, swa_num_tokens=0, mamba_num=5), + ) + sums = lt.detail["sums"] + self.assertEqual(sums["evict_plan_requested_mamba"], 5) + # Zero realized eviction must be recorded, not omitted. + self.assertIn("evict_evicted_mamba", sums) + self.assertEqual(sums["evict_evicted_mamba"], 0) + + def test_shortfall_is_per_call_not_aggregated(self): + allocator = _FakeAllocator(available=10_000) + tree = _FakeTreeCache(allocator) + lt = self._begin() + _common.evict_from_tree_cache(tree, 12_000) # shortfall 2_000, frees 12k + _common.evict_from_tree_cache(tree, 12_000) # now fits: shortfall 0 + sums = lt.detail["sums"] + self.assertEqual(sums["evict_demand_tokens"], 24_000) + self.assertEqual(sums["evict_shortfall_tokens"], 2_000) + + +if __name__ == "__main__": + unittest.main()