test: prefill_batch_init detail and eviction semantics
Covers the design's unit-level acceptance points: * disabled/unsampled loops never read the thread clock; * wall/CPU/non-CPU semantics, unmeasured != zero, partial measurement omits non-CPU, busy vs sleeping windows are distinguishable; * CPU > wall flags an accounting anomaly with a raw negative delta; * child spans carry the CPU split; * repeated phases (prefix indices) stay attribute-only, inclusive phases are not summed, and eviction demand/shortfall/plan plus Mamba plan/zero results are recorded as separate counters.
This commit is contained in:
@@ -0,0 +1,425 @@
|
||||
"""Unit tests for the `prefill_batch_init` observability design (2026-11-11).
|
||||
|
||||
Covers the design's §9.1 acceptance points that are verifiable without a live
|
||||
scheduler/GPU:
|
||||
|
||||
* disabled detail never calls ``time.thread_time_ns()``;
|
||||
* unsampled loops never call it either, and record nothing;
|
||||
* wall / thread-CPU / non-CPU semantics, and "not measured" != 0;
|
||||
* busy vs sleeping windows are distinguishable by the clocks;
|
||||
* repeated phases (prefix indices) stay attribute-only;
|
||||
* inclusive parent/child phases are reported separately, not summed;
|
||||
* attribute sums and setdefault keep low-cardinality eviction counters.
|
||||
|
||||
Scheduler-level cases (early return, mixed-chunk branch, decorator coverage)
|
||||
are listed as integration checks in the design and exercised on a live engine.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def _module_path():
|
||||
p = (
|
||||
pathlib.Path(__file__).resolve().parents[4]
|
||||
/ "python/sglang/srt/observability/batch_telemetry.py"
|
||||
)
|
||||
if p.exists():
|
||||
return p
|
||||
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
|
||||
return pathlib.Path(spec.origin)
|
||||
|
||||
|
||||
def _load(**env):
|
||||
base = {
|
||||
"SGLANG_BATCH_STAGE_TIMING": "1",
|
||||
"SGLANG_TRACE_LOOP_SPAN": "1",
|
||||
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
|
||||
"SGLANG_TRACE_LOOP_DETAIL": "1",
|
||||
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
|
||||
"SGLANG_TRACE_FORWARD_SPAN": "0",
|
||||
}
|
||||
base.update(env)
|
||||
with mock.patch.dict(os.environ, base):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"batch_telemetry_batch_init_under_test", _module_path()
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
class _FakeCounter:
|
||||
def __init__(self):
|
||||
self.data = {}
|
||||
|
||||
def labels(self, **kw):
|
||||
self._key = tuple(sorted(kw.items()))
|
||||
return self
|
||||
|
||||
def inc(self, value=1.0):
|
||||
self.data[self._key] = self.data.get(self._key, 0.0) + value
|
||||
|
||||
|
||||
def _fake_loop_metrics():
|
||||
return {
|
||||
"no_batch_seconds": _FakeCounter(),
|
||||
"no_batch_total": _FakeCounter(),
|
||||
"lockstep_seconds": _FakeCounter(),
|
||||
}
|
||||
|
||||
|
||||
class _FakeSpan:
|
||||
def __init__(self, name, **kwargs):
|
||||
self.name = name
|
||||
self.kwargs = kwargs
|
||||
self.attributes = dict(kwargs.get("attributes") or {})
|
||||
self.end_time = None
|
||||
|
||||
def set_attributes(self, attrs):
|
||||
self.attributes.update(attrs)
|
||||
|
||||
def get_span_context(self):
|
||||
return types.SimpleNamespace(is_valid=True, trace_id=1, span_id=2)
|
||||
|
||||
def end(self, end_time=None):
|
||||
self.end_time = end_time
|
||||
|
||||
|
||||
class _FakeTracer:
|
||||
def __init__(self):
|
||||
self.spans = []
|
||||
|
||||
def start_span(self, name, **kwargs):
|
||||
span = _FakeSpan(name, **kwargs)
|
||||
self.spans.append(span)
|
||||
return span
|
||||
|
||||
|
||||
def _fake_otel():
|
||||
tracer = _FakeTracer()
|
||||
trace = types.ModuleType("opentelemetry.trace")
|
||||
trace.get_tracer = lambda _name: tracer
|
||||
trace.set_span_in_context = lambda span: ("ctx", span)
|
||||
trace.Link = lambda ctx: ("link", ctx)
|
||||
package = types.ModuleType("opentelemetry")
|
||||
package.trace = trace
|
||||
patcher = mock.patch.dict(
|
||||
sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace}
|
||||
)
|
||||
return tracer, patcher
|
||||
|
||||
|
||||
def _state():
|
||||
return {
|
||||
"waiting": 0,
|
||||
"result_queue": 0,
|
||||
"result_limit": 2,
|
||||
"inflight": 0,
|
||||
"bootstrap": 0,
|
||||
"deferred": 0,
|
||||
"paused": False,
|
||||
"waiting_for_result": False,
|
||||
"skip_batch_creation": False,
|
||||
"forward_active": False,
|
||||
}
|
||||
|
||||
|
||||
def _loop_attrs(tracer):
|
||||
spans = [s for s in tracer.spans if s.name == "scheduler.loop"]
|
||||
return spans[-1].attributes
|
||||
|
||||
|
||||
def _no_thread_clock():
|
||||
return mock.patch.object(
|
||||
time, "thread_time_ns", side_effect=AssertionError("must not be called")
|
||||
)
|
||||
|
||||
|
||||
class PrefillBatchInitDetailTest(unittest.TestCase):
|
||||
def test_disabled_never_reads_thread_cpu(self):
|
||||
mod = _load(SGLANG_TRACE_LOOP_DETAIL="0")
|
||||
with _no_thread_clock():
|
||||
self.assertIsNone(mod.detail_begin("prefill_batch_init"))
|
||||
mod.detail_add("prefill_batch_init", 0, 1_000)
|
||||
mod.detail_attr_sum("evict_evicted_tokens", 3)
|
||||
mod.detail_attr_setdefault("evict_free_tokens_before", 1)
|
||||
with mod.detail_phase("prepare_for_extend"):
|
||||
pass
|
||||
|
||||
def test_unsampled_loop_never_reads_thread_cpu(self):
|
||||
mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0")
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
self.assertFalse(mod.loop_detail_enabled())
|
||||
with _no_thread_clock():
|
||||
self.assertIsNone(mod.detail_begin("prefill_batch_init"))
|
||||
mod.detail_add("prefill_batch_init", 0, 1_000, thread_cpu_ns=None)
|
||||
with mod.detail_phase("prepare_for_extend"):
|
||||
pass
|
||||
self.assertIsNone(lt.detail)
|
||||
|
||||
def test_wall_cpu_and_non_cpu_attributes(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add(
|
||||
"prefill_batch_init",
|
||||
t0,
|
||||
t0 + 100_000_000,
|
||||
thread_cpu_ns=60_000_000,
|
||||
)
|
||||
mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertAlmostEqual(attrs["sglang.prefill_batch_init_ms"], 100.0)
|
||||
self.assertAlmostEqual(
|
||||
attrs["sglang.prefill_batch_init_thread_cpu_ms"], 60.0
|
||||
)
|
||||
self.assertAlmostEqual(attrs["sglang.prefill_batch_init_non_cpu_ms"], 40.0)
|
||||
|
||||
def test_thread_cpu_omitted_when_not_measured(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("prefill_stats", t0, t0 + 5_000_000)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertAlmostEqual(attrs["sglang.prefill_stats_ms"], 5.0)
|
||||
self.assertNotIn("sglang.prefill_stats_thread_cpu_ms", attrs)
|
||||
self.assertNotIn("sglang.prefill_stats_non_cpu_ms", attrs)
|
||||
|
||||
def test_detail_phase_records_cpu_when_enabled(self):
|
||||
mod = _load()
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
with mod.detail_phase("prefill_stats"):
|
||||
pass
|
||||
rec = lt.detail["phases"]["prefill_stats"]
|
||||
self.assertEqual(rec[1], 1)
|
||||
self.assertEqual(rec[5], 1)
|
||||
self.assertGreaterEqual(rec[4], 0)
|
||||
|
||||
def test_busy_and_sleep_are_distinguishable(self):
|
||||
mod = _load()
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0, c0 = time.monotonic_ns(), time.thread_time_ns()
|
||||
deadline = t0 + 30_000_000
|
||||
while time.monotonic_ns() < deadline:
|
||||
pass
|
||||
mod.detail_add(
|
||||
"busy_window", t0, time.monotonic_ns(), thread_cpu_ns=time.thread_time_ns() - c0
|
||||
)
|
||||
t1, c1 = time.monotonic_ns(), time.thread_time_ns()
|
||||
time.sleep(0.03)
|
||||
mod.detail_add(
|
||||
"sleep_window",
|
||||
t1,
|
||||
time.monotonic_ns(),
|
||||
thread_cpu_ns=time.thread_time_ns() - c1,
|
||||
)
|
||||
phases = lt.detail["phases"]
|
||||
busy_wall, busy_cpu = phases["busy_window"][0], phases["busy_window"][4]
|
||||
sleep_wall, sleep_cpu = phases["sleep_window"][0], phases["sleep_window"][4]
|
||||
self.assertGreater(busy_cpu / busy_wall, 0.7)
|
||||
self.assertLess(sleep_cpu / sleep_wall, 0.5)
|
||||
|
||||
def test_repeated_prefix_phase_is_attribute_only(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add(
|
||||
"prefix_device_indices", t0, t0 + 2_000_000, thread_cpu_ns=1_500_000
|
||||
)
|
||||
mod.detail_add(
|
||||
"prefix_device_indices", t0, t0 + 3_000_000, thread_cpu_ns=1_000_000
|
||||
)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertAlmostEqual(attrs["sglang.prefix_device_indices_ms"], 5.0)
|
||||
self.assertAlmostEqual(
|
||||
attrs["sglang.prefix_device_indices_thread_cpu_ms"], 2.5
|
||||
)
|
||||
self.assertEqual(attrs["sglang.prefix_device_indices_count"], 2)
|
||||
names = [s.name for s in tracer.spans]
|
||||
self.assertNotIn("scheduler.loop.prefix_device_indices", names)
|
||||
|
||||
def test_inclusive_parent_and_children_are_not_summed(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("prepare_for_extend", t0, t0 + 100_000_000)
|
||||
mod.detail_add("alloc_for_extend", t0, t0 + 60_000_000)
|
||||
mod.detail_add("evict_to_free_tokens", t0, t0 + 20_000_000)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertAlmostEqual(attrs["sglang.prepare_for_extend_ms"], 100.0)
|
||||
self.assertAlmostEqual(attrs["sglang.alloc_for_extend_ms"], 60.0)
|
||||
self.assertAlmostEqual(attrs["sglang.evict_to_free_tokens_ms"], 20.0)
|
||||
# The event attrs are raw per-phase values, never a rolled-up sum.
|
||||
self.assertNotAlmostEqual(
|
||||
attrs["sglang.prepare_for_extend_ms"], 180.0
|
||||
)
|
||||
|
||||
def test_eviction_sums_and_setdefault(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
mod.detail_attr_setdefault("evict_free_tokens_before", 100)
|
||||
mod.detail_attr_setdefault("evict_free_tokens_before", 999)
|
||||
mod.detail_attr("evict_free_tokens_after", 40)
|
||||
mod.detail_attr_sum("evict_requested_tokens", 60)
|
||||
mod.detail_attr_sum("evict_requested_tokens", 15)
|
||||
mod.detail_attr_sum("evict_evicted_tokens", 50)
|
||||
mod.detail_count("evict_calls", 2)
|
||||
mod.detail_attr("evict_rank_consensus_checker", False)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertEqual(attrs["sglang.evict_free_tokens_before"], 100)
|
||||
self.assertEqual(attrs["sglang.evict_free_tokens_after"], 40)
|
||||
self.assertEqual(attrs["sglang.evict_requested_tokens"], 75)
|
||||
self.assertEqual(attrs["sglang.evict_evicted_tokens"], 50)
|
||||
self.assertEqual(attrs["sglang.evict_calls_count"], 2)
|
||||
self.assertFalse(attrs["sglang.evict_rank_consensus_checker"])
|
||||
|
||||
def test_batch_shape_attributes_are_scalar(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
mod.detail_attr("prefill_batch_n_reqs", 7)
|
||||
mod.detail_attr("prefill_batch_n_new", 4096)
|
||||
mod.detail_attr("prefill_batch_n_cached", 120000)
|
||||
mod.detail_attr("prefill_batch_chunked", True)
|
||||
mod.detail_attr("prefill_batch_mixed_chunk", False)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertEqual(attrs["sglang.prefill_batch_n_reqs"], 7)
|
||||
self.assertEqual(attrs["sglang.prefill_batch_n_new"], 4096)
|
||||
self.assertEqual(attrs["sglang.prefill_batch_n_cached"], 120000)
|
||||
self.assertTrue(attrs["sglang.prefill_batch_chunked"])
|
||||
self.assertFalse(attrs["sglang.prefill_batch_mixed_chunk"])
|
||||
for key in attrs:
|
||||
self.assertNotIn("rid", key)
|
||||
self.assertNotIn("hash", key)
|
||||
|
||||
def test_partial_cpu_measurement_omits_non_cpu(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add(
|
||||
"prepare_for_extend", t0, t0 + 60_000_000, thread_cpu_ns=60_000_000
|
||||
)
|
||||
mod.detail_add("prepare_for_extend", t0, t0 + 40_000_000)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertAlmostEqual(attrs["sglang.prepare_for_extend_ms"], 100.0)
|
||||
self.assertAlmostEqual(
|
||||
attrs["sglang.prepare_for_extend_thread_cpu_ms"], 60.0
|
||||
)
|
||||
self.assertEqual(
|
||||
attrs["sglang.prepare_for_extend_thread_cpu_measured_count"], 1
|
||||
)
|
||||
self.assertEqual(attrs["sglang.prepare_for_extend_count"], 2)
|
||||
self.assertNotIn("sglang.prepare_for_extend_non_cpu_ms", attrs)
|
||||
self.assertNotIn("sglang.prepare_for_extend_accounting_anomaly", attrs)
|
||||
|
||||
def test_full_cpu_measurement_emits_non_cpu(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add(
|
||||
"prepare_for_extend", t0, t0 + 60_000_000, thread_cpu_ns=55_000_000
|
||||
)
|
||||
mod.detail_add(
|
||||
"prepare_for_extend", t0, t0 + 40_000_000, thread_cpu_ns=35_000_000
|
||||
)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertAlmostEqual(
|
||||
attrs["sglang.prepare_for_extend_thread_cpu_ms"], 90.0
|
||||
)
|
||||
self.assertEqual(
|
||||
attrs["sglang.prepare_for_extend_thread_cpu_measured_count"], 2
|
||||
)
|
||||
self.assertAlmostEqual(attrs["sglang.prepare_for_extend_non_cpu_ms"], 10.0)
|
||||
|
||||
def test_unmeasured_phase_has_no_cpu_fields(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("prepare_for_extend", t0, t0 + 10_000_000)
|
||||
mod.detail_add("prepare_for_extend", t0, t0 + 20_000_000)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertNotIn("sglang.prepare_for_extend_thread_cpu_ms", attrs)
|
||||
self.assertNotIn("sglang.prepare_for_extend_non_cpu_ms", attrs)
|
||||
self.assertNotIn(
|
||||
"sglang.prepare_for_extend_thread_cpu_measured_count", attrs
|
||||
)
|
||||
|
||||
def test_cpu_greater_than_wall_flags_anomaly(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add(
|
||||
"prepare_for_extend", t0, t0 + 10_000_000, thread_cpu_ns=12_000_000
|
||||
)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _loop_attrs(tracer)
|
||||
self.assertEqual(attrs["sglang.prepare_for_extend_non_cpu_ms"], 0.0)
|
||||
self.assertAlmostEqual(
|
||||
attrs["sglang.prepare_for_extend_non_cpu_raw_ms"], -2.0
|
||||
)
|
||||
self.assertTrue(attrs["sglang.prepare_for_extend_accounting_anomaly"])
|
||||
self.assertEqual(attrs["sglang.accounting_anomaly_count"], 1)
|
||||
|
||||
def test_child_span_carries_cpu_attributes(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add(
|
||||
"prefill_batch_init", t0, t0 + 50_000_000, thread_cpu_ns=20_000_000
|
||||
)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
span = next(s for s in tracer.spans if s.name == "scheduler.loop.prefill_batch_init")
|
||||
self.assertAlmostEqual(span.attributes["sglang.phase_ms"], 50.0)
|
||||
self.assertAlmostEqual(span.attributes["sglang.phase_thread_cpu_ms"], 20.0)
|
||||
self.assertAlmostEqual(span.attributes["sglang.phase_non_cpu_ms"], 30.0)
|
||||
self.assertEqual(span.attributes["sglang.phase_cpu_measured_count"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Eviction accounting semantics for the prefill_batch_init instrumentation.
|
||||
|
||||
Verifies the review's P1-1: demand (what the batch wants allocatable),
|
||||
shortfall (what eviction must actually recover) and the plan handed to
|
||||
``tree_cache.evict_for_alloc`` are three distinct counters, and that a batch
|
||||
that already fits records a zero shortfall instead of a negative one.
|
||||
|
||||
Requires an importable ``sglang`` + ``torch``; skipped on bare interpreters.
|
||||
"""
|
||||
import os
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
_DETAIL_ENV = {
|
||||
"SGLANG_BATCH_STAGE_TIMING": "1",
|
||||
"SGLANG_TRACE_LOOP_SPAN": "1",
|
||||
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
|
||||
"SGLANG_TRACE_LOOP_DETAIL": "1",
|
||||
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
|
||||
"SGLANG_TRACE_FORWARD_SPAN": "0",
|
||||
}
|
||||
|
||||
_AVAILABLE = False
|
||||
_common = None
|
||||
_batch_telemetry = None
|
||||
try: # pragma: no cover - environment dependent
|
||||
import torch # noqa: F401
|
||||
from sglang.srt.mem_cache import common as _common
|
||||
from sglang.srt.observability import batch_telemetry as _batch_telemetry
|
||||
|
||||
_AVAILABLE = True
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
class _FakeAllocator:
|
||||
def __init__(self, available):
|
||||
self.available = available
|
||||
self.calls = []
|
||||
|
||||
def available_size(self):
|
||||
return self.available
|
||||
|
||||
def evict_to_free_tokens(self, tree_cache, num_tokens):
|
||||
self.calls.append(num_tokens)
|
||||
self.available += num_tokens
|
||||
return True
|
||||
|
||||
|
||||
class _FakeTreeCache:
|
||||
def __init__(self, allocator):
|
||||
self.token_to_kv_pool_allocator = allocator
|
||||
|
||||
def supports_prefix_sharing(self):
|
||||
return True
|
||||
|
||||
|
||||
class _ZeroEvictTree:
|
||||
def evict_for_alloc(self, params, **kwargs):
|
||||
return types.SimpleNamespace(
|
||||
num_tokens_evicted=0, swa_num_tokens_evicted=0, mamba_num_evicted=0
|
||||
)
|
||||
|
||||
|
||||
class _FakeTreeCacheForAlloc:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def evict_for_alloc(self, params, **kwargs):
|
||||
self.calls.append((params, kwargs))
|
||||
return types.SimpleNamespace(
|
||||
num_tokens_evicted=int(getattr(params, "num_tokens", 0) or 0),
|
||||
swa_num_tokens_evicted=int(getattr(params, "swa_num_tokens", 0) or 0),
|
||||
mamba_num_evicted=int(getattr(params, "mamba_num", 0) or 0),
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(_AVAILABLE, "requires importable sglang + torch")
|
||||
class EvictionSemanticsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._env = mock.patch.dict(os.environ, _DETAIL_ENV)
|
||||
self._env.start()
|
||||
# Force the module-level gate on for this process.
|
||||
_batch_telemetry._DETAIL_ENABLED = True
|
||||
|
||||
def tearDown(self):
|
||||
self._env.stop()
|
||||
|
||||
def _begin(self):
|
||||
return _batch_telemetry.begin_loop(
|
||||
"prefill", pre_has_work=True, forward_active=False
|
||||
)
|
||||
|
||||
def test_demand_shortfall_and_plan_are_distinct(self):
|
||||
allocator = _FakeAllocator(available=31_744)
|
||||
tree = _FakeTreeCache(allocator)
|
||||
lt = self._begin()
|
||||
_common.evict_from_tree_cache(tree, 32_768)
|
||||
sums = lt.detail["sums"]
|
||||
attrs = lt.detail["attrs"]
|
||||
self.assertEqual(sums["evict_demand_tokens"], 32_768)
|
||||
self.assertEqual(sums["evict_shortfall_tokens"], 1_024)
|
||||
self.assertEqual(attrs["evict_free_tokens_before"], 31_744)
|
||||
# The allocator itself computes the plan from the live shortfall.
|
||||
self.assertEqual(allocator.calls, [32_768])
|
||||
|
||||
def test_no_shortfall_is_zero_not_negative(self):
|
||||
allocator = _FakeAllocator(available=40_000)
|
||||
tree = _FakeTreeCache(allocator)
|
||||
lt = self._begin()
|
||||
_common.evict_from_tree_cache(tree, 32_768)
|
||||
sums = lt.detail["sums"]
|
||||
self.assertEqual(sums["evict_demand_tokens"], 32_768)
|
||||
self.assertEqual(sums["evict_shortfall_tokens"], 0)
|
||||
|
||||
def test_plan_and_realized_counts(self):
|
||||
lt = self._begin()
|
||||
_common.evict_for_alloc_observed(
|
||||
_FakeTreeCacheForAlloc(),
|
||||
types.SimpleNamespace(num_tokens=1_024, swa_num_tokens=256, mamba_num=0),
|
||||
)
|
||||
sums = lt.detail["sums"]
|
||||
self.assertEqual(sums["evict_plan_requested_tokens"], 1_024)
|
||||
self.assertEqual(sums["evict_plan_requested_swa_tokens"], 256)
|
||||
self.assertEqual(sums["evict_evicted_tokens"], 1_024)
|
||||
self.assertEqual(sums["evict_evicted_swa_tokens"], 256)
|
||||
self.assertEqual(lt.detail["counts"]["evict_for_alloc_calls"], 1)
|
||||
|
||||
def test_mamba_plan_and_zero_realized_are_visible(self):
|
||||
lt = self._begin()
|
||||
_common.evict_for_alloc_observed(
|
||||
_ZeroEvictTree(),
|
||||
types.SimpleNamespace(num_tokens=0, swa_num_tokens=0, mamba_num=5),
|
||||
)
|
||||
sums = lt.detail["sums"]
|
||||
self.assertEqual(sums["evict_plan_requested_mamba"], 5)
|
||||
# Zero realized eviction must be recorded, not omitted.
|
||||
self.assertIn("evict_evicted_mamba", sums)
|
||||
self.assertEqual(sums["evict_evicted_mamba"], 0)
|
||||
|
||||
def test_shortfall_is_per_call_not_aggregated(self):
|
||||
allocator = _FakeAllocator(available=10_000)
|
||||
tree = _FakeTreeCache(allocator)
|
||||
lt = self._begin()
|
||||
_common.evict_from_tree_cache(tree, 12_000) # shortfall 2_000, frees 12k
|
||||
_common.evict_from_tree_cache(tree, 12_000) # now fits: shortfall 0
|
||||
sums = lt.detail["sums"]
|
||||
self.assertEqual(sums["evict_demand_tokens"], 24_000)
|
||||
self.assertEqual(sums["evict_shortfall_tokens"], 2_000)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user