observability: optional thread-CPU recording for detail phases

Adds an optional thread CPU clock next to the existing wall clock:

* detail_begin/end record time.thread_time_ns() only when detail is on and the
  loop is sampled; detail_add() accepts an explicit thread_cpu_ns for callers
  that already hold absolute start/end points (the prefill_batch_init parent).
* phase records carry [wall, count, start_wall, end_wall, cpu_sum, cpu_count].
  thread_cpu_ms and thread_cpu_measured_count are emitted whenever any
  occurrence was measured; non_cpu_ms only when every occurrence was, so a
  partial measurement never attributes unmeasured wall to non-CPU time. A
  phase whose CPU exceeds its wall keeps the clamped value, adds
  non_cpu_raw_ms and accounting_anomaly, and bumps a per-loop anomaly count.
* new detail_attr_sum()/detail_attr_setdefault() accumulate low-cardinality
  numeric loop attributes (eviction tokens, free-token snapshots).
* prefix_device_indices joins the attribute-only prefixes so per-request
  repeats never create spans.
* child spans carry phase_thread_cpu_ms / phase_cpu_measured_count /
  phase_non_cpu_ms, so the CPU split is visible without the loop attributes.

Default-off path is unchanged: no thread clock, no new dicts, no spans.
This commit is contained in:
2026-11-12 10:30:29 +08:00
parent 9f9dce1ff6
commit fec10023e0
2 changed files with 131 additions and 24 deletions
@@ -1244,6 +1244,7 @@ _LAST_LOOP_SUMMARY: dict = {}
# grows with n_reqs / n_segments (design §4.2).
_DETAIL_ATTRIBUTE_ONLY_PREFIXES = (
"per_request_result",
"prefix_device_indices",
"kv_state_payload",
"kv_page_indices",
"kv_sender_send",
@@ -1279,7 +1280,7 @@ def _detail_state():
if lt is None or not loop_sample_decision(lt.predicted_seq):
return None
if lt.detail is None:
lt.detail = {"phases": {}, "counts": {}}
lt.detail = {"phases": {}, "counts": {}, "sums": {}}
return lt.detail
@@ -1295,7 +1296,7 @@ def detail_begin(name: str) -> Optional[int]:
if opens is None:
opens = {}
_DETAIL_TLS.open = opens
opens[name] = (now_ns, time.time_ns())
opens[name] = (now_ns, time.time_ns(), _thread_time_ns())
return now_ns
@@ -1310,17 +1311,39 @@ def detail_end(name: str, token: Optional[int] = None) -> None:
opens = getattr(_DETAIL_TLS, "open", None) or {}
entry = opens.pop(name, None)
start_wall_ns = entry[1] if entry is not None else None
thread_cpu_ns = None
if entry is not None and len(entry) > 2 and entry[2] is not None:
cpu_now = _thread_time_ns()
if cpu_now is not None:
thread_cpu_ns = max(0, cpu_now - int(entry[2]))
if token is None:
if entry is None:
return
token = entry[0]
if start_wall_ns is None:
start_wall_ns = time.time_ns() - max(0, now_ns - int(token))
_detail_record(state, name, int(token), now_ns, start_wall_ns, time.time_ns())
_detail_record(
state,
name,
int(token),
now_ns,
start_wall_ns,
time.time_ns(),
thread_cpu_ns=thread_cpu_ns,
)
def detail_add(name: str, start_ns: int, end_ns: int) -> None:
"""Accumulate one occurrence of a repeated phase (no span per occurrence)."""
def detail_add(
name: str,
start_ns: int,
end_ns: int,
thread_cpu_ns: Optional[int] = None,
) -> None:
"""Accumulate one occurrence of a phase (no span per occurrence).
``thread_cpu_ns`` is optional: callers that did not measure CPU time must
leave it as None so reports can tell "not measured" apart from "zero".
"""
if not _DETAIL_ENABLED or start_ns is None or end_ns is None:
return
state = _detail_state()
@@ -1332,23 +1355,76 @@ def detail_add(name: str, start_ns: int, end_ns: int) -> None:
end_ns = start_ns
now_wall_ns = time.time_ns()
_detail_record(
state, name, start_ns, end_ns, now_wall_ns - (end_ns - start_ns), now_wall_ns
state,
name,
start_ns,
end_ns,
now_wall_ns - (end_ns - start_ns),
now_wall_ns,
thread_cpu_ns=thread_cpu_ns,
)
def _detail_record(state, name, start_ns, end_ns, start_wall_ns, end_wall_ns) -> None:
def _detail_record(
state,
name,
start_ns,
end_ns,
start_wall_ns,
end_wall_ns,
thread_cpu_ns: Optional[int] = None,
) -> None:
rec = state["phases"].get(name)
if rec is None:
state["phases"][name] = [
end_ns - start_ns,
1,
int(start_wall_ns),
int(end_wall_ns),
]
rec = [end_ns - start_ns, 1, int(start_wall_ns), int(end_wall_ns), 0, 0]
state["phases"][name] = rec
else:
rec[0] += end_ns - start_ns
rec[1] += 1
rec[3] = int(end_wall_ns)
if thread_cpu_ns is not None:
rec[4] += int(thread_cpu_ns)
rec[5] += 1
def thread_cpu_ns() -> Optional[int]:
"""Thread CPU clock, or None when the platform has no such clock.
Callers outside this module (the scheduler's parent span) must use this
wrapper so an unsupported or failing clock degrades to "not measured"
instead of raising on the scheduling path.
"""
try:
return time.thread_time_ns()
except (AttributeError, OSError, ValueError):
return None
# Backwards-compatible private alias for module-local call sites.
_thread_time_ns = thread_cpu_ns
def detail_attr_setdefault(name: str, value) -> None:
"""Set a loop attribute only if this loop has not recorded it yet."""
if not _DETAIL_ENABLED:
return
rec[0] += end_ns - start_ns
rec[1] += 1
rec[3] = int(end_wall_ns)
state = _detail_state()
if state is None:
return
attrs = state.setdefault("attrs", {})
if name not in attrs:
attrs[name] = value
def detail_attr_sum(name: str, value: int) -> None:
"""Accumulate a numeric loop attribute (e.g. evicted tokens)."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
sums = state.setdefault("sums", {})
sums[name] = int(sums.get(name, 0)) + int(value)
def detail_count(name: str, n: int = 1) -> None:
@@ -2301,31 +2377,58 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None
detail = lt.detail or {}
phases = detail.get("phases") or {}
ms = {}
accounting_anomalies = 0
for name, rec in phases.items():
try:
total_ns, count, start_wall, end_wall = rec
total_ns, count, start_wall, end_wall = rec[:4]
except Exception:
continue
key = "sglang." + name.replace(".", "_") + "_ms"
ms[name] = total_ns / 1e6
attrs[key] = ms[name]
cpu_ns, cpu_count = (rec[4], rec[5]) if len(rec) > 5 else (0, 0)
prefix = "sglang." + name.replace(".", "_")
non_cpu_ms = None
if cpu_count:
attrs[prefix + "_thread_cpu_ms"] = cpu_ns / 1e6
attrs[prefix + "_thread_cpu_measured_count"] = int(cpu_count)
# Only a fully measured phase may report the wall - CPU remainder;
# a partial measurement would silently attribute unmeasured time to
# "non-CPU".
if cpu_count == count:
non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6)
attrs[prefix + "_non_cpu_ms"] = non_cpu_ms
if total_ns < cpu_ns:
attrs[prefix + "_non_cpu_raw_ms"] = (total_ns - cpu_ns) / 1e6
attrs[prefix + "_accounting_anomaly"] = True
accounting_anomalies += 1
if count > 1:
attrs["sglang." + name.replace(".", "_") + "_count"] = int(count)
attrs[prefix + "_count"] = int(count)
continue
if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES:
continue
child_attrs = {
"sglang.dp_rank": int(dp_rank),
"sglang.phase_ms": total_ns / 1e6,
}
if cpu_count:
child_attrs["sglang.phase_thread_cpu_ms"] = cpu_ns / 1e6
child_attrs["sglang.phase_cpu_measured_count"] = int(cpu_count)
if non_cpu_ms is not None:
child_attrs["sglang.phase_non_cpu_ms"] = non_cpu_ms
child(
"scheduler.loop." + name.replace(".", "_"),
start_wall,
end_wall,
{
"sglang.dp_rank": int(dp_rank),
"sglang.phase_ms": total_ns / 1e6,
},
child_attrs,
)
if accounting_anomalies:
attrs["sglang.accounting_anomaly_count"] = int(accounting_anomalies)
for name, value in (detail.get("counts") or {}).items():
attrs["sglang." + name.replace(".", "_") + "_count"] = int(value)
for name, value in (detail.get("sums") or {}).items():
attrs["sglang." + name.replace(".", "_")] = int(value)
for name, value in (detail.get("attrs") or {}).items():
attrs["sglang." + name.replace(".", "_")] = value
@@ -109,12 +109,16 @@ def after_execute(ticket) -> None:
_disable()
def detail(name: str, start_ns: int, end_ns: int) -> None:
def detail(
name: str, start_ns: int, end_ns: int, thread_cpu_ns=None
) -> None:
"""Attach a host phase window to the current loop span (absolute times)."""
try:
from sglang.srt.observability import batch_telemetry
batch_telemetry.detail_add(name, start_ns, end_ns)
batch_telemetry.detail_add(
name, start_ns, end_ns, thread_cpu_ns=thread_cpu_ns
)
except Exception:
pass