observability: optional thread-CPU recording for detail phases
Adds an optional thread CPU clock next to the existing wall clock: * detail_begin/end record time.thread_time_ns() only when detail is on and the loop is sampled; detail_add() accepts an explicit thread_cpu_ns for callers that already hold absolute start/end points (the prefill_batch_init parent). * phase records carry [wall, count, start_wall, end_wall, cpu_sum, cpu_count]. thread_cpu_ms and thread_cpu_measured_count are emitted whenever any occurrence was measured; non_cpu_ms only when every occurrence was, so a partial measurement never attributes unmeasured wall to non-CPU time. A phase whose CPU exceeds its wall keeps the clamped value, adds non_cpu_raw_ms and accounting_anomaly, and bumps a per-loop anomaly count. * new detail_attr_sum()/detail_attr_setdefault() accumulate low-cardinality numeric loop attributes (eviction tokens, free-token snapshots). * prefix_device_indices joins the attribute-only prefixes so per-request repeats never create spans. * child spans carry phase_thread_cpu_ms / phase_cpu_measured_count / phase_non_cpu_ms, so the CPU split is visible without the loop attributes. Default-off path is unchanged: no thread clock, no new dicts, no spans.
This commit is contained in:
@@ -1244,6 +1244,7 @@ _LAST_LOOP_SUMMARY: dict = {}
|
||||
# grows with n_reqs / n_segments (design §4.2).
|
||||
_DETAIL_ATTRIBUTE_ONLY_PREFIXES = (
|
||||
"per_request_result",
|
||||
"prefix_device_indices",
|
||||
"kv_state_payload",
|
||||
"kv_page_indices",
|
||||
"kv_sender_send",
|
||||
@@ -1279,7 +1280,7 @@ def _detail_state():
|
||||
if lt is None or not loop_sample_decision(lt.predicted_seq):
|
||||
return None
|
||||
if lt.detail is None:
|
||||
lt.detail = {"phases": {}, "counts": {}}
|
||||
lt.detail = {"phases": {}, "counts": {}, "sums": {}}
|
||||
return lt.detail
|
||||
|
||||
|
||||
@@ -1295,7 +1296,7 @@ def detail_begin(name: str) -> Optional[int]:
|
||||
if opens is None:
|
||||
opens = {}
|
||||
_DETAIL_TLS.open = opens
|
||||
opens[name] = (now_ns, time.time_ns())
|
||||
opens[name] = (now_ns, time.time_ns(), _thread_time_ns())
|
||||
return now_ns
|
||||
|
||||
|
||||
@@ -1310,17 +1311,39 @@ def detail_end(name: str, token: Optional[int] = None) -> None:
|
||||
opens = getattr(_DETAIL_TLS, "open", None) or {}
|
||||
entry = opens.pop(name, None)
|
||||
start_wall_ns = entry[1] if entry is not None else None
|
||||
thread_cpu_ns = None
|
||||
if entry is not None and len(entry) > 2 and entry[2] is not None:
|
||||
cpu_now = _thread_time_ns()
|
||||
if cpu_now is not None:
|
||||
thread_cpu_ns = max(0, cpu_now - int(entry[2]))
|
||||
if token is None:
|
||||
if entry is None:
|
||||
return
|
||||
token = entry[0]
|
||||
if start_wall_ns is None:
|
||||
start_wall_ns = time.time_ns() - max(0, now_ns - int(token))
|
||||
_detail_record(state, name, int(token), now_ns, start_wall_ns, time.time_ns())
|
||||
_detail_record(
|
||||
state,
|
||||
name,
|
||||
int(token),
|
||||
now_ns,
|
||||
start_wall_ns,
|
||||
time.time_ns(),
|
||||
thread_cpu_ns=thread_cpu_ns,
|
||||
)
|
||||
|
||||
|
||||
def detail_add(name: str, start_ns: int, end_ns: int) -> None:
|
||||
"""Accumulate one occurrence of a repeated phase (no span per occurrence)."""
|
||||
def detail_add(
|
||||
name: str,
|
||||
start_ns: int,
|
||||
end_ns: int,
|
||||
thread_cpu_ns: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Accumulate one occurrence of a phase (no span per occurrence).
|
||||
|
||||
``thread_cpu_ns`` is optional: callers that did not measure CPU time must
|
||||
leave it as None so reports can tell "not measured" apart from "zero".
|
||||
"""
|
||||
if not _DETAIL_ENABLED or start_ns is None or end_ns is None:
|
||||
return
|
||||
state = _detail_state()
|
||||
@@ -1332,23 +1355,76 @@ def detail_add(name: str, start_ns: int, end_ns: int) -> None:
|
||||
end_ns = start_ns
|
||||
now_wall_ns = time.time_ns()
|
||||
_detail_record(
|
||||
state, name, start_ns, end_ns, now_wall_ns - (end_ns - start_ns), now_wall_ns
|
||||
state,
|
||||
name,
|
||||
start_ns,
|
||||
end_ns,
|
||||
now_wall_ns - (end_ns - start_ns),
|
||||
now_wall_ns,
|
||||
thread_cpu_ns=thread_cpu_ns,
|
||||
)
|
||||
|
||||
|
||||
def _detail_record(state, name, start_ns, end_ns, start_wall_ns, end_wall_ns) -> None:
|
||||
def _detail_record(
|
||||
state,
|
||||
name,
|
||||
start_ns,
|
||||
end_ns,
|
||||
start_wall_ns,
|
||||
end_wall_ns,
|
||||
thread_cpu_ns: Optional[int] = None,
|
||||
) -> None:
|
||||
rec = state["phases"].get(name)
|
||||
if rec is None:
|
||||
state["phases"][name] = [
|
||||
end_ns - start_ns,
|
||||
1,
|
||||
int(start_wall_ns),
|
||||
int(end_wall_ns),
|
||||
]
|
||||
rec = [end_ns - start_ns, 1, int(start_wall_ns), int(end_wall_ns), 0, 0]
|
||||
state["phases"][name] = rec
|
||||
else:
|
||||
rec[0] += end_ns - start_ns
|
||||
rec[1] += 1
|
||||
rec[3] = int(end_wall_ns)
|
||||
if thread_cpu_ns is not None:
|
||||
rec[4] += int(thread_cpu_ns)
|
||||
rec[5] += 1
|
||||
|
||||
|
||||
def thread_cpu_ns() -> Optional[int]:
|
||||
"""Thread CPU clock, or None when the platform has no such clock.
|
||||
|
||||
Callers outside this module (the scheduler's parent span) must use this
|
||||
wrapper so an unsupported or failing clock degrades to "not measured"
|
||||
instead of raising on the scheduling path.
|
||||
"""
|
||||
try:
|
||||
return time.thread_time_ns()
|
||||
except (AttributeError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# Backwards-compatible private alias for module-local call sites.
|
||||
_thread_time_ns = thread_cpu_ns
|
||||
|
||||
|
||||
def detail_attr_setdefault(name: str, value) -> None:
|
||||
"""Set a loop attribute only if this loop has not recorded it yet."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return
|
||||
rec[0] += end_ns - start_ns
|
||||
rec[1] += 1
|
||||
rec[3] = int(end_wall_ns)
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return
|
||||
attrs = state.setdefault("attrs", {})
|
||||
if name not in attrs:
|
||||
attrs[name] = value
|
||||
|
||||
|
||||
def detail_attr_sum(name: str, value: int) -> None:
|
||||
"""Accumulate a numeric loop attribute (e.g. evicted tokens)."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return
|
||||
sums = state.setdefault("sums", {})
|
||||
sums[name] = int(sums.get(name, 0)) + int(value)
|
||||
|
||||
|
||||
def detail_count(name: str, n: int = 1) -> None:
|
||||
@@ -2301,31 +2377,58 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None
|
||||
detail = lt.detail or {}
|
||||
phases = detail.get("phases") or {}
|
||||
ms = {}
|
||||
accounting_anomalies = 0
|
||||
for name, rec in phases.items():
|
||||
try:
|
||||
total_ns, count, start_wall, end_wall = rec
|
||||
total_ns, count, start_wall, end_wall = rec[:4]
|
||||
except Exception:
|
||||
continue
|
||||
key = "sglang." + name.replace(".", "_") + "_ms"
|
||||
ms[name] = total_ns / 1e6
|
||||
attrs[key] = ms[name]
|
||||
cpu_ns, cpu_count = (rec[4], rec[5]) if len(rec) > 5 else (0, 0)
|
||||
prefix = "sglang." + name.replace(".", "_")
|
||||
non_cpu_ms = None
|
||||
if cpu_count:
|
||||
attrs[prefix + "_thread_cpu_ms"] = cpu_ns / 1e6
|
||||
attrs[prefix + "_thread_cpu_measured_count"] = int(cpu_count)
|
||||
# Only a fully measured phase may report the wall - CPU remainder;
|
||||
# a partial measurement would silently attribute unmeasured time to
|
||||
# "non-CPU".
|
||||
if cpu_count == count:
|
||||
non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6)
|
||||
attrs[prefix + "_non_cpu_ms"] = non_cpu_ms
|
||||
if total_ns < cpu_ns:
|
||||
attrs[prefix + "_non_cpu_raw_ms"] = (total_ns - cpu_ns) / 1e6
|
||||
attrs[prefix + "_accounting_anomaly"] = True
|
||||
accounting_anomalies += 1
|
||||
if count > 1:
|
||||
attrs["sglang." + name.replace(".", "_") + "_count"] = int(count)
|
||||
attrs[prefix + "_count"] = int(count)
|
||||
continue
|
||||
if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES:
|
||||
continue
|
||||
child_attrs = {
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.phase_ms": total_ns / 1e6,
|
||||
}
|
||||
if cpu_count:
|
||||
child_attrs["sglang.phase_thread_cpu_ms"] = cpu_ns / 1e6
|
||||
child_attrs["sglang.phase_cpu_measured_count"] = int(cpu_count)
|
||||
if non_cpu_ms is not None:
|
||||
child_attrs["sglang.phase_non_cpu_ms"] = non_cpu_ms
|
||||
child(
|
||||
"scheduler.loop." + name.replace(".", "_"),
|
||||
start_wall,
|
||||
end_wall,
|
||||
{
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.phase_ms": total_ns / 1e6,
|
||||
},
|
||||
child_attrs,
|
||||
)
|
||||
|
||||
if accounting_anomalies:
|
||||
attrs["sglang.accounting_anomaly_count"] = int(accounting_anomalies)
|
||||
for name, value in (detail.get("counts") or {}).items():
|
||||
attrs["sglang." + name.replace(".", "_") + "_count"] = int(value)
|
||||
for name, value in (detail.get("sums") or {}).items():
|
||||
attrs["sglang." + name.replace(".", "_")] = int(value)
|
||||
for name, value in (detail.get("attrs") or {}).items():
|
||||
attrs["sglang." + name.replace(".", "_")] = value
|
||||
|
||||
|
||||
@@ -109,12 +109,16 @@ def after_execute(ticket) -> None:
|
||||
_disable()
|
||||
|
||||
|
||||
def detail(name: str, start_ns: int, end_ns: int) -> None:
|
||||
def detail(
|
||||
name: str, start_ns: int, end_ns: int, thread_cpu_ns=None
|
||||
) -> None:
|
||||
"""Attach a host phase window to the current loop span (absolute times)."""
|
||||
try:
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
|
||||
batch_telemetry.detail_add(name, start_ns, end_ns)
|
||||
batch_telemetry.detail_add(
|
||||
name, start_ns, end_ns, thread_cpu_ns=thread_cpu_ns
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user