diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index f4d0170ffc..9e56d8c8bd 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -1244,6 +1244,7 @@ _LAST_LOOP_SUMMARY: dict = {} # grows with n_reqs / n_segments (design ยง4.2). _DETAIL_ATTRIBUTE_ONLY_PREFIXES = ( "per_request_result", + "prefix_device_indices", "kv_state_payload", "kv_page_indices", "kv_sender_send", @@ -1279,7 +1280,7 @@ def _detail_state(): if lt is None or not loop_sample_decision(lt.predicted_seq): return None if lt.detail is None: - lt.detail = {"phases": {}, "counts": {}} + lt.detail = {"phases": {}, "counts": {}, "sums": {}} return lt.detail @@ -1295,7 +1296,7 @@ def detail_begin(name: str) -> Optional[int]: if opens is None: opens = {} _DETAIL_TLS.open = opens - opens[name] = (now_ns, time.time_ns()) + opens[name] = (now_ns, time.time_ns(), _thread_time_ns()) return now_ns @@ -1310,17 +1311,39 @@ def detail_end(name: str, token: Optional[int] = None) -> None: opens = getattr(_DETAIL_TLS, "open", None) or {} entry = opens.pop(name, None) start_wall_ns = entry[1] if entry is not None else None + thread_cpu_ns = None + if entry is not None and len(entry) > 2 and entry[2] is not None: + cpu_now = _thread_time_ns() + if cpu_now is not None: + thread_cpu_ns = max(0, cpu_now - int(entry[2])) if token is None: if entry is None: return token = entry[0] if start_wall_ns is None: start_wall_ns = time.time_ns() - max(0, now_ns - int(token)) - _detail_record(state, name, int(token), now_ns, start_wall_ns, time.time_ns()) + _detail_record( + state, + name, + int(token), + now_ns, + start_wall_ns, + time.time_ns(), + thread_cpu_ns=thread_cpu_ns, + ) -def detail_add(name: str, start_ns: int, end_ns: int) -> None: - """Accumulate one occurrence of a repeated phase (no span per occurrence).""" +def detail_add( + name: str, + start_ns: int, + end_ns: int, + thread_cpu_ns: Optional[int] = None, +) -> None: + """Accumulate one occurrence of a phase (no span per occurrence). + + ``thread_cpu_ns`` is optional: callers that did not measure CPU time must + leave it as None so reports can tell "not measured" apart from "zero". + """ if not _DETAIL_ENABLED or start_ns is None or end_ns is None: return state = _detail_state() @@ -1332,23 +1355,76 @@ def detail_add(name: str, start_ns: int, end_ns: int) -> None: end_ns = start_ns now_wall_ns = time.time_ns() _detail_record( - state, name, start_ns, end_ns, now_wall_ns - (end_ns - start_ns), now_wall_ns + state, + name, + start_ns, + end_ns, + now_wall_ns - (end_ns - start_ns), + now_wall_ns, + thread_cpu_ns=thread_cpu_ns, ) -def _detail_record(state, name, start_ns, end_ns, start_wall_ns, end_wall_ns) -> None: +def _detail_record( + state, + name, + start_ns, + end_ns, + start_wall_ns, + end_wall_ns, + thread_cpu_ns: Optional[int] = None, +) -> None: rec = state["phases"].get(name) if rec is None: - state["phases"][name] = [ - end_ns - start_ns, - 1, - int(start_wall_ns), - int(end_wall_ns), - ] + rec = [end_ns - start_ns, 1, int(start_wall_ns), int(end_wall_ns), 0, 0] + state["phases"][name] = rec + else: + rec[0] += end_ns - start_ns + rec[1] += 1 + rec[3] = int(end_wall_ns) + if thread_cpu_ns is not None: + rec[4] += int(thread_cpu_ns) + rec[5] += 1 + + +def thread_cpu_ns() -> Optional[int]: + """Thread CPU clock, or None when the platform has no such clock. + + Callers outside this module (the scheduler's parent span) must use this + wrapper so an unsupported or failing clock degrades to "not measured" + instead of raising on the scheduling path. + """ + try: + return time.thread_time_ns() + except (AttributeError, OSError, ValueError): + return None + + +# Backwards-compatible private alias for module-local call sites. +_thread_time_ns = thread_cpu_ns + + +def detail_attr_setdefault(name: str, value) -> None: + """Set a loop attribute only if this loop has not recorded it yet.""" + if not _DETAIL_ENABLED: return - rec[0] += end_ns - start_ns - rec[1] += 1 - rec[3] = int(end_wall_ns) + state = _detail_state() + if state is None: + return + attrs = state.setdefault("attrs", {}) + if name not in attrs: + attrs[name] = value + + +def detail_attr_sum(name: str, value: int) -> None: + """Accumulate a numeric loop attribute (e.g. evicted tokens).""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + sums = state.setdefault("sums", {}) + sums[name] = int(sums.get(name, 0)) + int(value) def detail_count(name: str, n: int = 1) -> None: @@ -2301,31 +2377,58 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None detail = lt.detail or {} phases = detail.get("phases") or {} ms = {} + accounting_anomalies = 0 for name, rec in phases.items(): try: - total_ns, count, start_wall, end_wall = rec + total_ns, count, start_wall, end_wall = rec[:4] except Exception: continue key = "sglang." + name.replace(".", "_") + "_ms" ms[name] = total_ns / 1e6 attrs[key] = ms[name] + cpu_ns, cpu_count = (rec[4], rec[5]) if len(rec) > 5 else (0, 0) + prefix = "sglang." + name.replace(".", "_") + non_cpu_ms = None + if cpu_count: + attrs[prefix + "_thread_cpu_ms"] = cpu_ns / 1e6 + attrs[prefix + "_thread_cpu_measured_count"] = int(cpu_count) + # Only a fully measured phase may report the wall - CPU remainder; + # a partial measurement would silently attribute unmeasured time to + # "non-CPU". + if cpu_count == count: + non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6) + attrs[prefix + "_non_cpu_ms"] = non_cpu_ms + if total_ns < cpu_ns: + attrs[prefix + "_non_cpu_raw_ms"] = (total_ns - cpu_ns) / 1e6 + attrs[prefix + "_accounting_anomaly"] = True + accounting_anomalies += 1 if count > 1: - attrs["sglang." + name.replace(".", "_") + "_count"] = int(count) + attrs[prefix + "_count"] = int(count) continue if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES: continue + child_attrs = { + "sglang.dp_rank": int(dp_rank), + "sglang.phase_ms": total_ns / 1e6, + } + if cpu_count: + child_attrs["sglang.phase_thread_cpu_ms"] = cpu_ns / 1e6 + child_attrs["sglang.phase_cpu_measured_count"] = int(cpu_count) + if non_cpu_ms is not None: + child_attrs["sglang.phase_non_cpu_ms"] = non_cpu_ms child( "scheduler.loop." + name.replace(".", "_"), start_wall, end_wall, - { - "sglang.dp_rank": int(dp_rank), - "sglang.phase_ms": total_ns / 1e6, - }, + child_attrs, ) + if accounting_anomalies: + attrs["sglang.accounting_anomaly_count"] = int(accounting_anomalies) for name, value in (detail.get("counts") or {}).items(): attrs["sglang." + name.replace(".", "_") + "_count"] = int(value) + for name, value in (detail.get("sums") or {}).items(): + attrs["sglang." + name.replace(".", "_")] = int(value) for name, value in (detail.get("attrs") or {}).items(): attrs["sglang." + name.replace(".", "_")] = value diff --git a/python/sglang/srt/observability/fwd_timeline.py b/python/sglang/srt/observability/fwd_timeline.py index 47e1d20cd8..ed40df5473 100644 --- a/python/sglang/srt/observability/fwd_timeline.py +++ b/python/sglang/srt/observability/fwd_timeline.py @@ -109,12 +109,16 @@ def after_execute(ticket) -> None: _disable() -def detail(name: str, start_ns: int, end_ns: int) -> None: +def detail( + name: str, start_ns: int, end_ns: int, thread_cpu_ns=None +) -> None: """Attach a host phase window to the current loop span (absolute times).""" try: from sglang.srt.observability import batch_telemetry - batch_telemetry.detail_add(name, start_ns, end_ns) + batch_telemetry.detail_add( + name, start_ns, end_ns, thread_cpu_ns=thread_cpu_ns + ) except Exception: pass