Files
sglang/python/sglang/srt/observability/batch_telemetry.py
T
minke.yu f4de9f2141 observability: ignore sub-tolerance CPU>wall skew in accounting anomaly
A 2 ms phase whose CPU clock reads 0.1 us over its wall clock is clock
granularity, not an accounting bug; the anomaly flag fired on nearly every
short phase in the live verification run.

Only flag (and record non_cpu_raw_ms) when the deficit exceeds both 100 us and
1% of the phase wall. Larger real anomalies keep the previous behaviour.
2026-11-12 12:30:58 +08:00

2743 lines
94 KiB
Python

"""Per-batch scheduler telemetry (opt-in).
Adds, at the *batch* granularity (one record per scheduler batch per DP rank):
* Prometheus histograms (safe for the multiprocess collector):
sglang:batch_new_tokens{phase,dp_rank}
sglang:batch_cached_tokens{phase,dp_rank}
sglang:batch_duration_seconds{phase,dp_rank}
sglang:batch_queue_depth{phase,dp_rank}
sglang:batch_pending_tokens{phase,dp_rank}
sglang:batch_n_reqs{phase,dp_rank}
* An OTLP span ``scheduler.batch.{phase}`` with ``Link``s to the request
traces that took part in the batch (batch -> requests navigation), and a
``sglang.batch_id`` attribute for reverse lookups / log correlation.
Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
SGLANG_BATCH_SPAN=0 histograms only (no spans)
SGLANG_BATCH_SPAN_MIN_MS=50 only emit spans for batches slower than this
SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0)
SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span
SGLANG_TRACE_LOOP_DETAIL=1 split run_batch/process_result into host
phases (requires SGLANG_BATCH_STAGE_TIMING=1
and SGLANG_TRACE_LOOP_SPAN=1; only sampled
loops are recorded)
SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown
(queue/source/indices/transfer/join)
SGLANG_TRACE_BACKUP_DETAIL=1 per-flush HiCache backup sub-phases
(one ``scheduler.cache_flush_backup`` span;
requires SGLANG_TRACE_LOOP_DETAIL=1)
Cost model (measured on our 4xDP4 / 4-GPU instances):
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
* span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s.
Everything is behind a single module-level bool, so when disabled the hot
path pays one ``if not _ENABLED: return``.
NOTE: sglang runs prometheus_client in multiprocess mode, which does not
carry exemplars, so the span trace id is returned to the caller and appended
to the batch log line instead (see ``metrics_reporter``).
"""
from __future__ import annotations
import os
import random
import threading
import time
from typing import Any, Iterable, Optional
_ENABLED = os.getenv("SGLANG_BATCH_TELEMETRY", "0").strip().lower() not in (
"",
"0",
"false",
"no",
"off",
)
_SPAN_ENABLED = _ENABLED and os.getenv("SGLANG_BATCH_SPAN", "1").strip().lower() not in (
"0",
"false",
"no",
"off",
)
_SPAN_MIN_S = float(os.getenv("SGLANG_BATCH_SPAN_MIN_MS", "0")) / 1000.0
_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_SPAN_SAMPLE", "1.0"))
_MAX_LINKS = int(os.getenv("SGLANG_BATCH_SPAN_MAX_LINKS", "32"))
_HISTOGRAMS = None
_CHILDREN: dict[tuple[str, str, int], Any] = {}
_TRACER = None
def enabled() -> bool:
return _ENABLED
_REQUEST_COUNTER = None
_DP_RANK = None
_COMPLETION_RANK = None
def _should_report_completion() -> bool:
"""Whether this scheduler is the canonical reporter of request completions.
Every TP/CP rank runs a scheduler and observes the same requests, and a PP
stage only sees its own microbatches, so an ungated counter would be
multiplied across replicas once the multiprocess registry aggregates them.
Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same
shape the existing scheduler metrics use for rank-unique series.
"""
global _COMPLETION_RANK
if _COMPLETION_RANK is None:
try:
from sglang.srt.runtime_context import get_parallel
parallel = get_parallel()
_COMPLETION_RANK = bool(
parallel.attn_tp_rank == 0
and parallel.attn_cp_rank == 0
and parallel.pp_group.is_last_rank
)
except Exception:
# Unknown topology (e.g. a bare unit-test process): keep counting.
_COMPLETION_RANK = True
return _COMPLETION_RANK
def _current_dp_rank() -> int:
"""This process's attention-DP rank (0 for non-DP engines)."""
global _DP_RANK
if _DP_RANK is None:
try:
from sglang.srt.runtime_context import get_parallel
_DP_RANK = int(get_parallel().attn_dp_rank or 0)
except Exception:
_DP_RANK = 0
return _DP_RANK
def _ensure_request_counter():
global _REQUEST_COUNTER
if _REQUEST_COUNTER is None:
from prometheus_client import Counter
_REQUEST_COUNTER = Counter(
"sglang:batch_requests_total",
"Requests completed by the canonical scheduler rank, by phase. "
"phase=prefill: prefill compute finished and the request entered the "
"PD transfer queue (not a guarantee that KV transfer succeeded). "
"phase=decode: request finished decoding.",
["phase", "dp_rank"],
)
return _REQUEST_COUNTER
def note_request_completed(phase: str, n: int = 1) -> None:
"""Count finished requests per DP rank.
This is the per-rank RPS source: the frontend ``num_requests_total`` has no
rank label, while the scheduler knows exactly which DP shard completed the
request. Cheap (one counter increment per finished request) and gated by
SGLANG_BATCH_TELEMETRY like the other batch metrics.
"""
if not _ENABLED or n <= 0 or not _should_report_completion():
return
try:
_ensure_request_counter().labels(
phase=phase, dp_rank=str(_current_dp_rank())
).inc(int(n))
except Exception:
# Telemetry must never fail a request.
pass
def _ensure_histograms():
global _HISTOGRAMS
if _HISTOGRAMS is None:
from prometheus_client import Histogram
labels = ["phase", "dp_rank"]
_HISTOGRAMS = {
"batch_new_tokens": Histogram(
"sglang:batch_new_tokens",
"New (uncached) prompt tokens processed by one scheduler batch",
labels,
buckets=(128, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536),
),
"batch_cached_tokens": Histogram(
"sglang:batch_cached_tokens",
"Cached prompt tokens carried by one scheduler batch",
labels,
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
),
"batch_duration_seconds": Histogram(
"sglang:batch_duration_seconds",
"Wall time spent in one scheduler batch",
labels,
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0),
),
"batch_queue_depth": Histogram(
"sglang:batch_queue_depth",
"Waiting-queue depth observed by one scheduler batch",
labels,
buckets=(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
),
"batch_pending_tokens": Histogram(
"sglang:batch_pending_tokens",
"Prefill backlog (pending tokens) observed by one scheduler batch",
labels,
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
),
"batch_n_reqs": Histogram(
"sglang:batch_n_reqs",
"Requests carried by one scheduler batch",
labels,
buckets=(1, 2, 4, 8, 16, 32, 64, 128, 256),
),
}
return _HISTOGRAMS
def _child(name: str, phase: str, dp_rank: int):
key = (name, phase, dp_rank)
child = _CHILDREN.get(key)
if child is None:
child = _ensure_histograms()[name].labels(phase=phase, dp_rank=str(dp_rank))
_CHILDREN[key] = child
return child
def _request_span_context(req: Any):
"""SpanContext of a request, from either a live span or a serialized copy.
Scheduler processes receive a *copy* of the request trace context where
``root_span`` is None (see ``trace_async.copy_for_thread``) and only
``root_span_context`` (an OTel ``Context``) survives. Handle both.
"""
trace_ctx = getattr(getattr(req, "time_stats", None), "trace_ctx", None)
if trace_ctx is None:
return None
try:
root_span = getattr(trace_ctx, "root_span", None)
if root_span is not None:
span_ctx = root_span.get_span_context()
if span_ctx.is_valid:
return span_ctx
root_ctx = getattr(trace_ctx, "root_span_context", None)
if root_ctx is not None:
from opentelemetry import trace as otel_trace
span_ctx = otel_trace.get_current_span(root_ctx).get_span_context()
if span_ctx.is_valid:
return span_ctx
except Exception:
return None
return None
def _emit_span(
phase: str,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]],
) -> Optional[str]:
"""Create one batch span with links to the request traces. Returns trace id."""
if not _SPAN_ENABLED:
return None
if _SPAN_MIN_S > 0 and duration_s < _SPAN_MIN_S:
return None
if _SPAN_SAMPLE < 1.0 and random.random() > _SPAN_SAMPLE:
return None
global _TRACER
try:
from opentelemetry import trace as otel_trace
from opentelemetry.trace import Link
if _TRACER is None:
_TRACER = otel_trace.get_tracer("sglang.batch")
links = []
if reqs:
for req in reqs:
if len(links) >= _MAX_LINKS:
break
ctx = _request_span_context(req)
if ctx is not None:
links.append(Link(ctx))
end_ns = time.time_ns()
start_ns = end_ns - int(duration_s * 1e9)
span = _TRACER.start_span(
f"scheduler.batch.{phase}",
start_time=start_ns,
attributes={
"sglang.batch_id": f"{dp_rank}:{iter_no}",
"sglang.dp_rank": int(dp_rank),
"sglang.phase": phase,
"sglang.n_reqs": int(n_reqs),
"sglang.n_new_tokens": int(n_new),
"sglang.n_cached_tokens": int(n_cached),
"sglang.queue_depth": int(queue_depth),
"sglang.pending_tokens": int(pending_tokens),
"sglang.batch_duration_s": float(duration_s),
},
links=links,
)
span.end(end_time=end_ns)
span_ctx = span.get_span_context()
if not span_ctx.is_valid:
# No TracerProvider configured (tracing disabled) -> NoOp span.
return None
return format(span_ctx.trace_id, "032x")
except Exception:
return None
def _observe(
phase: str,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]],
) -> Optional[str]:
if not _ENABLED:
return None
try:
trace_id = _emit_span(
phase,
dp_rank,
iter_no,
n_reqs,
n_new,
n_cached,
queue_depth,
pending_tokens,
duration_s,
reqs,
)
_child("batch_new_tokens", phase, dp_rank).observe(n_new)
_child("batch_cached_tokens", phase, dp_rank).observe(n_cached)
_child("batch_duration_seconds", phase, dp_rank).observe(duration_s)
_child("batch_queue_depth", phase, dp_rank).observe(queue_depth)
_child("batch_pending_tokens", phase, dp_rank).observe(pending_tokens)
_child("batch_n_reqs", phase, dp_rank).observe(n_reqs)
return trace_id
except Exception:
# Telemetry must never break the scheduler loop.
return None
def observe_prefill(
*,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]] = None,
) -> Optional[str]:
return _observe(
"prefill",
dp_rank,
iter_no,
n_reqs,
n_new,
n_cached,
queue_depth,
pending_tokens,
duration_s,
reqs,
)
def observe_decode(
*,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]] = None,
) -> Optional[str]:
return _observe(
"decode",
dp_rank,
iter_no,
n_reqs,
n_new,
n_cached,
queue_depth,
pending_tokens,
duration_s,
reqs,
)
_DP_HISTS = None
def _ensure_dp_histograms():
global _DP_HISTS
if _DP_HISTS is None:
from prometheus_client import Histogram
_DP_HISTS = {
"dp_step_tokens": Histogram(
"sglang:dp_step_tokens",
"Per-rank token count in one DP-attention step",
["dp_rank", "phase"],
buckets=(0, 256, 1024, 2048, 4096, 8192, 16384, 32768),
),
"dp_step_padding_ratio": Histogram(
"sglang:dp_step_padding_ratio",
"1 - mean/max tokens across active DP ranks in one step (padding waste)",
["phase", "n_active"],
buckets=(0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8),
),
"dp_step_spread": Histogram(
"sglang:dp_step_spread",
"(max-min)/mean tokens across active DP ranks in one step",
["phase", "n_active"],
buckets=(0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0),
),
}
return _DP_HISTS
_DP_IDLE_MODE = 4 # ForwardMode.IDLE
def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = "unknown") -> None:
"""Record per-step shape alignment across *active* DP ranks.
``tokens``/``modes``/``health`` are the per-rank vectors from the DP
all-gather (info tensor columns 0, 5, 9). IDLE ranks and health-check
steps are excluded so the padding ratio reflects real batch-shape
misalignment rather than idle-slot padding.
"""
if not _ENABLED:
return
try:
if health and any(bool(h) for h in health):
return
sel = []
for i, t in enumerate(tokens):
if t is None:
continue
t = int(t)
if t <= 1:
continue
if modes is not None and i < len(modes) and int(modes[i]) == _DP_IDLE_MODE:
continue
sel.append(t)
if len(sel) < 2:
return
hist = _ensure_dp_histograms()
mx = max(sel)
mean = sum(sel) / len(sel)
n_active = str(len(sel))
key = ("dp_step_tokens", phase, dp_rank)
child = _CHILDREN.get(key)
if child is None:
child = hist["dp_step_tokens"].labels(dp_rank=str(dp_rank), phase=phase)
_CHILDREN[key] = child
child.observe(int(tokens[dp_rank]) if 0 <= dp_rank < len(tokens) else int(mean))
if mx > 0:
hist["dp_step_padding_ratio"].labels(phase=phase, n_active=n_active).observe(
1.0 - mean / mx
)
if mean > 0:
hist["dp_step_spread"].labels(phase=phase, n_active=n_active).observe(
(mx - min(sel)) / mean
)
except Exception:
pass
# ---------------------------------------------------------------------------
# Per-pass stage timing (SGLANG_BATCH_STAGE_TIMING=1) -- P (disagg prefill).
#
# One sample per get_next_disagg_prefill_batch_to_run() pass, including passes
# that produce no batch. Disabled by default; when off every hook below is a
# single module-level bool check. No CUDA calls, no per-pass logging.
#
# Cross-rank analysis uses a dedicated, independently sampled span
# ``scheduler.prefill.pass`` emitted from end_pass(): the sampling decision is a
# pure function of ``mlp_sync_seq`` (identical on every DP rank that took part
# in the same gather), so a sampled gather round is sampled on *all* ranks --
# unlike the per-rank random sampling of the batch span. Idle ranks and ranks
# with no local batch therefore still emit, which is exactly the case the
# skew analysis needs.
# ---------------------------------------------------------------------------
_STAGE_TIMING = os.getenv("SGLANG_BATCH_STAGE_TIMING", "0").strip().lower() not in (
"",
"0",
"false",
"no",
"off",
)
_STAGE_LABELS = ["phase", "dp_rank"]
_STAGE_BUCKETS = (
1e-5,
3e-5,
1e-4,
3e-4,
1e-3,
3e-3,
1e-2,
3e-2,
1e-1,
3e-1,
1.0,
3.0,
)
_STAGE_HISTS = None
_STAGE_COUNTERS = None
_STAGE_CHILDREN: dict = {}
_STAGE_TLS = threading.local()
# Set by the DP all-gather path (dp_attn.prepare_mlp_sync_batch_raw).
_LAST_DP_GATHER = None
_DP_GATHER_SEQ = 0
_PASS_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
"SGLANG_BATCH_PASS_SPAN", "1"
).strip().lower() not in ("", "0", "false", "no", "off")
_PASS_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_PASS_SPAN_SAMPLE", "0.05"))
_STAGE_HIST_DOCS = {
"get_next": "Wall time of one get_next_disagg_prefill_batch_to_run pass",
"match": "Per-pass sum of init_next_round_input (prefix match) over candidates",
"adder": "Per-pass sum of PrefillAdder.add_one_req (budget/admission)",
"hicache_events": "Per-pass time in _process_hicache_events (includes its collectives)",
"bootstrap": "Per-pass time in resolve_waiting_queue_bootstrap (poll + all-reduce)",
"mlp_sync": "Per-pass outer maybe_prepare_mlp_sync_batch time (not pure gather wait)",
"other": "get_next total minus the instrumented segments",
}
def stage_timing_enabled() -> bool:
return _STAGE_TIMING
def pass_span_enabled() -> bool:
return _PASS_SPAN_ENABLED
def pass_span_decision(seq: int) -> bool:
"""Deterministic, rank-identical sampling decision for one gather round.
``seq`` is the DP all-gather sequence number, which is the same on every
rank that participated in that round. Therefore one decision covers the
whole round, instead of four independent ``random.random()`` draws.
"""
if not _PASS_SPAN_ENABLED or seq <= 0:
return False
if _PASS_SPAN_SAMPLE >= 1.0:
return True
if _PASS_SPAN_SAMPLE <= 0.0:
return False
return (seq * 2654435761) % 10000 < int(_PASS_SPAN_SAMPLE * 10000)
def _ensure_stage_metrics():
global _STAGE_HISTS, _STAGE_COUNTERS
if _STAGE_HISTS is None:
from prometheus_client import Counter, Histogram
_STAGE_HISTS = {
name: Histogram(
f"sglang:batch_prefill_{name}_seconds",
doc,
_STAGE_LABELS,
buckets=_STAGE_BUCKETS,
)
for name, doc in _STAGE_HIST_DOCS.items()
}
_STAGE_COUNTERS = {
"candidates": Counter(
"sglang:batch_prefill_candidates_total",
"Requests examined by the per-pass admission candidate loop",
_STAGE_LABELS,
),
"skipped_prefetch": Counter(
"sglang:batch_prefill_skipped_prefetch_total",
"Candidates skipped because their L2/L3 prefetch was not ready",
_STAGE_LABELS,
),
"admitted": Counter(
"sglang:batch_prefill_admitted_total",
"Requests admitted into can_run_list by one pass",
_STAGE_LABELS,
),
"resumed_chunk": Counter(
"sglang:batch_prefill_resumed_chunk_total",
"Passes whose batch continues a chunked-prefill request "
"(including its last chunk)",
_STAGE_LABELS,
),
"new_chunk": Counter(
"sglang:batch_prefill_new_chunk_total",
"Passes that start a new chunked-prefill request",
_STAGE_LABELS,
),
}
return _STAGE_HISTS, _STAGE_COUNTERS
class PassTimings:
__slots__ = (
"phase",
"t0",
"match",
"adder",
"hicache",
"bootstrap",
"mlp_sync",
"pending_abort",
"chunk",
"new_batch",
"set_sched",
"chunk_count",
"dp_gather",
"candidates",
"skipped_prefetch",
"admitted",
"bootstrap_candidates",
"pending_bootstrap",
"had_chunked",
"new_chunk",
"seq",
)
def __init__(self, phase: str):
self.phase = phase
self.t0 = time.monotonic()
self.match = 0.0
self.adder = 0.0
self.hicache = 0.0
self.bootstrap = 0.0
self.mlp_sync = 0.0
self.pending_abort = 0.0
self.chunk = 0.0
self.new_batch = 0.0
self.set_sched = 0.0
self.chunk_count = 0
self.dp_gather = None
self.candidates = 0
self.skipped_prefetch = 0
self.admitted = 0
self.bootstrap_candidates = 0
self.pending_bootstrap = 0
self.had_chunked = False
self.new_chunk = False
self.seq = 0
def begin_pass(phase: str = "prefill") -> Optional[PassTimings]:
"""Start a new pass. Returns None when stage timing is disabled."""
if not _STAGE_TIMING:
return None
timings = PassTimings(phase)
_STAGE_TLS.current = timings
return timings
def _current() -> Optional[PassTimings]:
if not _STAGE_TIMING:
return None
return getattr(_STAGE_TLS, "current", None)
def add_match(seconds: float) -> None:
t = _current()
if t is not None:
t.match += seconds
def add_adder(seconds: float) -> None:
t = _current()
if t is not None:
t.adder += seconds
def add_hicache(seconds: float) -> None:
t = _current()
if t is not None:
t.hicache += seconds
def add_bootstrap(seconds: float) -> None:
t = _current()
if t is not None:
t.bootstrap += seconds
def add_pending_abort(seconds: float) -> None:
t = _current()
if t is not None:
t.pending_abort += seconds
def add_chunk(seconds: float, count: int = 0) -> None:
t = _current()
if t is not None:
t.chunk += seconds
t.chunk_count += int(count)
def add_new_batch(seconds: float) -> None:
t = _current()
if t is not None:
t.new_batch += seconds
def add_set_sched(seconds: float) -> None:
t = _current()
if t is not None:
t.set_sched += seconds
def mark_candidate(n: int = 1) -> None:
t = _current()
if t is not None:
t.candidates += n
def mark_skipped_prefetch(n: int = 1) -> None:
t = _current()
if t is not None:
t.skipped_prefetch += n
def set_admitted(n: int) -> None:
t = _current()
if t is not None:
t.admitted = int(n)
def set_bootstrap_pending(pending: int, candidates: int) -> None:
"""Reuse the bootstrap resolver's existing pass; do not rescan the queue."""
t = _current()
if t is not None:
t.pending_bootstrap = int(pending)
t.bootstrap_candidates = int(candidates)
def note_had_chunked(flag: bool) -> None:
"""Record whether a chunked request was already in flight when the pass began.
Must be called at pass start, before process_prefill_chunk()/admission can
consume the chunked request (the last chunk clears self.chunked_req).
"""
t = _current()
if t is not None:
t.had_chunked = bool(flag)
def note_new_chunk(flag: bool) -> None:
"""Record whether this pass started a new chunked request (first chunk)."""
t = _current()
if t is not None:
t.new_chunk = bool(flag)
def note_dp_gather(seconds: Optional[float]) -> None:
"""Called by the DP all-gather path; None means no gather ran this pass."""
global _LAST_DP_GATHER, _DP_GATHER_SEQ
if not _STAGE_TIMING:
return
if seconds is None:
_LAST_DP_GATHER = None
return
_LAST_DP_GATHER = float(seconds)
_DP_GATHER_SEQ += 1
def _take_dp_gather():
global _LAST_DP_GATHER
value = _LAST_DP_GATHER
_LAST_DP_GATHER = None
return value
def add_mlp_sync(seconds: float) -> None:
t = _current()
if t is None:
return
t.mlp_sync += seconds
gather = _take_dp_gather()
if gather is not None:
t.dp_gather = gather
t.seq = _DP_GATHER_SEQ
def _emit_pass_span(
dp_rank: int,
t: PassTimings,
total: float,
other: float,
residual: float = 0.0,
) -> None:
if not pass_span_decision(t.seq):
return
loop = getattr(_LOOP_TLS, "current", None)
loop_ctx = loop.span_ctx if loop is not None else None
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
# An unsampled loop owns this pass; do not leak an independent pass trace.
# If work is discovered later, end_loop emits the snapshot as its child.
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.pass")
end_ns = time.time_ns()
start_ns = end_ns - int(total * 1e9)
tracer.start_span(
"scheduler.prefill.pass",
context=loop_ctx,
start_time=start_ns,
attributes={
"sglang.phase": t.phase,
"sglang.dp_rank": int(dp_rank),
"sglang.mlp_sync_seq": int(t.seq),
"sglang.get_next_ms": total * 1e3,
"sglang.match_ms": t.match * 1e3,
"sglang.adder_ms": t.adder * 1e3,
"sglang.hicache_ms": t.hicache * 1e3,
"sglang.bootstrap_ms": t.bootstrap * 1e3,
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
"sglang.other_ms": other * 1e3,
"sglang.pending_abort_ms": t.pending_abort * 1e3,
"sglang.chunk_ms": t.chunk * 1e3,
"sglang.new_batch_ms": t.new_batch * 1e3,
"sglang.set_sched_ms": t.set_sched * 1e3,
"sglang.get_next_residual_ms": residual * 1e3,
"sglang.chunk_count": int(t.chunk_count),
"sglang.candidates": int(t.candidates),
"sglang.skipped_prefetch": int(t.skipped_prefetch),
"sglang.admitted": int(t.admitted),
"sglang.resumed_chunked": int(bool(t.had_chunked)),
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
},
).end(end_time=end_ns)
if loop is not None and loop_ctx is not None:
loop.pass_span_child = True
except Exception:
# Tracing must never break the scheduler loop.
pass
def end_pass(dp_rank: int) -> None:
"""Close the current pass: observe histograms/counters and emit the pass span."""
t = _current()
if t is None:
return
_STAGE_TLS.current = None
try:
total = time.monotonic() - t.t0
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
other = total - measured
if other < 0.0:
other = 0.0
# Residual after also charging the get_next sub-phases.
measured2 = measured + (
t.pending_abort + t.chunk + t.new_batch + t.set_sched
)
residual = total - measured2
if residual < 0.0:
residual = 0.0
hist, counters = _ensure_stage_metrics()
child_key = (t.phase, dp_rank)
keyed = _STAGE_CHILDREN.get(child_key)
if keyed is None:
keyed = {
name: h.labels(phase=t.phase, dp_rank=str(dp_rank))
for name, h in hist.items()
}
keyed.update(
{
name: c.labels(phase=t.phase, dp_rank=str(dp_rank))
for name, c in counters.items()
}
)
_STAGE_CHILDREN[child_key] = keyed
keyed["get_next"].observe(total)
keyed["match"].observe(t.match)
keyed["adder"].observe(t.adder)
keyed["hicache_events"].observe(t.hicache)
keyed["bootstrap"].observe(t.bootstrap)
keyed["mlp_sync"].observe(t.mlp_sync)
keyed["other"].observe(other)
if t.candidates:
keyed["candidates"].inc(t.candidates)
if t.skipped_prefetch:
keyed["skipped_prefetch"].inc(t.skipped_prefetch)
if t.admitted:
keyed["admitted"].inc(t.admitted)
if t.had_chunked:
keyed["resumed_chunk"].inc(1)
elif t.new_chunk:
keyed["new_chunk"].inc(1)
note_pass_snapshot(t, total)
_emit_pass_span(dp_rank, t, total, other, residual)
except Exception:
# Telemetry must never break the scheduler loop
_STAGE_TLS.current = None
# ---------------------------------------------------------------------------
# Scheduler receive path (same SGLANG_BATCH_STAGE_TIMING gate): split the
# recv_requests stage into pull / broadcast / unwrap / mm.
#
# NOTE: tokenization itself runs in the tokenizer worker processes (HTTP side).
# What is measured here is the scheduler-side IPC: ZMQ pull + decode of the
# tokenized request objects, the cross-rank broadcast of those objects, and
# the unpickling of their token-id buffers. Large prompts make all three
# visible (e.g. 14.6k tokens per request).
# ---------------------------------------------------------------------------
_RECV_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
"SGLANG_RECV_SPAN", "1"
).strip().lower() not in ("", "0", "false", "no", "off")
_RECV_SPAN_SAMPLE = float(os.getenv("SGLANG_RECV_SPAN_SAMPLE", "0.05"))
_RECV_PARTS = ("total", "pull", "broadcast", "unwrap", "mm")
_RECV_HISTS = None
_RECV_COUNTERS = None
_RECV_SEQ = 0
def recv_timing_enabled() -> bool:
return _STAGE_TIMING
def _ensure_recv_metrics():
global _RECV_HISTS, _RECV_COUNTERS
if _RECV_HISTS is None:
from prometheus_client import Counter, Histogram
_RECV_HISTS = Histogram(
"sglang:recv_requests_seconds",
"Scheduler recv_requests wall time by part "
"(total/pull/broadcast/unwrap/mm)",
["dp_rank", "part"],
buckets=_STAGE_BUCKETS,
)
_RECV_COUNTERS = {
"msgs": Counter(
"sglang:recv_requests_msgs_total",
"Messages received by the scheduler recv path",
["dp_rank"],
),
"prompt_tokens": Counter(
"sglang:recv_requests_prompt_tokens_total",
"Prompt tokens carried by messages received in the recv path",
["dp_rank"],
),
}
return _RECV_HISTS, _RECV_COUNTERS
def observe_recv(
*,
dp_rank: int,
total: float,
pull: float,
broadcast: float,
unwrap: float,
mm: float,
n_msgs: int,
n_tokens: int,
) -> None:
if not _STAGE_TIMING:
return
global _RECV_SEQ
_RECV_SEQ += 1
try:
hist, counters = _ensure_recv_metrics()
for part, value in zip(_RECV_PARTS, (total, pull, broadcast, unwrap, mm)):
hist.labels(dp_rank=str(dp_rank), part=part).observe(value)
if n_msgs:
counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs)
if n_tokens:
counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens)
end_ns = time.time_ns()
start_ns = end_ns - int(total * 1e9)
note_recv_phase(
start_wall_ns=start_ns,
end_wall_ns=end_ns,
parts={
"sglang.recv_total_ms": total * 1e3,
"sglang.recv_pull_ms": pull * 1e3,
"sglang.recv_broadcast_ms": broadcast * 1e3,
"sglang.recv_unwrap_ms": unwrap * 1e3,
"sglang.recv_mm_ms": mm * 1e3,
"sglang.recv_msgs": int(n_msgs),
"sglang.recv_prompt_tokens": int(n_tokens),
},
)
if not _RECV_SPAN_ENABLED:
return
if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int(
_RECV_SPAN_SAMPLE * 10000
):
return
loop = current_loop()
loop_ctx = current_loop_context()
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
# Keep the snapshot only; end_loop emits it if this loop has work.
return
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.recv")
tracer.start_span(
"scheduler.recv_requests",
context=loop_ctx,
start_time=start_ns,
attributes={
"sglang.phase": "recv",
"sglang.dp_rank": int(dp_rank),
"sglang.recv_msgs": int(n_msgs),
"sglang.recv_prompt_tokens": int(n_tokens),
"sglang.recv_total_ms": total * 1e3,
"sglang.recv_pull_ms": pull * 1e3,
"sglang.recv_broadcast_ms": broadcast * 1e3,
"sglang.recv_unwrap_ms": unwrap * 1e3,
"sglang.recv_mm_ms": mm * 1e3,
},
).end(end_time=end_ns)
if loop is not None and loop_ctx is not None:
loop.recv_span_child = True
except Exception:
# Telemetry must never break the scheduler loop.
pass
# ---------------------------------------------------------------------------
# Storage prefetch ready latency (same SGLANG_BATCH_STAGE_TIMING gate).
#
# Request-level wait from "prefetch in flight" to "usable" (the scheduler
# check returns True), split by L3 hit/miss. Under wait_complete this is the
# admission delay a request pays for its L3 lookup.
# ---------------------------------------------------------------------------
_PREFETCH_READY_HIST = None
def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None:
if not _STAGE_TIMING:
return
global _PREFETCH_READY_HIST
try:
if _PREFETCH_READY_HIST is None:
from prometheus_client import Histogram
_PREFETCH_READY_HIST = Histogram(
"sglang:storage_prefetch_ready_seconds",
"Prefetch in-flight -> usable latency, split by L3 hit/miss",
["dp_rank", "result"],
buckets=(1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0, 3.0, 10.0, 30.0),
)
_PREFETCH_READY_HIST.labels(dp_rank=str(dp_rank), result=result).observe(
seconds
)
except Exception:
# Telemetry must never break the scheduler loop.
pass
# ---------------------------------------------------------------------------
# Scheduler loop span + no-batch accounting (opt-in, same stage-timing gate).
#
# One ``scheduler.loop`` span per scheduler iteration (sampled), with phase
# children (recv / get_next / run_batch / process_result / kv_send). The
# no-batch counters are incremented on *every* loop (sampling only controls
# spans), split by a low-cardinality reason and by whether a forward was in
# flight (forward_overlap). Fully idle loops never emit a span.
#
# Enable with ``SGLANG_TRACE_LOOP_SPAN=1`` (requires SGLANG_BATCH_STAGE_TIMING).
# Knobs:
# SGLANG_TRACE_LOOP_SAMPLE=0.05 head-sample ratio (deterministic per
# mlp_sync_seq, identical on all ranks)
# SGLANG_TRACE_FORWARD_SPAN=0 emit approximate forward.gpu lanes
# SGLANG_TRACE_KV_SEND_SPAN=1 emit the kv_send child (default on
# when the loop span is on)
# ---------------------------------------------------------------------------
from collections import deque as _deque
_LOOP_ENABLED = _STAGE_TIMING
_LOOP_SPAN_ENABLED = _LOOP_ENABLED and os.getenv(
"SGLANG_TRACE_LOOP_SPAN", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_LOOP_SPAN_SAMPLE = float(os.getenv("SGLANG_TRACE_LOOP_SAMPLE", "0.05"))
_FORWARD_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
"SGLANG_TRACE_FORWARD_SPAN", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_KV_SEND_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
"SGLANG_TRACE_KV_SEND_SPAN", "1"
).strip().lower() not in ("", "0", "false", "no", "off")
_NO_BATCH_REASONS = (
"fully_idle",
"waiting_prefetch",
"waiting_bootstrap",
"result_backpressure",
"deferred_input",
"admission_blocked",
"waiting_pd",
"no_ready_other",
)
_LOOP_METRICS = None
_LOOP_TLS = threading.local()
# Monotonic total of completed forward device time (from the DeviceTimer),
# used to derive one loop's gpu_ms without adding a second timer.
_FORWARD_DEVICE_NS_TOTAL = 0
_FORWARD_EVENTS = _deque(maxlen=128) # (completion_wall_ns, elapsed_ns, category)
# Set by dp_attn.prepare_mlp_sync_batch_raw() once per pass.
_LAST_MLP_OUTCOME = ("none", "none")
_LAST_NO_BATCH_REASON = "no_ready_other"
def loop_accounting_enabled() -> bool:
return _LOOP_ENABLED
def loop_span_enabled() -> bool:
return _LOOP_SPAN_ENABLED
def forward_span_enabled() -> bool:
return _FORWARD_SPAN_ENABLED
def kv_send_span_enabled() -> bool:
if not _KV_SEND_SPAN_ENABLED:
return False
lt = getattr(_LOOP_TLS, "current", None)
return lt is not None and lt.span is not None
def loop_sample_decision(seq: int) -> bool:
"""Deterministic, rank-identical loop-span sampling decision."""
if not _LOOP_SPAN_ENABLED or seq <= 0:
return False
if _LOOP_SPAN_SAMPLE >= 1.0:
return True
if _LOOP_SPAN_SAMPLE <= 0.0:
return False
return (seq * 2654435761) % 10000 < int(_LOOP_SPAN_SAMPLE * 10000)
class LoopTimings:
__slots__ = (
"phase",
"t0_ns",
"t0_wall_ns",
"predicted_seq",
"active0",
"has_work0",
"forward_total0",
"recv",
"get_next",
"run_batch",
"process",
"kv_send_start",
"kv_send_end",
"kv_chunks",
"span",
"span_ctx",
"pass_span_child",
"recv_span_child",
"launch_blocked_reason",
"seq",
"no_batch_reason",
"pass_end_wall_ns",
"arrival_est_wall_ns",
"detail",
)
def __init__(
self,
phase: str,
t0_ns: int,
t0_wall_ns: int,
predicted_seq: int,
active0: bool,
has_work0: bool,
forward_total0: int,
):
self.phase = phase
self.t0_ns = t0_ns
self.t0_wall_ns = t0_wall_ns
self.predicted_seq = predicted_seq
self.active0 = bool(active0)
self.has_work0 = bool(has_work0)
self.forward_total0 = forward_total0
self.recv = None
self.get_next = None
self.run_batch = None
self.process = None
self.kv_send_start = None
self.kv_send_end = None
self.kv_chunks = 0
self.span = None
self.span_ctx = None
self.pass_span_child = False
self.recv_span_child = False
self.launch_blocked_reason = ""
self.seq = 0
self.no_batch_reason = ""
self.pass_end_wall_ns = 0
self.arrival_est_wall_ns = 0
# Only populated when SGLANG_TRACE_LOOP_DETAIL=1 and this loop is
# sampled; see the "Loop detail phases" section below.
self.detail = None
def ensure_loop_span(lt: Optional[LoopTimings]) -> None:
"""Materialize a sampled span once the loop is known to have work."""
if (
lt is None
or lt.span is not None
or not loop_sample_decision(lt.predicted_seq)
):
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.loop")
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
except Exception:
lt.span = None
lt.span_ctx = None
def note_launch_blocked(lt: Optional[LoopTimings], reason: str) -> None:
"""Preserve why this loop could not launch before queues are consumed."""
if lt is not None and not lt.launch_blocked_reason:
lt.launch_blocked_reason = reason
def begin_loop(
phase: str = "prefill",
*,
pre_has_work: bool = True,
forward_active=None,
):
"""Start one scheduler-loop accounting record.
``pre_has_work`` must be cheap to compute (queue lengths / last_batch):
fully idle loops are still counted, but never create a span.
``forward_active`` is a callable (usually DeviceTimer.is_active) or a bool.
"""
global _LAST_MLP_OUTCOME
if not _LOOP_ENABLED:
return None
try:
active0 = bool(forward_active() if callable(forward_active) else forward_active)
except Exception:
active0 = False
lt = LoopTimings(
phase=phase,
t0_ns=time.monotonic_ns(),
t0_wall_ns=time.time_ns(),
predicted_seq=_DP_GATHER_SEQ + 1,
active0=active0,
has_work0=bool(pre_has_work),
forward_total0=_FORWARD_DEVICE_NS_TOTAL,
)
_LAST_MLP_OUTCOME = ("none", "none")
_LOOP_TLS.current = lt
if lt.has_work0:
ensure_loop_span(lt)
return lt
def current_loop() -> Optional[LoopTimings]:
return getattr(_LOOP_TLS, "current", None)
def current_loop_context():
lt = getattr(_LOOP_TLS, "current", None)
return lt.span_ctx if lt is not None else None
# ---------------------------------------------------------------------------
# Loop detail phases (SGLANG_TRACE_LOOP_DETAIL=1).
#
# Splits run_batch / process_result into host phases. Gated twice: the module
# switch must be on AND the current loop must be sampled (same deterministic
# decision as the loop span), so detail data never covers an unsampled loop.
#
# Accounting rules (see handoff/prefill-cpu-kv-overlap-design-20261106.md §3.2):
# * single phases -> one child span (``scheduler.loop.<name>``)
# * repeated phases -> summed attributes only (``sglang.<name>_ms`` + count)
# * durations use time.monotonic_ns(); OTLP start/end use time.time_ns()
# * residuals are clamped >= 0 and reported, never folded into a phase
# ---------------------------------------------------------------------------
_DETAIL_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
"SGLANG_TRACE_LOOP_DETAIL", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_DETAIL_TLS = threading.local()
# Chunk-level (Mooncake transfer worker) detail tracing. Independent switch:
# the worker runs on other threads and has no scheduler loop TLS.
_MOONCAKE_DETAIL_ENABLED = os.getenv(
"SGLANG_TRACE_MOONCAKE_DETAIL", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
# Previous loop summary per scheduler thread, used for DP-skew correlation
# (previous loop's phases vs this loop's arrival skew).
_LAST_LOOP_SUMMARY: dict = {}
# Phases that are inherently per-request / per-segment: always attributes,
# even when a loop happens to have exactly one occurrence, so span count never
# grows with n_reqs / n_segments (design §4.2).
_DETAIL_ATTRIBUTE_ONLY_PREFIXES = (
"per_request_result",
"prefix_device_indices",
"kv_state_payload",
"kv_page_indices",
"kv_sender_send",
)
_DETAIL_PREV_KEYS = (
"run_batch_ms",
"process_result_ms",
"copy_done_wait_ms",
"kv_state_payload_ms",
"kv_page_indices_ms",
"kv_sender_send_ms",
)
def loop_detail_enabled() -> bool:
"""True when detail is on and the *current* loop is sampled."""
if not _DETAIL_ENABLED:
return False
lt = getattr(_LOOP_TLS, "current", None)
return lt is not None and loop_sample_decision(lt.predicted_seq)
def mooncake_detail_enabled() -> bool:
return _MOONCAKE_DETAIL_ENABLED
def _detail_state():
"""Detail state of the current loop, only when that loop is sampled."""
if not _DETAIL_ENABLED:
return None
lt = getattr(_LOOP_TLS, "current", None)
if lt is None or not loop_sample_decision(lt.predicted_seq):
return None
if lt.detail is None:
lt.detail = {"phases": {}, "counts": {}, "sums": {}}
return lt.detail
def detail_begin(name: str) -> Optional[int]:
"""Start a single-occurrence phase. Returns a monotonic-ns token."""
if not _DETAIL_ENABLED:
return None
state = _detail_state()
if state is None:
return None
now_ns = time.monotonic_ns()
opens = getattr(_DETAIL_TLS, "open", None)
if opens is None:
opens = {}
_DETAIL_TLS.open = opens
opens[name] = (now_ns, time.time_ns(), _thread_time_ns())
return now_ns
def detail_end(name: str, token: Optional[int] = None) -> None:
"""Close a phase opened by ``detail_begin`` (or started at ``token``)."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
now_ns = time.monotonic_ns()
opens = getattr(_DETAIL_TLS, "open", None) or {}
entry = opens.pop(name, None)
start_wall_ns = entry[1] if entry is not None else None
thread_cpu_ns = None
if entry is not None and len(entry) > 2 and entry[2] is not None:
cpu_now = _thread_time_ns()
if cpu_now is not None:
thread_cpu_ns = max(0, cpu_now - int(entry[2]))
if token is None:
if entry is None:
return
token = entry[0]
if start_wall_ns is None:
start_wall_ns = time.time_ns() - max(0, now_ns - int(token))
_detail_record(
state,
name,
int(token),
now_ns,
start_wall_ns,
time.time_ns(),
thread_cpu_ns=thread_cpu_ns,
)
def detail_add(
name: str,
start_ns: int,
end_ns: int,
thread_cpu_ns: Optional[int] = None,
) -> None:
"""Accumulate one occurrence of a phase (no span per occurrence).
``thread_cpu_ns`` is optional: callers that did not measure CPU time must
leave it as None so reports can tell "not measured" apart from "zero".
"""
if not _DETAIL_ENABLED or start_ns is None or end_ns is None:
return
state = _detail_state()
if state is None:
return
start_ns = int(start_ns)
end_ns = int(end_ns)
if end_ns < start_ns:
end_ns = start_ns
now_wall_ns = time.time_ns()
_detail_record(
state,
name,
start_ns,
end_ns,
now_wall_ns - (end_ns - start_ns),
now_wall_ns,
thread_cpu_ns=thread_cpu_ns,
)
def _detail_record(
state,
name,
start_ns,
end_ns,
start_wall_ns,
end_wall_ns,
thread_cpu_ns: Optional[int] = None,
) -> None:
rec = state["phases"].get(name)
if rec is None:
rec = [end_ns - start_ns, 1, int(start_wall_ns), int(end_wall_ns), 0, 0]
state["phases"][name] = rec
else:
rec[0] += end_ns - start_ns
rec[1] += 1
rec[3] = int(end_wall_ns)
if thread_cpu_ns is not None:
rec[4] += int(thread_cpu_ns)
rec[5] += 1
def thread_cpu_ns() -> Optional[int]:
"""Thread CPU clock, or None when the platform has no such clock.
Callers outside this module (the scheduler's parent span) must use this
wrapper so an unsupported or failing clock degrades to "not measured"
instead of raising on the scheduling path.
"""
try:
return time.thread_time_ns()
except (AttributeError, OSError, ValueError):
return None
# Backwards-compatible private alias for module-local call sites.
_thread_time_ns = thread_cpu_ns
def detail_attr_setdefault(name: str, value) -> None:
"""Set a loop attribute only if this loop has not recorded it yet."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
attrs = state.setdefault("attrs", {})
if name not in attrs:
attrs[name] = value
def detail_attr_sum(name: str, value: int) -> None:
"""Accumulate a numeric loop attribute (e.g. evicted tokens)."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
sums = state.setdefault("sums", {})
sums[name] = int(sums.get(name, 0)) + int(value)
def detail_count(name: str, n: int = 1) -> None:
"""Accumulate a low-cardinality counter for this loop."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
state["counts"][name] = int(state["counts"].get(name, 0)) + int(n)
def detail_attr(name: str, value) -> None:
"""Set a low-cardinality loop attribute (last write wins)."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
state.setdefault("attrs", {})[name] = value
def detail_phase_total_ns(names) -> int:
"""Sum of accumulated ns for the given phases in the current loop.
``names`` are *base* names: a phase recorded as ``kv_sender_send.last``
(per send kind) is included when ``kv_sender_send`` is requested. Phase
names are only generated by this module, so prefix matching cannot pick up
an unrelated phase that merely shares a leading string.
"""
state = _detail_state()
if state is None:
return 0
phases = state.get("phases") or {}
total = 0
for name, rec in phases.items():
if rec is None:
continue
for want in names:
if name == want or name.startswith(want + "."):
total += int(rec[0])
break
return total
class detail_phase:
"""Context manager for one single-occurrence phase (cheap when disabled)."""
__slots__ = ("name", "token")
def __init__(self, name: str):
self.name = name
self.token = None
def __enter__(self):
self.token = detail_begin(self.name)
return self
def __exit__(self, exc_type, exc, tb):
detail_end(self.name, self.token)
return False
# ---------------------------------------------------------------------------
# HiCache flush micro timing (SGLANG_TRACE_BACKUP_DETAIL=1).
#
# One tracer per ``cache_controller.start_writing()`` call (i.e. per
# ``flush_pending_backups``). Sub-phases are host-wall timings on the caller
# thread; a synchronous ``.cpu()`` therefore shows up as a long host phase and
# must not be read as device-transfer time. The tracer keeps per-flush
# resolution by emitting ONE child span of the current loop
# (``scheduler.cache_flush_backup``), because the loop-level detail only keeps
# per-loop sums (design: handoff/prefill-flush-backups-validation-20261107.md).
#
# Disabled by default: every entry point is a single module-level bool check,
# and when off no object, dict or span is created.
# ---------------------------------------------------------------------------
_BACKUP_DETAIL_ENABLED = _DETAIL_ENABLED and os.getenv(
"SGLANG_TRACE_BACKUP_DETAIL", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_FLUSH_TLS = threading.local()
def backup_detail_enabled() -> bool:
"""True when per-flush micro timing is compiled in (``*_DETAIL`` is on)."""
return _BACKUP_DETAIL_ENABLED
def set_flush_origin(origin: Optional[str]) -> Optional[str]:
"""Tag the next flushes with the caller's scheduler context.
Only used to tell ``process_batch_result`` flushes (the current suspect)
from the ones issued by ``check_hicache_events`` / ``write``. Returns the
previous value so the caller can restore it.
"""
if not _BACKUP_DETAIL_ENABLED:
return None
previous = getattr(_FLUSH_TLS, "origin", None)
_FLUSH_TLS.origin = origin
return previous
def current_flush_origin() -> str:
return getattr(_FLUSH_TLS, "origin", None) or "other"
class BackupFlushTracer:
"""Sub-phase timings for one flush; emits a single span on ``end``."""
__slots__ = (
"origin",
"start_ns",
"loop_seq",
"loop_ctx",
"phases",
"attrs",
"counts",
"_open",
"_prev",
)
def __init__(self, origin: str):
self.origin = origin
# Captured on the scheduler thread: the flushed span may be emitted by
# the async submit worker, which has no scheduler loop TLS.
self.loop_seq = current_loop_seq()
self.loop_ctx = current_loop_context_for_spans()
self.start_ns = time.monotonic_ns()
self.phases: dict[str, int] = {}
self.attrs: dict[str, Any] = {}
self.counts: dict[str, int] = {}
self._open: dict[str, int] = {}
self._prev = None
# -- phase helpers -----------------------------------------------------
def begin(self, name: str) -> int:
start = time.monotonic_ns()
self._open[name] = start
return start
def end(self, name: str, start_ns: Optional[int] = None) -> None:
start = self._open.pop(name, None)
if start is None:
start = start_ns
if start is None:
return
self.add(name, start)
def add(self, name: str, start_ns: int) -> None:
self.record(name, int(start_ns), time.monotonic_ns())
def record(self, name: str, start_ns: int, end_ns: int) -> None:
self.phases[name] = self.phases.get(name, 0) + max(
0, int(end_ns) - int(start_ns)
)
def set(self, key: str, value: Any) -> None:
self.attrs[key] = value
def inc(self, key: str, n: int = 1) -> None:
self.counts[key] = self.counts.get(key, 0) + int(n)
def total_ns(self) -> int:
return max(0, time.monotonic_ns() - self.start_ns)
# -- emission ----------------------------------------------------------
def end_and_emit(self) -> None:
now_ns = time.monotonic_ns()
total_ns = max(0, now_ns - self.start_ns)
attrs = {
"sglang.flush_origin": self.origin,
"sglang.flush_total_ms": total_ns / 1e6,
"sglang.loop_seq": int(self.loop_seq or 0),
}
for name, ns in self.phases.items():
attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6
for name, value in self.counts.items():
attrs["sglang.flush_" + name] = int(value)
attrs.update(self.attrs)
# OTel rejects None; keep a single malformed dimension from dropping the
# whole span.
attrs = {k: v for k, v in attrs.items() if v is not None}
try:
from opentelemetry import trace as otel_trace
ctx = self.loop_ctx
start_wall_ns = time.time_ns() - total_ns
span = otel_trace.get_tracer("sglang.cache.flush").start_span(
"scheduler.cache_flush_backup",
context=ctx,
start_time=int(start_wall_ns),
attributes=attrs,
)
span.end(end_time=int(time.time_ns()))
except Exception:
# Telemetry must never break the cache controller.
pass
class backup_phase:
"""Context manager for one sub-phase of the active flush (no-op when off)."""
__slots__ = ("name", "tracer", "start_ns")
def __init__(self, name: str):
self.name = name
self.tracer = None
self.start_ns = None
def __enter__(self):
if not _BACKUP_DETAIL_ENABLED:
return self
self.tracer = getattr(_FLUSH_TLS, "tracer", None)
if self.tracer is not None:
self.start_ns = self.tracer.begin(self.name)
return self
def __exit__(self, exc_type, exc, tb):
if self.tracer is not None:
self.tracer.end(self.name, self.start_ns)
return False
def backup_flush_begin(origin: Optional[str] = None) -> Optional[BackupFlushTracer]:
"""Start tracing one flush; returns None when disabled / not sampled."""
if not _BACKUP_DETAIL_ENABLED:
return None
if not loop_detail_enabled():
return None
tracer = BackupFlushTracer(origin or current_flush_origin())
tracer._prev = getattr(_FLUSH_TLS, "tracer", None)
_FLUSH_TLS.tracer = tracer
return tracer
def backup_flush_end(tracer: Optional[BackupFlushTracer]) -> None:
if tracer is None:
return
if getattr(_FLUSH_TLS, "tracer", None) is tracer:
_FLUSH_TLS.tracer = tracer._prev
try:
tracer.end_and_emit()
except Exception:
pass
def current_flush_tracer() -> Optional[BackupFlushTracer]:
if not _BACKUP_DETAIL_ENABLED:
return None
return getattr(_FLUSH_TLS, "tracer", None)
def backup_flush_attach(tracer: Optional[BackupFlushTracer]):
"""Bind ``tracer`` to this thread so sub-phases land on the same span.
Used by the async submit worker, which executes phases on a different
thread than the one that created the tracer. Returns the previous binding.
"""
if not _BACKUP_DETAIL_ENABLED:
return None
previous = getattr(_FLUSH_TLS, "tracer", None)
_FLUSH_TLS.tracer = tracer
return previous
def backup_flush_detach(previous) -> None:
if not _BACKUP_DETAIL_ENABLED:
return
_FLUSH_TLS.tracer = previous
def flush_note(key: str, value: Any) -> None:
"""Attach a dimension to the active flush (no-op when disabled)."""
if not _BACKUP_DETAIL_ENABLED:
return
tracer = getattr(_FLUSH_TLS, "tracer", None)
if tracer is not None:
tracer.set(key, value)
def flush_count(key: str, n: int = 1) -> None:
"""Accumulate a counter on the active flush (no-op when disabled)."""
if not _BACKUP_DETAIL_ENABLED:
return
tracer = getattr(_FLUSH_TLS, "tracer", None)
if tracer is not None:
tracer.inc(key, n)
def flush_mark() -> Optional[int]:
"""Timestamp for a manual phase (None when no flush is being traced)."""
if not _BACKUP_DETAIL_ENABLED:
return None
if getattr(_FLUSH_TLS, "tracer", None) is None:
return None
return time.monotonic_ns()
def flush_phase_since(name: str, start_ns: Optional[int]) -> None:
"""Record ``name`` from ``start_ns`` to now (no-op when untraced)."""
if start_ns is None:
return
tracer = getattr(_FLUSH_TLS, "tracer", None)
if tracer is not None:
tracer.add(name, start_ns)
def current_loop_seq() -> int:
"""Loop sequence for cross-record correlation (0 when unavailable)."""
lt = getattr(_LOOP_TLS, "current", None)
if lt is None:
return 0
return int(lt.seq or lt.predicted_seq or 0)
def current_loop_context_for_spans():
"""OTel context of the current loop span (None if unsampled)."""
lt = getattr(_LOOP_TLS, "current", None)
return lt.span_ctx if lt is not None else None
def _chunk_parent_context(chunk):
trace_ctx = getattr(chunk, "trace_ctx", None)
return getattr(trace_ctx, "root_span_context", None)
def split_queue_wait(
queue_wait_accum_ns: int, last_enqueue_mono_ns: int, dequeue_mono_ns: int
) -> tuple:
"""Return ``(attempt_queue_wait_ns, total_queue_wait_ns)``.
The attempt value matches the last_enqueue->dequeue wall window used by the
``mooncake.queue_wait`` child span; the total value additionally includes
the queue waits of earlier deferred attempts (design §5.3 metric).
Keeping them separate prevents a span whose wall length disagrees with its
``wait_ms`` attribute (review finding).
"""
attempt = max(0, int(dequeue_mono_ns) - int(last_enqueue_mono_ns))
return attempt, max(0, int(queue_wait_accum_ns)) + attempt
class RoomDetailState:
"""Thread-safe per-room detail bookkeeping (scheduler vs worker threads).
The scheduler aborts/retracts a room while the transfer worker may still be
finishing an attempt. A reset therefore records a cancellation mark, and
the worker's join update re-checks under the same lock before writing, so a
late worker can never resurrect stats after abort (review finding).
"""
# Bounded: a marker only needs to survive until the in-flight worker
# attempt finishes (milliseconds). The cap prevents unbounded growth if an
# aborted room never gets a worker teardown.
_MAX_CANCELLED = 8192
def __init__(self):
self._lock = threading.Lock()
self._join: dict = {}
self._seq: dict = {}
self._first: dict = {}
self._cancelled: dict = {}
def _mark_cancelled(self, room: int) -> None:
self._cancelled[int(room)] = time.monotonic_ns()
while len(self._cancelled) > self._MAX_CANCELLED:
self._cancelled.pop(next(iter(self._cancelled)))
def begin_chunk(self, room: int, now_mono_ns: int, now_wall_ns: int):
"""Stamp a real enqueue. Returns ``(chunk_seq, first_enqueue_wall_ns)``.
A new chunk clears a previous cancellation mark: it is the start of a
fresh flow (a retry may land on the same room id).
"""
with self._lock:
self._cancelled.pop(int(room), None)
seq = int(self._seq.get(int(room), 0))
self._seq[int(room)] = seq + 1
first = self._first.get(int(room))
if first is None:
first = (int(now_mono_ns), int(now_wall_ns))
self._first[int(room)] = first
return seq, int(first[1])
def record_join(
self,
room: int,
*,
status: str,
room_failed: bool,
now_wall_ns: int,
queue_wait_total_ns: int,
worker_busy_ns: int,
enqueue_wall_ns: int,
is_last_chunk: bool,
chunk_seq: int,
) -> bool:
"""Update the per-room join summary; False when the write was refused."""
if status != "ok":
return False
with self._lock:
if room_failed or int(room) in self._cancelled:
return False
stats = self._join.get(int(room))
if stats is None:
stats = {"chunk_count": 0}
self._join[int(room)] = stats
stats["chunk_count"] = int(stats.get("chunk_count", 0)) + 1
first = self._first.get(int(room))
if first is not None:
stats["first_enqueue_wall_ns"] = int(first[1])
if is_last_chunk:
stats["last_chunk_enqueue_wall_ns"] = int(enqueue_wall_ns)
stats["last_chunk_complete_wall_ns"] = int(now_wall_ns)
stats["last_chunk_queue_wait_ms"] = queue_wait_total_ns / 1e6
stats["last_chunk_worker_ms"] = worker_busy_ns / 1e6
stats["last_chunk_chunk_seq"] = int(chunk_seq)
return True
def reset(self, room: int) -> None:
with self._lock:
self._join.pop(int(room), None)
self._seq.pop(int(room), None)
self._first.pop(int(room), None)
self._mark_cancelled(room)
def teardown(self, room: int, keep_join: bool) -> None:
with self._lock:
if not keep_join:
self._join.pop(int(room), None)
self._seq.pop(int(room), None)
self._first.pop(int(room), None)
self._cancelled.pop(int(room), None)
def pop_join(self, room: int):
with self._lock:
return self._join.pop(int(room), None)
def note_chunk_requeue(chunk, now_mono_ns: int, now_wall_ns: int) -> None:
"""Accumulate only the *queue* part of a finished attempt before re-enqueue.
The previous implementation added ``requeue - last_enqueue``, which also
contains the worker's own execution and the staging watermark wait, thus
inflating queue wait on every defer (review finding). Only the interval
from the last enqueue to this attempt's dequeue is queue wait.
"""
if not _MOONCAKE_DETAIL_ENABLED or not getattr(chunk, "observe", False):
return
last_enqueue = int(getattr(chunk, "last_enqueue_mono_ns", 0) or 0)
dequeue = int(getattr(chunk, "dequeue_mono_ns", 0) or 0)
if last_enqueue and dequeue:
chunk.queue_wait_accum_ns = int(
getattr(chunk, "queue_wait_accum_ns", 0) or 0
) + max(0, dequeue - last_enqueue)
chunk.last_enqueue_mono_ns = int(now_mono_ns)
chunk.last_enqueue_wall_ns = int(now_wall_ns)
chunk.dequeue_mono_ns = 0
chunk.requeue_count = int(getattr(chunk, "requeue_count", 0) or 0) + 1
def emit_mooncake_chunk(
*,
chunk,
status: str,
queue_wait_ns: int,
queue_wait_total_ns: int,
source_wait_ns: int,
indices_wait_ns: int,
kv_transfer_ns: int,
state_transfer_ns: int,
aux_transfer_ns: int,
attempt_wall_ns: int,
dequeue_wall_ns: int,
end_wall_ns: int,
n_destinations: int,
source_wait_wall=None,
indices_wait_wall=None,
dp_rank: int = -1,
tp_rank: int = -1,
) -> None:
"""One span per transfer-worker attempt, attributed by room/chunk_seq.
``worker_busy_ms`` follows the design definition: attempt wall time minus
the source/indices event waits (those are blocking waits, not worker work).
``attempt_wall_ms`` is the raw dequeue->end time.
Wait child spans use the real event timestamps captured by the worker.
A wait without a recorded window is attributes-only; no synthetic
``end - duration`` placement is drawn (review finding).
"""
if not _MOONCAKE_DETAIL_ENABLED:
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.mooncake.chunk")
# Attempt-scoped start: the last enqueue of this attempt, not the
# first enqueue of the chunk, so deferred retries do not overlap.
start_wall_ns = int(
getattr(chunk, "last_enqueue_wall_ns", 0)
or chunk.enqueue_wall_ns
or dequeue_wall_ns
) or end_wall_ns
worker_busy_ns = max(
0, int(attempt_wall_ns) - int(source_wait_ns) - int(indices_wait_ns)
)
residual_ns = max(
0,
worker_busy_ns
- int(kv_transfer_ns)
- int(state_transfer_ns)
- int(aux_transfer_ns),
)
span = tracer.start_span(
"mooncake.kv_chunk",
context=_chunk_parent_context(chunk),
start_time=start_wall_ns,
attributes={
"sglang.bootstrap_room": int(chunk.room),
"sglang.chunk_seq": int(chunk.chunk_seq),
"sglang.loop_seq": int(chunk.loop_seq),
"sglang.send_kind": str(chunk.send_kind or "unknown"),
"sglang.is_last_chunk": bool(chunk.is_last_chunk),
"sglang.queue_id": int(chunk.queue_id),
"sglang.requeue_count": int(chunk.requeue_count),
"sglang.num_kv_tokens": int(chunk.num_kv_tokens or 0),
"sglang.num_pages": int(chunk.num_pages),
"sglang.approx_bytes": int(chunk.approx_bytes),
"sglang.n_destinations": int(n_destinations),
"sglang.chunk_status": str(status),
"sglang.dp_rank": int(dp_rank),
"sglang.tp_rank": int(tp_rank),
# Attempt window (matches the child span) vs cumulative wait
# across deferred attempts (design §5.3 metric).
"sglang.queue_wait_ms": queue_wait_ns / 1e6,
"sglang.queue_wait_total_ms": queue_wait_total_ns / 1e6,
"sglang.source_wait_ms": source_wait_ns / 1e6,
"sglang.indices_wait_ms": indices_wait_ns / 1e6,
"sglang.kv_transfer_ms": kv_transfer_ns / 1e6,
"sglang.state_transfer_ms": state_transfer_ns / 1e6,
"sglang.aux_transfer_ms": aux_transfer_ns / 1e6,
"sglang.attempt_wall_ms": attempt_wall_ns / 1e6,
"sglang.worker_busy_ms": worker_busy_ns / 1e6,
"sglang.worker_residual_ms": residual_ns / 1e6,
},
)
span_ctx = otel_trace.set_span_in_context(span)
def _child(name, start_ns, stop_ns, wait_ns):
start_ns = int(start_ns or 0)
stop_ns = int(stop_ns or 0)
if start_ns <= 0 or stop_ns <= start_ns:
return
try:
tracer.start_span(
name,
context=span_ctx,
start_time=start_ns,
attributes={"sglang.wait_ms": max(0, int(wait_ns)) / 1e6},
).end(end_time=stop_ns)
except Exception:
pass
# Real event windows only; queue wait ends at dequeue. Zero-wait
# attempts (immediate dequeue) get no span.
if queue_wait_ns > 0:
_child(
"mooncake.queue_wait",
getattr(chunk, "last_enqueue_wall_ns", 0),
dequeue_wall_ns,
queue_wait_ns,
)
if source_wait_wall is not None:
_child(
"mooncake.wait_source",
source_wait_wall[0],
source_wait_wall[1],
source_wait_ns,
)
if indices_wait_wall is not None:
_child(
"mooncake.wait_indices",
indices_wait_wall[0],
indices_wait_wall[1],
indices_wait_ns,
)
span.end(end_time=int(end_wall_ns))
except Exception:
# Telemetry must never break the transfer worker.
pass
def emit_request_kv_join(
*,
req,
stats: dict,
dp_rank: int = -1,
completion_wall_ns: Optional[int] = None,
) -> None:
"""Request-level join summary: last-chunk exposure + completion poll."""
if not _MOONCAKE_DETAIL_ENABLED or not stats:
return
try:
from opentelemetry import trace as otel_trace
from opentelemetry.trace import Link
now_wall_ns = int(completion_wall_ns or time.time_ns())
enqueue_wall_ns = int(stats.get("last_chunk_enqueue_wall_ns", 0) or 0)
complete_wall_ns = int(stats.get("last_chunk_complete_wall_ns", 0) or 0)
final_exposed_ns = (
max(0, complete_wall_ns - enqueue_wall_ns)
if enqueue_wall_ns and complete_wall_ns
else 0
)
completion_poll_ns = (
max(0, now_wall_ns - complete_wall_ns) if complete_wall_ns else 0
)
# First enqueue comes from the manager's real queue.put() event
# (wall clock), not from pre-send preparation (review finding).
first_enqueue_wall_ns = int(stats.get("first_enqueue_wall_ns", 0) or 0)
first_enqueue_to_completion_ms = None
if first_enqueue_wall_ns:
first_enqueue_to_completion_ms = max(
0.0, (now_wall_ns - first_enqueue_wall_ns) / 1e6
)
req_ctx = _request_span_context(req)
attrs = {
"sglang.bootstrap_room": int(getattr(req, "bootstrap_room", 0) or 0),
"sglang.rid": str(getattr(req, "rid", "")),
"sglang.dp_rank": int(dp_rank),
"sglang.chunk_count": int(stats.get("chunk_count", 0)),
"sglang.last_chunk_queue_wait_ms": float(
stats.get("last_chunk_queue_wait_ms", 0.0)
),
"sglang.last_chunk_worker_ms": float(
stats.get("last_chunk_worker_ms", 0.0)
),
"sglang.final_exposed_ms": final_exposed_ns / 1e6,
"sglang.completion_poll_ms": completion_poll_ns / 1e6,
# Per-rank request join. rank_join_skew_ms (max-min over TP/DP
# ranks) is computed offline by joining this span by room.
"sglang.request_join_ms": (final_exposed_ns + completion_poll_ns) / 1e6,
}
if first_enqueue_to_completion_ms is not None:
attrs["sglang.first_enqueue_to_completion_ms"] = (
first_enqueue_to_completion_ms
)
tracer = otel_trace.get_tracer("sglang.kv.request")
tracer.start_span(
"kv.request_join",
# Cover the whole request transfer when first enqueue is known.
start_time=first_enqueue_wall_ns or enqueue_wall_ns or now_wall_ns,
links=[Link(req_ctx)] if req_ctx is not None else [],
attributes=attrs,
).end(end_time=now_wall_ns)
except Exception:
pass
def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None:
"""Record the recv_requests phase inside the current loop (if any)."""
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
lt.recv = (int(start_wall_ns), int(end_wall_ns), dict(parts))
def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
"""Record get_next timings for the current loop (called from end_pass)."""
lt = getattr(_LOOP_TLS, "current", None)
if lt is None:
return
end_wall_ns = time.time_ns()
start_wall_ns = end_wall_ns - int(total_s * 1e9)
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
other = max(0.0, total_s - measured)
lt.get_next = (
start_wall_ns,
end_wall_ns,
{
"sglang.get_next_ms": total_s * 1e3,
"sglang.match_ms": t.match * 1e3,
"sglang.adder_ms": t.adder * 1e3,
"sglang.hicache_ms": t.hicache * 1e3,
"sglang.bootstrap_ms": t.bootstrap * 1e3,
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
"sglang.other_ms": other * 1e3,
"sglang.candidates": int(t.candidates),
"sglang.skipped_prefetch": int(t.skipped_prefetch),
"sglang.admitted": int(t.admitted),
"sglang.bootstrap_candidates": int(t.bootstrap_candidates),
"sglang.pending_bootstrap": int(t.pending_bootstrap),
"sglang.resumed_chunked": int(bool(t.had_chunked)),
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
"sglang.mlp_sync_seq": int(t.seq),
},
)
lt.seq = int(t.seq)
lt.pass_end_wall_ns = end_wall_ns
# Estimated DP-sync arrival: the pass ended, then mlp_sync waited.
# See design §1.2: arrival_est = pass_end - mlp_sync.
lt.arrival_est_wall_ns = end_wall_ns - int(t.mlp_sync * 1e9)
def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
attrs = dict(attrs or {})
attrs.setdefault(
"sglang.run_batch_ms", max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6
)
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), attrs)
def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
attrs = dict(attrs or {})
attrs.setdefault(
"sglang.process_result_ms",
max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6,
)
lt.process = (int(start_wall_ns), int(end_wall_ns), attrs)
def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
lt = getattr(_LOOP_TLS, "current", None)
if lt is None:
return
lt.kv_chunks += 1
if lt.kv_send_start is None or start_wall_ns < lt.kv_send_start:
lt.kv_send_start = int(start_wall_ns)
if lt.kv_send_end is None or end_wall_ns > lt.kv_send_end:
lt.kv_send_end = int(end_wall_ns)
def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None:
"""Set by dp_attn.prepare_mlp_sync_batch_raw once per pass."""
if not _LOOP_ENABLED:
return
global _LAST_MLP_OUTCOME
_LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome))
if local_kind != "none" or global_outcome != "none":
# DP idle work is only known after the sync. Activate the span before
# end_pass so the get_next child can share the loop trace.
ensure_loop_span(getattr(_LOOP_TLS, "current", None))
def note_forward_device(t: float, category: str = "", **kwargs) -> None:
"""DeviceTimer reporter: accumulate completed forward device time."""
global _FORWARD_DEVICE_NS_TOTAL
try:
ns = int(float(t) * 1e9)
except Exception:
return
_FORWARD_DEVICE_NS_TOTAL += ns
if _FORWARD_SPAN_ENABLED:
_FORWARD_EVENTS.append((time.time_ns(), ns, str(category)))
def install_forward_reporter(scheduler) -> None:
"""Attach the forward device-time reporter to the existing DeviceTimer.
No CUDA event is added here: SGLANG_ENABLE_METRICS_DEVICE_TIMER already
runs the timer; this only reads its completed intervals.
"""
if not _LOOP_ENABLED:
return
try:
reporter = getattr(scheduler, "metrics_reporter", None)
timer = getattr(reporter, "forward_pass_device_timer", None)
if timer is not None:
timer.add_reporter(note_forward_device)
except Exception:
# Telemetry must never break scheduler init.
pass
def _ensure_loop_metrics():
global _LOOP_METRICS
if _LOOP_METRICS is None:
from prometheus_client import Counter
_LOOP_METRICS = {
"no_batch_seconds": Counter(
"sglang:scheduler_no_batch_seconds_total",
"Wall time of scheduler loops that ended without a runnable batch, "
"by reason and forward_overlap",
["dp_rank", "reason", "forward_overlap"],
),
"no_batch_total": Counter(
"sglang:scheduler_no_batch_total",
"Count of scheduler loops that ended without a runnable batch, "
"by reason and forward_overlap",
["dp_rank", "reason", "forward_overlap"],
),
"lockstep_seconds": Counter(
"sglang:scheduler_dp_lockstep_idle_seconds_total",
"Wall time of DP-lockstep idle-forward loops (this rank had no "
"local batch but the DP group required an idle forward)",
["dp_rank", "forward_overlap"],
),
}
return _LOOP_METRICS
def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str:
"""Low-cardinality no-batch reason, in the design's priority order.
``pass_snapshot`` is the get_next attribute dict (candidates / skipped /
admitted). It is None when the loop skipped batch preparation entirely.
"""
if state.get("result_queue", 0) >= state.get("result_limit", 2):
return "result_backpressure"
if state.get("deferred", 0) > 0 or state.get("paused"):
return "deferred_input"
waiting = int(state.get("waiting", 0) or 0)
if waiting > 0 and pass_snapshot is not None:
bootstrap_candidates = int(
pass_snapshot.get("sglang.bootstrap_candidates", 0) or 0
)
pending_bootstrap = int(
pass_snapshot.get("sglang.pending_bootstrap", 0) or 0
)
if bootstrap_candidates > 0 and pending_bootstrap == bootstrap_candidates:
return "waiting_bootstrap"
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
if candidates > 0 and skipped == candidates:
return "waiting_prefetch"
if admitted == 0:
return "admission_blocked"
inflight = int(state.get("inflight", 0) or 0)
bootstrap = int(state.get("bootstrap", 0) or 0)
if waiting == 0 and bootstrap:
return "waiting_bootstrap"
if waiting == 0 and inflight:
return "waiting_pd"
if (
waiting == 0
and inflight == 0
and bootstrap == 0
and int(state.get("result_queue", 0) or 0) == 0
and int(state.get("deferred", 0) or 0) == 0
and not state.get("waiting_for_result")
):
return "fully_idle"
if state.get("skip_batch_creation") and pass_snapshot is None:
# A failed preparation defers retries; keep the last classified reason.
return _LAST_NO_BATCH_REASON
return "no_ready_other"
def _batch_attrs(batch) -> dict:
if batch is None:
return {
"sglang.n_reqs": 0,
"sglang.n_new": 0,
"sglang.n_cached": 0,
"sglang.chunked": False,
}
try:
n_reqs = int(batch.batch_size())
except Exception:
n_reqs = len(getattr(batch, "reqs", ()) or ())
try:
n_new = int(getattr(batch, "extend_num_tokens", 0) or 0)
except Exception:
n_new = 0
try:
n_cached = int(sum(getattr(batch, "prefix_lens", ()) or ()))
except Exception:
n_cached = 0
try:
chunked = getattr(batch, "chunked_req", None) is not None
except Exception:
chunked = False
return {
"sglang.n_reqs": n_reqs,
"sglang.n_new": n_new,
"sglang.n_cached": n_cached,
"sglang.chunked": bool(chunked),
}
_LAST_LOOP_SEQ = 0
_LAST_LOOP_HAS_DETAIL = False
def _total_ms(ms: dict, base: str) -> float:
return sum(
value for name, value in ms.items() if name == base or name.startswith(base + ".")
)
def _update_prev_state(lt: LoopTimings, loop_seq: int):
"""Capture the previous loop summary and store this loop for the next one.
Returns ``(prev_seq, prev_has_detail, prev_summary, consecutive)``. Only a
loop that actually recorded detail becomes a valid predecessor, so a
sampled loop after an unsampled one is marked non-consecutive instead of
silently publishing a stale window (design §4.5).
"""
global _LAST_LOOP_SEQ, _LAST_LOOP_HAS_DETAIL
prev_seq = int(_LAST_LOOP_SEQ)
prev_has = bool(_LAST_LOOP_HAS_DETAIL)
prev_summary = dict(_LAST_LOOP_SUMMARY)
consecutive = prev_has and prev_seq == loop_seq - 1
ms = {
name: rec[0] / 1e6
for name, rec in ((lt.detail or {}).get("phases") or {}).items()
if rec
}
_LAST_LOOP_SUMMARY.clear()
if lt.detail is not None:
_LAST_LOOP_SUMMARY.update(
{
"run_batch_ms": float(
lt.run_batch[2].get("sglang.run_batch_ms", 0.0)
)
if lt.run_batch is not None
else 0.0,
"process_result_ms": float(
lt.process[2].get("sglang.process_result_ms", 0.0)
)
if lt.process is not None
else 0.0,
"copy_done_wait_ms": _total_ms(ms, "copy_done_wait"),
"kv_state_payload_ms": _total_ms(ms, "kv_state_payload"),
"kv_page_indices_ms": _total_ms(ms, "kv_page_indices"),
"kv_sender_send_ms": _total_ms(ms, "kv_sender_send"),
}
)
_LAST_LOOP_SEQ = int(loop_seq)
_LAST_LOOP_HAS_DETAIL = lt.detail is not None
return prev_seq, prev_has, prev_summary, consecutive
def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None:
"""Expose detail phases on the loop span and its child spans.
Single-occurrence phases become children (``scheduler.loop.<name>``);
repeated phases only contribute summed attributes, so span count never
grows with n_reqs / n_segments.
"""
detail = lt.detail or {}
phases = detail.get("phases") or {}
ms = {}
accounting_anomalies = 0
for name, rec in phases.items():
try:
total_ns, count, start_wall, end_wall = rec[:4]
except Exception:
continue
key = "sglang." + name.replace(".", "_") + "_ms"
ms[name] = total_ns / 1e6
attrs[key] = ms[name]
cpu_ns, cpu_count = (rec[4], rec[5]) if len(rec) > 5 else (0, 0)
prefix = "sglang." + name.replace(".", "_")
non_cpu_ms = None
if cpu_count:
attrs[prefix + "_thread_cpu_ms"] = cpu_ns / 1e6
attrs[prefix + "_thread_cpu_measured_count"] = int(cpu_count)
# Only a fully measured phase may report the wall - CPU remainder;
# a partial measurement would silently attribute unmeasured time to
# "non-CPU".
if cpu_count == count:
non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6)
attrs[prefix + "_non_cpu_ms"] = non_cpu_ms
# Only a deficit beyond clock granularity is an accounting
# anomaly; a few hundred ns of skew on a 2 ms phase is noise.
deficit_ns = cpu_ns - total_ns
if deficit_ns > max(100_000, total_ns // 100):
attrs[prefix + "_non_cpu_raw_ms"] = -deficit_ns / 1e6
attrs[prefix + "_accounting_anomaly"] = True
accounting_anomalies += 1
if count > 1:
attrs[prefix + "_count"] = int(count)
continue
if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES:
continue
child_attrs = {
"sglang.dp_rank": int(dp_rank),
"sglang.phase_ms": total_ns / 1e6,
}
if cpu_count:
child_attrs["sglang.phase_thread_cpu_ms"] = cpu_ns / 1e6
child_attrs["sglang.phase_cpu_measured_count"] = int(cpu_count)
if non_cpu_ms is not None:
child_attrs["sglang.phase_non_cpu_ms"] = non_cpu_ms
child(
"scheduler.loop." + name.replace(".", "_"),
start_wall,
end_wall,
child_attrs,
)
if accounting_anomalies:
attrs["sglang.accounting_anomaly_count"] = int(accounting_anomalies)
for name, value in (detail.get("counts") or {}).items():
attrs["sglang." + name.replace(".", "_") + "_count"] = int(value)
for name, value in (detail.get("sums") or {}).items():
attrs["sglang." + name.replace(".", "_")] = int(value)
for name, value in (detail.get("attrs") or {}).items():
attrs["sglang." + name.replace(".", "_")] = value
# Accounting residuals (design §3.2): never negative, never folded in.
def _total(*bases):
total = 0.0
for base in bases:
for name, value in ms.items():
if name == base or name.startswith(base + "."):
total += value
return total
# Roll per-kind variants (kv_page_indices.last, ...) up to one base
# attribute so analysis can use the total without knowing the kinds.
bases = {name.split(".", 1)[0] for name in ms}
for base in bases:
if base not in ms:
attrs["sglang." + base + "_ms"] = _total(base)
if lt.run_batch is not None:
run_ms = float(lt.run_batch[2].get("sglang.run_batch_ms", 0.0))
parts = _total(
"early_send_host",
"resolve_inputs",
"model_forward_host",
"copy_result_submit",
)
attrs["sglang.run_residual_ms"] = max(0.0, run_ms - parts)
if lt.process is not None:
proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0))
parts = _total(
# Outer phases of process_batch_result (scheduler.py).
"flush_trace",
"load_snapshot",
"flush_backups",
"step_counters",
"metrics_report",
# Inner disagg-prefill phases (prefill.py).
"copy_done_wait",
"auxiliary_and_logprob",
"per_request_result",
"report_stats",
)
if "sglang.process_kv_ms" in lt.process[2]:
# KV phases charged to this call only (sends issued later in the
# loop, e.g. pending-bootstrap, are not part of process_result).
parts += float(lt.process[2]["sglang.process_kv_ms"])
else:
parts += _total(
"kv_state_payload", "kv_page_indices", "kv_sender_send"
)
attrs["sglang.process_residual_ms"] = max(0.0, proc_ms - parts)
def end_loop(
lt: Optional[LoopTimings],
*,
dp_rank: int,
has_batch: bool,
batch=None,
state: Optional[dict] = None,
) -> None:
"""Close one loop: counters (always), loop span + phases (if sampled)."""
if lt is None:
return
_LOOP_TLS.current = None
state = state or {}
try:
now_ns = time.monotonic_ns()
now_wall_ns = time.time_ns()
loop_s = max(0.0, (now_ns - lt.t0_ns) / 1e9)
gpu_ns = max(0, _FORWARD_DEVICE_NS_TOTAL - lt.forward_total0)
active1 = bool(state.get("forward_active", False))
overlap = (
"full"
if lt.active0 and active1
else "partial"
if (lt.active0 or active1)
else "none"
)
local_kind, global_outcome = _LAST_MLP_OUTCOME
pass_snapshot = lt.get_next[2] if lt.get_next is not None else None
pending = bool(
state.get("waiting", 0)
or state.get("result_queue", 0)
or state.get("inflight", 0)
or state.get("bootstrap", 0)
or state.get("deferred", 0)
or state.get("paused")
or state.get("waiting_for_result")
)
reason = ""
try:
if local_kind == "none" and global_outcome == "idle":
# The DP sync replaces local None with a real idle batch, so this
# accounting must not depend on has_batch being False.
_ensure_loop_metrics()["lockstep_seconds"].labels(
dp_rank=str(dp_rank), forward_overlap=overlap
).inc(loop_s)
elif not has_batch:
reason = lt.launch_blocked_reason or _classify_no_batch(
state, pass_snapshot
)
global _LAST_NO_BATCH_REASON
_LAST_NO_BATCH_REASON = reason
metrics = _ensure_loop_metrics()
metrics["no_batch_seconds"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc(loop_s)
metrics["no_batch_total"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc()
except Exception:
# Telemetry must never break the scheduler loop.
pass
lt.no_batch_reason = reason
# Update the prev-loop chain on every loop (sampled or not) so a
# sampled loop after an unsampled one is reported as non-consecutive
# instead of silently carrying a stale window (review finding).
loop_seq_cur = int(lt.seq or lt.predicted_seq)
prev_state = None
if _DETAIL_ENABLED:
prev_state = _update_prev_state(lt, loop_seq_cur)
if lt.span is None and (has_batch or pending):
# Covers work discovered after begin_loop, including first input and
# deferred-only loops. DP outcomes activate earlier for pass parenting.
ensure_loop_span(lt)
if lt.span is None:
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.loop")
attrs = {
"sglang.phase": lt.phase,
"sglang.dp_rank": int(dp_rank),
"sglang.loop_seq": int(lt.seq or lt.predicted_seq),
"sglang.loop_ms": loop_s * 1e3,
"sglang.gpu_ms": gpu_ns / 1e6,
"sglang.gpu_idle_ms": max(0.0, loop_s * 1e3 - gpu_ns / 1e6),
"sglang.has_batch": bool(has_batch),
"sglang.engine_has_pending_work": pending,
"sglang.local_batch_kind": local_kind,
"sglang.global_outcome": global_outcome,
"sglang.no_batch_reason": reason,
"sglang.forward_overlap": overlap,
}
attrs.update(_batch_attrs(batch))
def _child(name, start_ns, end_ns, child_attrs):
try:
tracer.start_span(
name,
context=lt.span_ctx,
start_time=int(start_ns),
attributes=child_attrs,
).end(end_time=int(end_ns))
except Exception:
pass
if lt.detail is not None:
_emit_loop_detail(lt, attrs, _child, int(dp_rank))
if _DETAIL_ENABLED:
if lt.pass_end_wall_ns:
# DP-skew join fields (design §4.5); independent of phases.
attrs["sglang.pass_end_wall_ns"] = int(lt.pass_end_wall_ns)
attrs["sglang.arrival_est_wall_ns"] = int(lt.arrival_est_wall_ns)
if prev_state is not None:
prev_seq, prev_has, prev_summary, consecutive = prev_state
attrs["sglang.prev_loop_seq"] = int(prev_seq)
if consecutive:
for key in _DETAIL_PREV_KEYS:
if key in prev_summary:
attrs["sglang.prev_" + key] = prev_summary[key]
elif prev_seq:
# The previous loop was unsampled or had no detail;
# its phases must not be treated as seq-1 values.
attrs["sglang.prev_detail_missing"] = True
lt.span.set_attributes(attrs)
if lt.recv is not None and not lt.recv_span_child:
start, end, parts = lt.recv
child = {"sglang.dp_rank": int(dp_rank)}
child.update(parts)
_child("scheduler.loop.recv", start, end, child)
if lt.get_next is not None and not lt.pass_span_child:
start, end, ng_attrs = lt.get_next
_child("scheduler.loop.get_next", start, end, ng_attrs)
if lt.run_batch is not None:
start, end, rb_attrs = lt.run_batch
_child("scheduler.loop.run_batch", start, end, rb_attrs)
if lt.process is not None:
start, end, pr_attrs = lt.process
_child("scheduler.loop.process_result", start, end, pr_attrs)
if _KV_SEND_SPAN_ENABLED and lt.kv_send_start is not None:
_child(
"scheduler.loop.kv_send",
lt.kv_send_start,
lt.kv_send_end,
{"sglang.n_chunks": int(lt.kv_chunks)},
)
if _FORWARD_SPAN_ENABLED:
for completion_wall, elapsed_ns, category in list(_FORWARD_EVENTS):
if completion_wall < lt.t0_wall_ns or completion_wall > now_wall_ns:
continue
_child(
"scheduler.loop.forward.gpu",
completion_wall - elapsed_ns,
completion_wall,
{
"sglang.gpu_ms": elapsed_ns / 1e6,
"sglang.category": category,
},
)
lt.span.end(end_time=now_wall_ns)
except Exception:
# Telemetry must never break the scheduler loop.
pass
finally:
_LOOP_TLS.current = None