observability: add scheduler loop span and no-batch accounting

Adds an opt-in scheduler.loop span (SGLANG_TRACE_LOOP_SPAN) with recv /
get_next / run_batch / process_result / kv_send children. The span is only
created for sampled loops that have pending work; fully idle loops stay in
the counters so the 714 pass/s idle spin never floods Jaeger.

No-batch accounting is always on with SGLANG_BATCH_STAGE_TIMING:
- sglang:scheduler_no_batch_{seconds,total}{dp_rank,reason,forward_overlap}
- sglang:scheduler_dp_lockstep_idle_seconds_total{dp_rank,forward_overlap}
with the design's reason priority (result_backpressure, deferred_input,
waiting_bootstrap, waiting_prefetch, admission_blocked, waiting_pd,
fully_idle, no_ready_other) and local_batch_kind/global_outcome from the DP
sync. DP lockstep idle is counted separately from pure no-batch wait.

loop gpu_ms reuses the existing DeviceTimer intervals (no new CUDA events);
the forward.gpu lane is optional and approximate. Pass/recv spans become
children of the loop span when it is active.

Tests: test_loop_span_accounting (5) and the existing stage-timing suite (8)
pass.
This commit is contained in:
2026-11-05 15:00:23 +08:00
parent 309af6e6e1
commit 5d5870f3ea
5 changed files with 871 additions and 2 deletions
+113
View File
@@ -790,10 +790,90 @@ class SchedulerDisaggregationPrefillMixin:
batch = None
batch_result = None
def _forward_active() -> bool:
reporter = getattr(self, "metrics_reporter", None)
timer = (
getattr(reporter, "forward_pass_device_timer", None)
if reporter is not None
else None
)
return timer.is_active() if timer is not None else False
def _pre_has_work() -> bool:
bootstrap_queue = getattr(
self.disagg_prefill_bootstrap_queue, "queue", None
)
return bool(
self.last_batch is not None
or len(self.result_queue) > 0
or len(self.waiting_queue) > 0
or (
self.disagg_prefill_inflight_queue is not None
and len(self.disagg_prefill_inflight_queue) > 0
)
or (bootstrap_queue is not None and len(bootstrap_queue) > 0)
)
def _loop_state(
waiting_for_result: bool = False, paused: bool = False
) -> dict:
bootstrap_queue = getattr(
self.disagg_prefill_bootstrap_queue, "queue", None
)
inflight = self.disagg_prefill_inflight_queue
return {
"waiting": len(self.waiting_queue),
"waiting_all_bootstrap": bool(self.waiting_queue)
and all(
getattr(r, "pending_bootstrap", False) for r in self.waiting_queue
),
"result_queue": len(self.result_queue),
"result_limit": 2,
"inflight": len(inflight) if inflight is not None else 0,
"bootstrap": len(bootstrap_queue) if bootstrap_queue is not None else 0,
"deferred": len(self._deferred_input_requests),
"paused": bool(paused or self._engine_paused),
"waiting_for_result": bool(waiting_for_result),
"skip_batch_creation": bool(skip_batch_creation),
"forward_active": _forward_active(),
}
def _end_loop(
lt,
has_batch: bool,
loop_batch,
waiting_for_result: bool = False,
paused: bool = False,
) -> None:
if lt is None:
return
try:
batch_telemetry.end_loop(
lt,
dp_rank=get_parallel().attn_dp_rank,
has_batch=has_batch,
batch=loop_batch,
state=_loop_state(
waiting_for_result=waiting_for_result, paused=paused
),
)
except Exception:
# Telemetry must never break the scheduler loop.
pass
while True:
if self.gracefully_exit:
break
_lt = None
if batch_telemetry.loop_accounting_enabled():
_lt = batch_telemetry.begin_loop(
"prefill",
pre_has_work=_pre_has_work(),
forward_active=_forward_active,
)
loop_batch = None
# Stop dispatching at pause requests while waiting for a batch result.
received_inputs = self.ingest_requests(
stop_at_pause=waiting_for_batch_result
@@ -801,6 +881,7 @@ class SchedulerDisaggregationPrefillMixin:
if self._engine_paused:
self._record_scheduler_state_for_paused_engine()
self._yield_gil_if_needed()
_end_loop(_lt, False, None, paused=True)
continue
bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
self.waiting_queue.extend(bootstrapped_reqs)
@@ -827,6 +908,7 @@ class SchedulerDisaggregationPrefillMixin:
batch = self.ngram_embedding_manager.prepare_for_forward(
batch, chunked_req=self.chunked_req
)
loop_batch = batch
self.cur_batch_for_debug = batch
# Skip batch creation next round to avoid repeated preparation failures.
skip_batch_creation = batch is None
@@ -835,7 +917,23 @@ class SchedulerDisaggregationPrefillMixin:
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
_rb_t0 = time.time_ns()
batch_result = self.run_batch(batch)
if _lt is not None:
try:
batch_telemetry.mark_run_batch(
_rb_t0,
time.time_ns(),
{
"sglang.forward_mode": str(batch.forward_mode),
"sglang.bs": batch.batch_size(),
"sglang.num_tokens": int(
getattr(batch, "extend_num_tokens", 0) or 0
),
},
)
except Exception:
pass
self._apply_war_barrier()
self.result_queue.append((batch.copy(), batch_result))
else:
@@ -853,10 +951,21 @@ class SchedulerDisaggregationPrefillMixin:
):
waiting_for_batch_result = True
self._yield_gil_if_needed()
_end_loop(_lt, False, None, waiting_for_result=True)
continue
# Process the oldest batch result.
oldest_batch, oldest_batch_result = self.result_queue.popleft()
_pr_t0 = time.time_ns()
self.process_batch_result(oldest_batch, oldest_batch_result)
if _lt is not None:
try:
batch_telemetry.mark_process(
_pr_t0,
time.time_ns(),
{"sglang.n_reqs": len(oldest_batch.reqs)},
)
except Exception:
pass
elif batch is None:
# When the server is idle, do self-check and re-init some states
self.on_idle()
@@ -871,6 +980,7 @@ class SchedulerDisaggregationPrefillMixin:
self._sched_idled = batch is None
self.last_batch = batch
waiting_for_batch_result = False
_end_loop(_lt, loop_batch is not None, loop_batch)
def is_disagg_prefill_batch_result_ready(
self: Scheduler,
@@ -1511,7 +1621,10 @@ class SchedulerDisaggregationPrefillMixin:
state_indices = state_indices_for_request(self, req, end_idx)
value = computer.compute(page_indices_gpu, state_indices)
self.disagg_metadata_buffers.set_kv_checksum(req, value)
_kv_t0 = time.time_ns() if batch_telemetry.kv_send_span_enabled() else 0
self._send_kv_chunk(req, last_chunk=last_chunk, end_idx=end_idx)
if _kv_t0:
batch_telemetry.add_kv_send(_kv_t0, time.time_ns())
def _send_kv_chunk(
self: Scheduler,
+2
View File
@@ -1417,6 +1417,8 @@ class Scheduler(
metrics_collector_context=self.metrics_collector_context,
metrics_collector=self.metrics_collector,
)
# Read the already-running DeviceTimer; adds no CUDA events.
batch_telemetry.install_forward_reporter(self)
def init_schedule_policy(self):
# Init schedule policy and new token estimation
@@ -675,6 +675,15 @@ def prepare_mlp_sync_batch_raw(
else:
need_idle_batch = max(mlp_sync_info.global_num_tokens) > 0
# Capture the pre-sync local kind before the idle replacement below: the
# loop span's local_batch_kind describes what this rank brought in, not the
# idle batch it may be asked to run for DP lockstep.
_loop_local_kind = (
"none"
if local_batch is None
else ("idle" if local_batch.forward_mode.is_idle() else "extend")
)
batch_to_gather = local_batch
if need_idle_batch:
if local_batch is None:
@@ -683,6 +692,13 @@ def prepare_mlp_sync_batch_raw(
# NOTE: for prebuilt batch, we add an inner idle batch to run MLP sync
batch_to_gather = local_batch.inner_idle_batch = get_idle_batch()
_loop_global_outcome = (
"idle"
if (_loop_local_kind == "none" and need_idle_batch)
else ("none" if _loop_local_kind == "none" else _loop_local_kind)
)
batch_telemetry.note_mlp_sync_outcome(_loop_local_kind, _loop_global_outcome)
if batch_to_gather is not None:
_update_gather_batch(
batch_to_gather,
@@ -663,10 +663,13 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) ->
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.pass")
loop = getattr(_LOOP_TLS, "current", None)
loop_ctx = loop.span_ctx if loop is not None else None
end_ns = time.time_ns()
start_ns = end_ns - int(total * 1e9)
tracer.start_span(
"scheduler.prefill.pass",
context=loop_ctx,
start_time=start_ns,
attributes={
"sglang.phase": t.phase,
@@ -687,6 +690,8 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) ->
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
},
).end(end_time=end_ns)
if loop is not None and loop_ctx is not None:
loop.pass_span_child = True
except Exception:
# Tracing must never break the scheduler loop.
pass
@@ -736,6 +741,7 @@ def end_pass(dp_rank: int) -> None:
keyed["resumed_chunk"].inc(1)
elif t.new_chunk:
keyed["new_chunk"].inc(1)
note_pass_snapshot(t, total)
_emit_pass_span(dp_rank, t, total, other)
except Exception:
# Telemetry must never break the scheduler loop
@@ -817,6 +823,22 @@ def observe_recv(
counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs)
if n_tokens:
counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens)
end_ns = time.time_ns()
start_ns = end_ns - int(total * 1e9)
note_recv_phase(
start_wall_ns=start_ns,
end_wall_ns=end_ns,
parts={
"sglang.recv_total_ms": total * 1e3,
"sglang.recv_pull_ms": pull * 1e3,
"sglang.recv_broadcast_ms": broadcast * 1e3,
"sglang.recv_unwrap_ms": unwrap * 1e3,
"sglang.recv_mm_ms": mm * 1e3,
"sglang.recv_msgs": int(n_msgs),
"sglang.recv_prompt_tokens": int(n_tokens),
},
)
if not _RECV_SPAN_ENABLED:
return
if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int(
@@ -826,10 +848,10 @@ def observe_recv(
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.recv")
end_ns = time.time_ns()
tracer.start_span(
"scheduler.recv_requests",
start_time=end_ns - int(total * 1e9),
context=current_loop_context(),
start_time=start_ns,
attributes={
"sglang.phase": "recv",
"sglang.dp_rank": int(dp_rank),
@@ -878,3 +900,530 @@ def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None
except Exception:
# Telemetry must never break the scheduler loop.
pass
# ---------------------------------------------------------------------------
# Scheduler loop span + no-batch accounting (opt-in, same stage-timing gate).
#
# One ``scheduler.loop`` span per scheduler iteration (sampled), with phase
# children (recv / get_next / run_batch / process_result / kv_send). The
# no-batch counters are incremented on *every* loop (sampling only controls
# spans), split by a low-cardinality reason and by whether a forward was in
# flight (forward_overlap). Fully idle loops never emit a span.
#
# Enable with ``SGLANG_TRACE_LOOP_SPAN=1`` (requires SGLANG_BATCH_STAGE_TIMING).
# Knobs:
# SGLANG_TRACE_LOOP_SAMPLE=0.05 head-sample ratio (deterministic per
# mlp_sync_seq, identical on all ranks)
# SGLANG_TRACE_FORWARD_SPAN=0 emit approximate forward.gpu lanes
# SGLANG_TRACE_KV_SEND_SPAN=1 emit the kv_send child (default on
# when the loop span is on)
# ---------------------------------------------------------------------------
from collections import deque as _deque
_LOOP_ENABLED = _STAGE_TIMING
_LOOP_SPAN_ENABLED = _LOOP_ENABLED and os.getenv(
"SGLANG_TRACE_LOOP_SPAN", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_LOOP_SPAN_SAMPLE = float(os.getenv("SGLANG_TRACE_LOOP_SAMPLE", "0.05"))
_FORWARD_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
"SGLANG_TRACE_FORWARD_SPAN", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_KV_SEND_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
"SGLANG_TRACE_KV_SEND_SPAN", "1"
).strip().lower() not in ("", "0", "false", "no", "off")
_NO_BATCH_REASONS = (
"fully_idle",
"waiting_prefetch",
"waiting_bootstrap",
"result_backpressure",
"deferred_input",
"admission_blocked",
"waiting_pd",
"no_ready_other",
)
_LOOP_METRICS = None
_LOOP_TLS = threading.local()
# Monotonic total of completed forward device time (from the DeviceTimer),
# used to derive one loop's gpu_ms without adding a second timer.
_FORWARD_DEVICE_NS_TOTAL = 0
_FORWARD_EVENTS = _deque(maxlen=128) # (completion_wall_ns, elapsed_ns, category)
# Set by dp_attn.prepare_mlp_sync_batch_raw() once per pass.
_LAST_MLP_OUTCOME = ("none", "none")
_LAST_NO_BATCH_REASON = "no_ready_other"
def loop_accounting_enabled() -> bool:
return _LOOP_ENABLED
def loop_span_enabled() -> bool:
return _LOOP_SPAN_ENABLED
def forward_span_enabled() -> bool:
return _FORWARD_SPAN_ENABLED
def kv_send_span_enabled() -> bool:
return _KV_SEND_SPAN_ENABLED
def loop_sample_decision(seq: int) -> bool:
"""Deterministic, rank-identical loop-span sampling decision."""
if not _LOOP_SPAN_ENABLED or seq <= 0:
return False
if _LOOP_SPAN_SAMPLE >= 1.0:
return True
if _LOOP_SPAN_SAMPLE <= 0.0:
return False
return (seq * 2654435761) % 10000 < int(_LOOP_SPAN_SAMPLE * 10000)
class LoopTimings:
__slots__ = (
"phase",
"t0_ns",
"t0_wall_ns",
"predicted_seq",
"active0",
"has_work0",
"forward_total0",
"recv",
"get_next",
"run_batch",
"process",
"kv_send_start",
"kv_send_end",
"kv_chunks",
"span",
"span_ctx",
"pass_span_child",
"seq",
"no_batch_reason",
)
def __init__(
self,
phase: str,
t0_ns: int,
t0_wall_ns: int,
predicted_seq: int,
active0: bool,
has_work0: bool,
forward_total0: int,
):
self.phase = phase
self.t0_ns = t0_ns
self.t0_wall_ns = t0_wall_ns
self.predicted_seq = predicted_seq
self.active0 = bool(active0)
self.has_work0 = bool(has_work0)
self.forward_total0 = forward_total0
self.recv = None
self.get_next = None
self.run_batch = None
self.process = None
self.kv_send_start = None
self.kv_send_end = None
self.kv_chunks = 0
self.span = None
self.span_ctx = None
self.pass_span_child = False
self.seq = 0
self.no_batch_reason = ""
def begin_loop(
phase: str = "prefill",
*,
pre_has_work: bool = True,
forward_active=None,
):
"""Start one scheduler-loop accounting record.
``pre_has_work`` must be cheap to compute (queue lengths / last_batch):
fully idle loops are still counted, but never create a span.
``forward_active`` is a callable (usually DeviceTimer.is_active) or a bool.
"""
global _LAST_MLP_OUTCOME
if not _LOOP_ENABLED:
return None
try:
active0 = bool(forward_active() if callable(forward_active) else forward_active)
except Exception:
active0 = False
lt = LoopTimings(
phase=phase,
t0_ns=time.monotonic_ns(),
t0_wall_ns=time.time_ns(),
predicted_seq=_DP_GATHER_SEQ + 1,
active0=active0,
has_work0=bool(pre_has_work),
forward_total0=_FORWARD_DEVICE_NS_TOTAL,
)
_LAST_MLP_OUTCOME = ("none", "none")
_LOOP_TLS.current = lt
if _LOOP_SPAN_ENABLED and lt.has_work0 and loop_sample_decision(lt.predicted_seq):
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.loop")
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
except Exception:
lt.span = None
lt.span_ctx = None
return lt
def current_loop() -> Optional[LoopTimings]:
return getattr(_LOOP_TLS, "current", None)
def current_loop_context():
lt = getattr(_LOOP_TLS, "current", None)
return lt.span_ctx if lt is not None else None
def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None:
"""Record the recv_requests phase inside the current loop (if any)."""
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
lt.recv = (int(start_wall_ns), int(end_wall_ns), dict(parts))
def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
"""Record get_next timings for the current loop (called from end_pass)."""
lt = getattr(_LOOP_TLS, "current", None)
if lt is None:
return
end_wall_ns = time.time_ns()
start_wall_ns = end_wall_ns - int(total_s * 1e9)
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
other = max(0.0, total_s - measured)
lt.get_next = (
start_wall_ns,
end_wall_ns,
{
"sglang.get_next_ms": total_s * 1e3,
"sglang.match_ms": t.match * 1e3,
"sglang.adder_ms": t.adder * 1e3,
"sglang.hicache_ms": t.hicache * 1e3,
"sglang.bootstrap_ms": t.bootstrap * 1e3,
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
"sglang.other_ms": other * 1e3,
"sglang.candidates": int(t.candidates),
"sglang.skipped_prefetch": int(t.skipped_prefetch),
"sglang.admitted": int(t.admitted),
"sglang.resumed_chunked": int(bool(t.had_chunked)),
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
"sglang.mlp_sync_seq": int(t.seq),
},
)
lt.seq = int(t.seq)
def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
lt.process = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
lt = getattr(_LOOP_TLS, "current", None)
if lt is None:
return
lt.kv_chunks += 1
if lt.kv_send_start is None or start_wall_ns < lt.kv_send_start:
lt.kv_send_start = int(start_wall_ns)
if lt.kv_send_end is None or end_wall_ns > lt.kv_send_end:
lt.kv_send_end = int(end_wall_ns)
def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None:
"""Set by dp_attn.prepare_mlp_sync_batch_raw once per pass."""
global _LAST_MLP_OUTCOME
_LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome))
def note_forward_device(t: float, category: str = "", **kwargs) -> None:
"""DeviceTimer reporter: accumulate completed forward device time."""
global _FORWARD_DEVICE_NS_TOTAL
try:
ns = int(float(t) * 1e9)
except Exception:
return
_FORWARD_DEVICE_NS_TOTAL += ns
if _FORWARD_SPAN_ENABLED:
_FORWARD_EVENTS.append((time.time_ns(), ns, str(category)))
def install_forward_reporter(scheduler) -> None:
"""Attach the forward device-time reporter to the existing DeviceTimer.
No CUDA event is added here: SGLANG_ENABLE_METRICS_DEVICE_TIMER already
runs the timer; this only reads its completed intervals.
"""
if not _LOOP_ENABLED:
return
try:
reporter = getattr(scheduler, "metrics_reporter", None)
timer = getattr(reporter, "forward_pass_device_timer", None)
if timer is not None:
timer.add_reporter(note_forward_device)
except Exception:
# Telemetry must never break scheduler init.
pass
def _ensure_loop_metrics():
global _LOOP_METRICS
if _LOOP_METRICS is None:
from prometheus_client import Counter
_LOOP_METRICS = {
"no_batch_seconds": Counter(
"sglang:scheduler_no_batch_seconds_total",
"Wall time of scheduler loops that ended without a runnable batch, "
"by reason and forward_overlap",
["dp_rank", "reason", "forward_overlap"],
),
"no_batch_total": Counter(
"sglang:scheduler_no_batch_total",
"Count of scheduler loops that ended without a runnable batch, "
"by reason and forward_overlap",
["dp_rank", "reason", "forward_overlap"],
),
"lockstep_seconds": Counter(
"sglang:scheduler_dp_lockstep_idle_seconds_total",
"Wall time of DP-lockstep idle-forward loops (this rank had no "
"local batch but the DP group required an idle forward)",
["dp_rank", "forward_overlap"],
),
}
return _LOOP_METRICS
def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str:
"""Low-cardinality no-batch reason, in the design's priority order.
``pass_snapshot`` is the get_next attribute dict (candidates / skipped /
admitted). It is None when the loop skipped batch preparation entirely.
"""
if state.get("result_queue", 0) >= state.get("result_limit", 2):
return "result_backpressure"
if state.get("deferred", 0) > 0 or state.get("paused"):
return "deferred_input"
waiting = int(state.get("waiting", 0) or 0)
if waiting > 0:
if state.get("waiting_all_bootstrap"):
return "waiting_bootstrap"
if pass_snapshot is not None:
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
if candidates > 0 and skipped == candidates:
return "waiting_prefetch"
if admitted == 0:
return "admission_blocked"
inflight = int(state.get("inflight", 0) or 0)
bootstrap = int(state.get("bootstrap", 0) or 0)
if waiting == 0 and (inflight or bootstrap):
return "waiting_pd"
if (
waiting == 0
and inflight == 0
and bootstrap == 0
and int(state.get("result_queue", 0) or 0) == 0
and int(state.get("deferred", 0) or 0) == 0
and not state.get("waiting_for_result")
):
return "fully_idle"
if state.get("skip_batch_creation") and pass_snapshot is None:
# A failed preparation defers retries; keep the last classified reason.
return _LAST_NO_BATCH_REASON
return "no_ready_other"
def _batch_attrs(batch) -> dict:
if batch is None:
return {
"sglang.n_reqs": 0,
"sglang.n_new": 0,
"sglang.n_cached": 0,
"sglang.chunked": False,
}
try:
n_reqs = int(batch.batch_size())
except Exception:
n_reqs = len(getattr(batch, "reqs", ()) or ())
try:
n_new = int(getattr(batch, "extend_num_tokens", 0) or 0)
except Exception:
n_new = 0
try:
n_cached = int(sum(getattr(batch, "prefix_lens", ()) or ()))
except Exception:
n_cached = 0
try:
chunked = getattr(batch, "chunked_req", None) is not None
except Exception:
chunked = False
return {
"sglang.n_reqs": n_reqs,
"sglang.n_new": n_new,
"sglang.n_cached": n_cached,
"sglang.chunked": bool(chunked),
}
def end_loop(
lt: Optional[LoopTimings],
*,
dp_rank: int,
has_batch: bool,
batch=None,
state: Optional[dict] = None,
) -> None:
"""Close one loop: counters (always), loop span + phases (if sampled)."""
if lt is None:
return
_LOOP_TLS.current = None
state = state or {}
try:
now_ns = time.monotonic_ns()
now_wall_ns = time.time_ns()
loop_s = max(0.0, (now_ns - lt.t0_ns) / 1e9)
gpu_ns = max(0, _FORWARD_DEVICE_NS_TOTAL - lt.forward_total0)
active1 = bool(state.get("forward_active", False))
overlap = (
"full"
if lt.active0 and active1
else "partial"
if (lt.active0 or active1)
else "none"
)
local_kind, global_outcome = _LAST_MLP_OUTCOME
pass_snapshot = lt.get_next[2] if lt.get_next is not None else None
reason = ""
if not has_batch:
try:
metrics = _ensure_loop_metrics()
if local_kind == "none" and global_outcome == "idle":
metrics["lockstep_seconds"].labels(
dp_rank=str(dp_rank), forward_overlap=overlap
).inc(loop_s)
else:
reason = _classify_no_batch(state, pass_snapshot)
global _LAST_NO_BATCH_REASON
_LAST_NO_BATCH_REASON = reason
metrics["no_batch_seconds"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc(loop_s)
metrics["no_batch_total"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc()
except Exception:
# Telemetry must never break the scheduler loop.
pass
lt.no_batch_reason = reason
if lt.span is None:
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.loop")
pending = bool(
state.get("waiting", 0)
or state.get("result_queue", 0)
or state.get("inflight", 0)
or state.get("bootstrap", 0)
or state.get("waiting_for_result")
)
attrs = {
"sglang.phase": lt.phase,
"sglang.dp_rank": int(dp_rank),
"sglang.loop_seq": int(lt.seq or lt.predicted_seq),
"sglang.loop_ms": loop_s * 1e3,
"sglang.gpu_ms": gpu_ns / 1e6,
"sglang.gpu_idle_ms": max(0.0, loop_s * 1e3 - gpu_ns / 1e6),
"sglang.has_batch": bool(has_batch),
"sglang.engine_has_pending_work": pending,
"sglang.local_batch_kind": local_kind,
"sglang.global_outcome": global_outcome,
"sglang.no_batch_reason": reason,
"sglang.forward_overlap": overlap,
}
attrs.update(_batch_attrs(batch))
lt.span.set_attributes(attrs)
def _child(name, start_ns, end_ns, child_attrs):
try:
tracer.start_span(
name,
context=lt.span_ctx,
start_time=int(start_ns),
attributes=child_attrs,
).end(end_time=int(end_ns))
except Exception:
pass
if lt.recv is not None:
start, end, parts = lt.recv
child = {"sglang.dp_rank": int(dp_rank)}
child.update(parts)
_child("scheduler.loop.recv", start, end, child)
if lt.get_next is not None and not lt.pass_span_child:
start, end, ng_attrs = lt.get_next
_child("scheduler.loop.get_next", start, end, ng_attrs)
if lt.run_batch is not None:
start, end, rb_attrs = lt.run_batch
_child("scheduler.loop.run_batch", start, end, rb_attrs)
if lt.process is not None:
start, end, pr_attrs = lt.process
_child("scheduler.loop.process_result", start, end, pr_attrs)
if _KV_SEND_SPAN_ENABLED and lt.kv_send_start is not None:
_child(
"scheduler.loop.kv_send",
lt.kv_send_start,
lt.kv_send_end,
{"sglang.n_chunks": int(lt.kv_chunks)},
)
if _FORWARD_SPAN_ENABLED:
for completion_wall, elapsed_ns, category in list(_FORWARD_EVENTS):
if completion_wall < lt.t0_wall_ns or completion_wall > now_wall_ns:
continue
_child(
"scheduler.loop.forward.gpu",
completion_wall - elapsed_ns,
completion_wall,
{
"sglang.gpu_ms": elapsed_ns / 1e6,
"sglang.category": category,
},
)
lt.span.end(end_time=now_wall_ns)
except Exception:
# Telemetry must never break the scheduler loop.
pass
finally:
_LOOP_TLS.current = None
@@ -0,0 +1,189 @@
"""Unit tests for the scheduler loop span + no-batch accounting.
Covers the design's acceptance points:
* deterministic, rank-identical loop-span sampling by mlp_sync_seq;
* no-batch reason priority and the no_ready_other fallback;
* the no-batch counters are independent of span sampling;
* DP lockstep idle is counted separately from pure no-batch wait;
* forward device-time accumulation feeds the loop gpu_ms source.
"""
import importlib.util
import os
import pathlib
import unittest
from unittest import mock
def _module_path():
p = (
pathlib.Path(__file__).resolve().parents[4]
/ "python/sglang/srt/observability/batch_telemetry.py"
)
if p.exists():
return p
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
return pathlib.Path(spec.origin)
def _load(**env):
base = {
"SGLANG_BATCH_STAGE_TIMING": "1",
"SGLANG_TRACE_LOOP_SPAN": "0",
"SGLANG_TRACE_LOOP_SAMPLE": "0.05",
"SGLANG_TRACE_FORWARD_SPAN": "0",
}
base.update(env)
with mock.patch.dict(os.environ, base):
spec = importlib.util.spec_from_file_location(
"batch_telemetry_loop_under_test", _module_path()
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
class _FakeCounter:
def __init__(self):
self.data = {}
def labels(self, **kw):
self._key = tuple(sorted(kw.items()))
return self
def inc(self, value=1.0):
self.data[self._key] = self.data.get(self._key, 0.0) + value
def _fake_metrics():
fakes = {
"no_batch_seconds": _FakeCounter(),
"no_batch_total": _FakeCounter(),
"lockstep_seconds": _FakeCounter(),
}
return fakes
class LoopAccountingTest(unittest.TestCase):
def test_sampling_is_deterministic_and_bounded(self):
mod = _load()
mod._LOOP_SPAN_ENABLED = True
mod._LOOP_SPAN_SAMPLE = 1.0
self.assertTrue(mod.loop_sample_decision(12345))
self.assertTrue(mod.loop_sample_decision(12345))
self.assertFalse(mod.loop_sample_decision(0))
mod._LOOP_SPAN_SAMPLE = 0.0
self.assertFalse(mod.loop_sample_decision(1))
mod._LOOP_SPAN_SAMPLE = 0.05
a = [mod.loop_sample_decision(i) for i in range(200)]
b = [mod.loop_sample_decision(i) for i in range(200)]
self.assertEqual(a, b)
def test_no_batch_reason_priority(self):
mod = _load()
cases = [
({"result_queue": 2}, None, "result_backpressure"),
({"deferred": 1}, None, "deferred_input"),
({"waiting": 2, "waiting_all_bootstrap": True}, None, "waiting_bootstrap"),
(
{"waiting": 2},
{
"sglang.candidates": 5,
"sglang.skipped_prefetch": 5,
"sglang.admitted": 0,
},
"waiting_prefetch",
),
(
{"waiting": 2},
{
"sglang.candidates": 5,
"sglang.skipped_prefetch": 1,
"sglang.admitted": 0,
},
"admission_blocked",
),
({"waiting": 0, "inflight": 1}, None, "waiting_pd"),
(
{
"waiting": 0,
"inflight": 0,
"bootstrap": 0,
"result_queue": 0,
"deferred": 0,
},
None,
"fully_idle",
),
]
for state, snapshot, expected in cases:
self.assertEqual(
mod._classify_no_batch(state, snapshot), expected, (state, snapshot)
)
def test_forward_device_time_accumulates(self):
mod = _load()
before = mod._FORWARD_DEVICE_NS_TOTAL
mod.note_forward_device(0.0025, "extend")
mod.note_forward_device(0.0075, "extend")
self.assertGreaterEqual(mod._FORWARD_DEVICE_NS_TOTAL - before, 10_000_000)
def test_no_batch_counters_do_not_depend_on_span_sampling(self):
mod = _load()
fakes = _fake_metrics()
mod._ensure_loop_metrics = lambda: fakes
def run_once():
mod._LAST_MLP_OUTCOME = ("none", "none")
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
mod.note_mlp_sync_outcome("none", "none")
mod.end_loop(
lt,
dp_rank=0,
has_batch=False,
state={
"waiting": 0,
"inflight": 0,
"bootstrap": 0,
"result_queue": 0,
"deferred": 0,
"waiting_for_result": False,
"forward_active": False,
},
)
mod._LOOP_SPAN_ENABLED = False
run_once()
mod._LOOP_SPAN_ENABLED = True
mod._LOOP_SPAN_SAMPLE = 0.0
run_once()
self.assertEqual(sum(fakes["no_batch_total"].data.values()), 2.0)
keys = list(fakes["no_batch_total"].data)
self.assertTrue(any(("reason", "fully_idle") in k for k in keys))
def test_dp_lockstep_idle_is_not_no_batch(self):
mod = _load()
fakes = _fake_metrics()
mod._ensure_loop_metrics = lambda: fakes
mod._LAST_MLP_OUTCOME = ("none", "idle")
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
mod.note_mlp_sync_outcome("none", "idle")
mod.end_loop(
lt,
dp_rank=1,
has_batch=False,
state={
"waiting": 0,
"inflight": 0,
"bootstrap": 0,
"result_queue": 0,
"deferred": 0,
"waiting_for_result": False,
"forward_active": False,
},
)
self.assertFalse(fakes["no_batch_seconds"].data)
self.assertTrue(fakes["lockstep_seconds"].data)
if __name__ == "__main__":
unittest.main()