observability: fix scheduler no-batch accounting

This commit is contained in:
2026-11-05 16:42:39 +08:00
parent 5d5870f3ea
commit 145e05c44d
3 changed files with 342 additions and 90 deletions
+35 -10
View File
@@ -617,11 +617,15 @@ class SchedulerDisaggregationPrefillMixin:
self.attn_tp_cpu_group,
)
failed = set()
bootstrap_candidates = len(candidates)
pending_bootstrap = 0
for req, poll in zip(candidates, polls):
if poll == KVPoll.Failed:
self.handle_bootstrap_failure(req)
failed.add(req)
elif (
bootstrap_candidates -= 1
continue
if (
poll == KVPoll.WaitingForInput
and req.pending_bootstrap
and not should_force_retry(req)
@@ -630,6 +634,10 @@ class SchedulerDisaggregationPrefillMixin:
# finalize cannot fail here; if it ever does, the request stays
# pending and the post-forward check resolves it.
self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req)
pending_bootstrap += int(req.pending_bootstrap)
batch_telemetry.set_bootstrap_pending(
pending_bootstrap, bootstrap_candidates
)
if failed:
self.waiting_queue = [
req for req in self.waiting_queue if req not in failed
@@ -807,6 +815,8 @@ class SchedulerDisaggregationPrefillMixin:
self.last_batch is not None
or len(self.result_queue) > 0
or len(self.waiting_queue) > 0
or len(self._deferred_input_requests) > 0
or self._engine_paused
or (
self.disagg_prefill_inflight_queue is not None
and len(self.disagg_prefill_inflight_queue) > 0
@@ -823,10 +833,6 @@ class SchedulerDisaggregationPrefillMixin:
inflight = self.disagg_prefill_inflight_queue
return {
"waiting": len(self.waiting_queue),
"waiting_all_bootstrap": bool(self.waiting_queue)
and all(
getattr(r, "pending_bootstrap", False) for r in self.waiting_queue
),
"result_queue": len(self.result_queue),
"result_limit": 2,
"inflight": len(inflight) if inflight is not None else 0,
@@ -889,6 +895,14 @@ class SchedulerDisaggregationPrefillMixin:
if not waiting_for_batch_result or received_inputs or bootstrapped_reqs:
skip_batch_creation = False
# Preserve the launch decision before processing a result mutates the
# queues; end-of-loop depths cannot explain why this loop did not launch.
if _lt is not None:
if len(self.result_queue) >= 2:
batch_telemetry.note_launch_blocked(_lt, "result_backpressure")
elif self._deferred_input_requests:
batch_telemetry.note_launch_blocked(_lt, "deferred_input")
# Keep at most two batches in flight.
if (
len(self.result_queue) < 2
@@ -917,9 +931,13 @@ class SchedulerDisaggregationPrefillMixin:
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
_rb_t0 = time.time_ns()
_rb_t0 = (
time.time_ns()
if _lt is not None and _lt.span is not None
else 0
)
batch_result = self.run_batch(batch)
if _lt is not None:
if _rb_t0:
try:
batch_telemetry.mark_run_batch(
_rb_t0,
@@ -951,13 +969,20 @@ class SchedulerDisaggregationPrefillMixin:
):
waiting_for_batch_result = True
self._yield_gil_if_needed()
_end_loop(_lt, False, None, waiting_for_result=True)
_end_loop(
_lt,
loop_batch is not None,
loop_batch,
waiting_for_result=True,
)
continue
# Process the oldest batch result.
oldest_batch, oldest_batch_result = self.result_queue.popleft()
_pr_t0 = time.time_ns()
_pr_t0 = (
time.time_ns() if _lt is not None and _lt.span is not None else 0
)
self.process_batch_result(oldest_batch, oldest_batch_result)
if _lt is not None:
if _pr_t0:
try:
batch_telemetry.mark_process(
_pr_t0,
@@ -529,6 +529,8 @@ class PassTimings:
"candidates",
"skipped_prefetch",
"admitted",
"bootstrap_candidates",
"pending_bootstrap",
"had_chunked",
"new_chunk",
"seq",
@@ -546,6 +548,8 @@ class PassTimings:
self.candidates = 0
self.skipped_prefetch = 0
self.admitted = 0
self.bootstrap_candidates = 0
self.pending_bootstrap = 0
self.had_chunked = False
self.new_chunk = False
self.seq = 0
@@ -608,6 +612,14 @@ def set_admitted(n: int) -> None:
t.admitted = int(n)
def set_bootstrap_pending(pending: int, candidates: int) -> None:
"""Reuse the bootstrap resolver's existing pass; do not rescan the queue."""
t = _current()
if t is not None:
t.pending_bootstrap = int(pending)
t.bootstrap_candidates = int(candidates)
def note_had_chunked(flag: bool) -> None:
"""Record whether a chunked request was already in flight when the pass began.
@@ -659,12 +671,16 @@ def add_mlp_sync(seconds: float) -> None:
def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None:
if not pass_span_decision(t.seq):
return
loop = getattr(_LOOP_TLS, "current", None)
loop_ctx = loop.span_ctx if loop is not None else None
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
# An unsampled loop owns this pass; do not leak an independent pass trace.
# If work is discovered later, end_loop emits the snapshot as its child.
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.pass")
loop = getattr(_LOOP_TLS, "current", None)
loop_ctx = loop.span_ctx if loop is not None else None
end_ns = time.time_ns()
start_ns = end_ns - int(total * 1e9)
tracer.start_span(
@@ -845,12 +861,17 @@ def observe_recv(
_RECV_SPAN_SAMPLE * 10000
):
return
loop = current_loop()
loop_ctx = current_loop_context()
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
# Keep the snapshot only; end_loop emits it if this loop has work.
return
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.recv")
tracer.start_span(
"scheduler.recv_requests",
context=current_loop_context(),
context=loop_ctx,
start_time=start_ns,
attributes={
"sglang.phase": "recv",
@@ -864,6 +885,8 @@ def observe_recv(
"sglang.recv_mm_ms": mm * 1e3,
},
).end(end_time=end_ns)
if loop is not None and loop_ctx is not None:
loop.recv_span_child = True
except Exception:
# Telemetry must never break the scheduler loop.
pass
@@ -970,7 +993,10 @@ def forward_span_enabled() -> bool:
def kv_send_span_enabled() -> bool:
return _KV_SEND_SPAN_ENABLED
if not _KV_SEND_SPAN_ENABLED:
return False
lt = getattr(_LOOP_TLS, "current", None)
return lt is not None and lt.span is not None
def loop_sample_decision(seq: int) -> bool:
@@ -1003,6 +1029,8 @@ class LoopTimings:
"span",
"span_ctx",
"pass_span_child",
"recv_span_child",
"launch_blocked_reason",
"seq",
"no_batch_reason",
)
@@ -1034,10 +1062,37 @@ class LoopTimings:
self.span = None
self.span_ctx = None
self.pass_span_child = False
self.recv_span_child = False
self.launch_blocked_reason = ""
self.seq = 0
self.no_batch_reason = ""
def ensure_loop_span(lt: Optional[LoopTimings]) -> None:
"""Materialize a sampled span once the loop is known to have work."""
if (
lt is None
or lt.span is not None
or not loop_sample_decision(lt.predicted_seq)
):
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.loop")
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
except Exception:
lt.span = None
lt.span_ctx = None
def note_launch_blocked(lt: Optional[LoopTimings], reason: str) -> None:
"""Preserve why this loop could not launch before queues are consumed."""
if lt is not None and not lt.launch_blocked_reason:
lt.launch_blocked_reason = reason
def begin_loop(
phase: str = "prefill",
*,
@@ -1068,16 +1123,8 @@ def begin_loop(
)
_LAST_MLP_OUTCOME = ("none", "none")
_LOOP_TLS.current = lt
if _LOOP_SPAN_ENABLED and lt.has_work0 and loop_sample_decision(lt.predicted_seq):
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.loop")
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
except Exception:
lt.span = None
lt.span_ctx = None
if lt.has_work0:
ensure_loop_span(lt)
return lt
@@ -1121,6 +1168,8 @@ def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
"sglang.candidates": int(t.candidates),
"sglang.skipped_prefetch": int(t.skipped_prefetch),
"sglang.admitted": int(t.admitted),
"sglang.bootstrap_candidates": int(t.bootstrap_candidates),
"sglang.pending_bootstrap": int(t.pending_bootstrap),
"sglang.resumed_chunked": int(bool(t.had_chunked)),
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
"sglang.mlp_sync_seq": int(t.seq),
@@ -1154,8 +1203,14 @@ def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None:
"""Set by dp_attn.prepare_mlp_sync_batch_raw once per pass."""
if not _LOOP_ENABLED:
return
global _LAST_MLP_OUTCOME
_LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome))
if local_kind != "none" or global_outcome != "none":
# DP idle work is only known after the sync. Activate the span before
# end_pass so the get_next child can share the loop trace.
ensure_loop_span(getattr(_LOOP_TLS, "current", None))
def note_forward_device(t: float, category: str = "", **kwargs) -> None:
@@ -1227,20 +1282,27 @@ def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str:
if state.get("deferred", 0) > 0 or state.get("paused"):
return "deferred_input"
waiting = int(state.get("waiting", 0) or 0)
if waiting > 0:
if state.get("waiting_all_bootstrap"):
if waiting > 0 and pass_snapshot is not None:
bootstrap_candidates = int(
pass_snapshot.get("sglang.bootstrap_candidates", 0) or 0
)
pending_bootstrap = int(
pass_snapshot.get("sglang.pending_bootstrap", 0) or 0
)
if bootstrap_candidates > 0 and pending_bootstrap == bootstrap_candidates:
return "waiting_bootstrap"
if pass_snapshot is not None:
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
if candidates > 0 and skipped == candidates:
return "waiting_prefetch"
if admitted == 0:
return "admission_blocked"
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
if candidates > 0 and skipped == candidates:
return "waiting_prefetch"
if admitted == 0:
return "admission_blocked"
inflight = int(state.get("inflight", 0) or 0)
bootstrap = int(state.get("bootstrap", 0) or 0)
if waiting == 0 and (inflight or bootstrap):
if waiting == 0 and bootstrap:
return "waiting_bootstrap"
if waiting == 0 and inflight:
return "waiting_pd"
if (
waiting == 0
@@ -1317,34 +1379,50 @@ def end_loop(
)
local_kind, global_outcome = _LAST_MLP_OUTCOME
pass_snapshot = lt.get_next[2] if lt.get_next is not None else None
pending = bool(
state.get("waiting", 0)
or state.get("result_queue", 0)
or state.get("inflight", 0)
or state.get("bootstrap", 0)
or state.get("deferred", 0)
or state.get("paused")
or state.get("waiting_for_result")
)
reason = ""
if not has_batch:
try:
try:
if local_kind == "none" and global_outcome == "idle":
# The DP sync replaces local None with a real idle batch, so this
# accounting must not depend on has_batch being False.
_ensure_loop_metrics()["lockstep_seconds"].labels(
dp_rank=str(dp_rank), forward_overlap=overlap
).inc(loop_s)
elif not has_batch:
reason = lt.launch_blocked_reason or _classify_no_batch(
state, pass_snapshot
)
global _LAST_NO_BATCH_REASON
_LAST_NO_BATCH_REASON = reason
metrics = _ensure_loop_metrics()
if local_kind == "none" and global_outcome == "idle":
metrics["lockstep_seconds"].labels(
dp_rank=str(dp_rank), forward_overlap=overlap
).inc(loop_s)
else:
reason = _classify_no_batch(state, pass_snapshot)
global _LAST_NO_BATCH_REASON
_LAST_NO_BATCH_REASON = reason
metrics["no_batch_seconds"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc(loop_s)
metrics["no_batch_total"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc()
except Exception:
# Telemetry must never break the scheduler loop.
pass
metrics["no_batch_seconds"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc(loop_s)
metrics["no_batch_total"].labels(
dp_rank=str(dp_rank),
reason=reason,
forward_overlap=overlap,
).inc()
except Exception:
# Telemetry must never break the scheduler loop.
pass
lt.no_batch_reason = reason
if lt.span is None and (has_batch or pending):
# Covers work discovered after begin_loop, including first input and
# deferred-only loops. DP outcomes activate earlier for pass parenting.
ensure_loop_span(lt)
if lt.span is None:
return
@@ -1352,13 +1430,6 @@ def end_loop(
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.loop")
pending = bool(
state.get("waiting", 0)
or state.get("result_queue", 0)
or state.get("inflight", 0)
or state.get("bootstrap", 0)
or state.get("waiting_for_result")
)
attrs = {
"sglang.phase": lt.phase,
"sglang.dp_rank": int(dp_rank),
@@ -1387,7 +1458,7 @@ def end_loop(
except Exception:
pass
if lt.recv is not None:
if lt.recv is not None and not lt.recv_span_child:
start, end, parts = lt.recv
child = {"sglang.dp_rank": int(dp_rank)}
child.update(parts)
@@ -10,6 +10,8 @@ Covers the design's acceptance points:
import importlib.util
import os
import pathlib
import sys
import types
import unittest
from unittest import mock
@@ -63,6 +65,69 @@ def _fake_metrics():
return fakes
class _FakeHistogram:
def labels(self, **kw):
return self
def observe(self, value):
pass
class _FakeSpan:
def __init__(self, name, **kwargs):
self.name = name
self.kwargs = kwargs
self.attributes = {}
self.end_time = None
def set_attributes(self, attrs):
self.attributes.update(attrs)
def end(self, end_time=None):
self.end_time = end_time
class _FakeTracer:
def __init__(self):
self.spans = []
def start_span(self, name, **kwargs):
span = _FakeSpan(name, **kwargs)
self.spans.append(span)
return span
def _fake_otel():
tracer = _FakeTracer()
trace = types.ModuleType("opentelemetry.trace")
trace.get_tracer = lambda _name: tracer
trace.set_span_in_context = lambda span: ("loop-context", span)
package = types.ModuleType("opentelemetry")
package.trace = trace
patcher = mock.patch.dict(
sys.modules,
{"opentelemetry": package, "opentelemetry.trace": trace},
)
return tracer, patcher
def _state(**overrides):
state = {
"waiting": 0,
"result_queue": 0,
"result_limit": 2,
"inflight": 0,
"bootstrap": 0,
"deferred": 0,
"paused": False,
"waiting_for_result": False,
"skip_batch_creation": False,
"forward_active": False,
}
state.update(overrides)
return state
class LoopAccountingTest(unittest.TestCase):
def test_sampling_is_deterministic_and_bounded(self):
mod = _load()
@@ -83,7 +148,15 @@ class LoopAccountingTest(unittest.TestCase):
cases = [
({"result_queue": 2}, None, "result_backpressure"),
({"deferred": 1}, None, "deferred_input"),
({"waiting": 2, "waiting_all_bootstrap": True}, None, "waiting_bootstrap"),
(
{"waiting": 2},
{
"sglang.bootstrap_candidates": 2,
"sglang.pending_bootstrap": 2,
"sglang.admitted": 0,
},
"waiting_bootstrap",
),
(
{"waiting": 2},
{
@@ -102,6 +175,7 @@ class LoopAccountingTest(unittest.TestCase):
},
"admission_blocked",
),
({"waiting": 0, "bootstrap": 1}, None, "waiting_bootstrap"),
({"waiting": 0, "inflight": 1}, None, "waiting_pd"),
(
{
@@ -133,22 +207,13 @@ class LoopAccountingTest(unittest.TestCase):
mod._ensure_loop_metrics = lambda: fakes
def run_once():
mod._LAST_MLP_OUTCOME = ("none", "none")
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
mod.note_mlp_sync_outcome("none", "none")
mod.end_loop(
lt,
dp_rank=0,
has_batch=False,
state={
"waiting": 0,
"inflight": 0,
"bootstrap": 0,
"result_queue": 0,
"deferred": 0,
"waiting_for_result": False,
"forward_active": False,
},
state=_state(),
)
mod._LOOP_SPAN_ENABLED = False
@@ -160,30 +225,121 @@ class LoopAccountingTest(unittest.TestCase):
keys = list(fakes["no_batch_total"].data)
self.assertTrue(any(("reason", "fully_idle") in k for k in keys))
def test_dp_lockstep_idle_is_not_no_batch(self):
def test_dp_lockstep_idle_is_counted_with_real_idle_batch(self):
mod = _load()
fakes = _fake_metrics()
mod._ensure_loop_metrics = lambda: fakes
mod._LAST_MLP_OUTCOME = ("none", "idle")
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
lt = mod.begin_loop("prefill", pre_has_work=False, forward_active=False)
mod.note_mlp_sync_outcome("none", "idle")
# prepare_mlp_sync_batch_raw replaces local None with a real idle batch.
mod.end_loop(
lt,
dp_rank=1,
has_batch=False,
state={
"waiting": 0,
"inflight": 0,
"bootstrap": 0,
"result_queue": 0,
"deferred": 0,
"waiting_for_result": False,
"forward_active": False,
},
has_batch=True,
state=_state(),
)
self.assertFalse(fakes["no_batch_seconds"].data)
self.assertFalse(fakes["no_batch_total"].data)
self.assertTrue(fakes["lockstep_seconds"].data)
def test_launch_backpressure_survives_result_queue_popleft(self):
mod = _load()
fakes = _fake_metrics()
mod._ensure_loop_metrics = lambda: fakes
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
mod.note_launch_blocked(lt, "result_backpressure")
mod.note_mlp_sync_outcome("none", "none")
# The regular overlap path consumes one result before end_loop.
mod.end_loop(
lt,
dp_rank=0,
has_batch=False,
state=_state(result_queue=1, waiting=2),
)
keys = list(fakes["no_batch_total"].data)
self.assertTrue(any(("reason", "result_backpressure") in k for k in keys))
self.assertFalse(any(("reason", "no_ready_other") in k for k in keys))
def test_late_work_activates_loop_before_pass_span(self):
mod = _load()
mod._LOOP_SPAN_ENABLED = True
mod._LOOP_SPAN_SAMPLE = 1.0
mod._PASS_SPAN_ENABLED = True
mod._PASS_SPAN_SAMPLE = 1.0
mod._ensure_loop_metrics = lambda: _fake_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=False, forward_active=False)
self.assertIsNone(lt.span)
# First input or DP idle is discovered only after begin_loop.
mod.note_mlp_sync_outcome("extend", "extend")
self.assertIsNotNone(lt.span)
pass_t = mod.PassTimings("prefill")
pass_t.seq = lt.predicted_seq
mod._emit_pass_span(0, pass_t, 0.001, 0.0)
self.assertTrue(lt.pass_span_child)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
names = [span.name for span in tracer.spans]
self.assertEqual(names.count("scheduler.loop"), 1)
self.assertEqual(names.count("scheduler.prefill.pass"), 1)
self.assertNotIn("scheduler.loop.get_next", names)
pass_span = next(
span for span in tracer.spans if span.name == "scheduler.prefill.pass"
)
self.assertEqual(pass_span.kwargs["context"], lt.span_ctx)
self.assertTrue(lt.span.attributes["sglang.has_batch"])
def test_deferred_work_activates_span_at_loop_end(self):
mod = _load()
mod._LOOP_SPAN_ENABLED = True
mod._LOOP_SPAN_SAMPLE = 1.0
mod._ensure_loop_metrics = lambda: _fake_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=False, forward_active=False)
self.assertIsNone(lt.span)
mod.end_loop(
lt,
dp_rank=0,
has_batch=False,
state=_state(deferred=1),
)
self.assertEqual([span.name for span in tracer.spans], ["scheduler.loop"])
self.assertEqual(lt.no_batch_reason, "deferred_input")
def test_recv_span_is_not_duplicated_under_loop(self):
mod = _load()
mod._LOOP_SPAN_ENABLED = True
mod._LOOP_SPAN_SAMPLE = 1.0
mod._RECV_SPAN_ENABLED = True
mod._RECV_SPAN_SAMPLE = 1.0
mod._ensure_loop_metrics = lambda: _fake_metrics()
mod._ensure_recv_metrics = lambda: (
_FakeHistogram(),
{"msgs": _FakeCounter(), "prompt_tokens": _FakeCounter()},
)
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
mod.observe_recv(
dp_rank=0,
total=0.001,
pull=0.0005,
broadcast=0.0,
unwrap=0.0005,
mm=0.0,
n_msgs=1,
n_tokens=16,
)
self.assertTrue(lt.recv_span_child)
mod.note_mlp_sync_outcome("extend", "extend")
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
names = [span.name for span in tracer.spans]
self.assertEqual(names.count("scheduler.recv_requests"), 1)
self.assertNotIn("scheduler.loop.recv", names)
if __name__ == "__main__":
unittest.main()