From 145e05c44d33759c5800f629ee99619725cd201e Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Thu, 5 Nov 2026 16:42:39 +0800 Subject: [PATCH] observability: fix scheduler no-batch accounting --- python/sglang/srt/disaggregation/prefill.py | 45 +++- .../srt/observability/batch_telemetry.py | 183 +++++++++++----- .../test_loop_span_accounting.py | 204 +++++++++++++++--- 3 files changed, 342 insertions(+), 90 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 456c9c48a9..ab6be253a2 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -617,11 +617,15 @@ class SchedulerDisaggregationPrefillMixin: self.attn_tp_cpu_group, ) failed = set() + bootstrap_candidates = len(candidates) + pending_bootstrap = 0 for req, poll in zip(candidates, polls): if poll == KVPoll.Failed: self.handle_bootstrap_failure(req) failed.add(req) - elif ( + bootstrap_candidates -= 1 + continue + if ( poll == KVPoll.WaitingForInput and req.pending_bootstrap and not should_force_retry(req) @@ -630,6 +634,10 @@ class SchedulerDisaggregationPrefillMixin: # finalize cannot fail here; if it ever does, the request stays # pending and the post-forward check resolves it. self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req) + pending_bootstrap += int(req.pending_bootstrap) + batch_telemetry.set_bootstrap_pending( + pending_bootstrap, bootstrap_candidates + ) if failed: self.waiting_queue = [ req for req in self.waiting_queue if req not in failed @@ -807,6 +815,8 @@ class SchedulerDisaggregationPrefillMixin: self.last_batch is not None or len(self.result_queue) > 0 or len(self.waiting_queue) > 0 + or len(self._deferred_input_requests) > 0 + or self._engine_paused or ( self.disagg_prefill_inflight_queue is not None and len(self.disagg_prefill_inflight_queue) > 0 @@ -823,10 +833,6 @@ class SchedulerDisaggregationPrefillMixin: inflight = self.disagg_prefill_inflight_queue return { "waiting": len(self.waiting_queue), - "waiting_all_bootstrap": bool(self.waiting_queue) - and all( - getattr(r, "pending_bootstrap", False) for r in self.waiting_queue - ), "result_queue": len(self.result_queue), "result_limit": 2, "inflight": len(inflight) if inflight is not None else 0, @@ -889,6 +895,14 @@ class SchedulerDisaggregationPrefillMixin: if not waiting_for_batch_result or received_inputs or bootstrapped_reqs: skip_batch_creation = False + # Preserve the launch decision before processing a result mutates the + # queues; end-of-loop depths cannot explain why this loop did not launch. + if _lt is not None: + if len(self.result_queue) >= 2: + batch_telemetry.note_launch_blocked(_lt, "result_backpressure") + elif self._deferred_input_requests: + batch_telemetry.note_launch_blocked(_lt, "deferred_input") + # Keep at most two batches in flight. if ( len(self.result_queue) < 2 @@ -917,9 +931,13 @@ class SchedulerDisaggregationPrefillMixin: if batch: if self.enable_staging: self.maybe_prefetch_staging_for_batch(batch) - _rb_t0 = time.time_ns() + _rb_t0 = ( + time.time_ns() + if _lt is not None and _lt.span is not None + else 0 + ) batch_result = self.run_batch(batch) - if _lt is not None: + if _rb_t0: try: batch_telemetry.mark_run_batch( _rb_t0, @@ -951,13 +969,20 @@ class SchedulerDisaggregationPrefillMixin: ): waiting_for_batch_result = True self._yield_gil_if_needed() - _end_loop(_lt, False, None, waiting_for_result=True) + _end_loop( + _lt, + loop_batch is not None, + loop_batch, + waiting_for_result=True, + ) continue # Process the oldest batch result. oldest_batch, oldest_batch_result = self.result_queue.popleft() - _pr_t0 = time.time_ns() + _pr_t0 = ( + time.time_ns() if _lt is not None and _lt.span is not None else 0 + ) self.process_batch_result(oldest_batch, oldest_batch_result) - if _lt is not None: + if _pr_t0: try: batch_telemetry.mark_process( _pr_t0, diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index 31ccdf3ac9..aea740e682 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -529,6 +529,8 @@ class PassTimings: "candidates", "skipped_prefetch", "admitted", + "bootstrap_candidates", + "pending_bootstrap", "had_chunked", "new_chunk", "seq", @@ -546,6 +548,8 @@ class PassTimings: self.candidates = 0 self.skipped_prefetch = 0 self.admitted = 0 + self.bootstrap_candidates = 0 + self.pending_bootstrap = 0 self.had_chunked = False self.new_chunk = False self.seq = 0 @@ -608,6 +612,14 @@ def set_admitted(n: int) -> None: t.admitted = int(n) +def set_bootstrap_pending(pending: int, candidates: int) -> None: + """Reuse the bootstrap resolver's existing pass; do not rescan the queue.""" + t = _current() + if t is not None: + t.pending_bootstrap = int(pending) + t.bootstrap_candidates = int(candidates) + + def note_had_chunked(flag: bool) -> None: """Record whether a chunked request was already in flight when the pass began. @@ -659,12 +671,16 @@ def add_mlp_sync(seconds: float) -> None: def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None: if not pass_span_decision(t.seq): return + loop = getattr(_LOOP_TLS, "current", None) + loop_ctx = loop.span_ctx if loop is not None else None + if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None: + # An unsampled loop owns this pass; do not leak an independent pass trace. + # If work is discovered later, end_loop emits the snapshot as its child. + return try: from opentelemetry import trace as otel_trace tracer = otel_trace.get_tracer("sglang.prefill.pass") - loop = getattr(_LOOP_TLS, "current", None) - loop_ctx = loop.span_ctx if loop is not None else None end_ns = time.time_ns() start_ns = end_ns - int(total * 1e9) tracer.start_span( @@ -845,12 +861,17 @@ def observe_recv( _RECV_SPAN_SAMPLE * 10000 ): return + loop = current_loop() + loop_ctx = current_loop_context() + if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None: + # Keep the snapshot only; end_loop emits it if this loop has work. + return from opentelemetry import trace as otel_trace tracer = otel_trace.get_tracer("sglang.recv") tracer.start_span( "scheduler.recv_requests", - context=current_loop_context(), + context=loop_ctx, start_time=start_ns, attributes={ "sglang.phase": "recv", @@ -864,6 +885,8 @@ def observe_recv( "sglang.recv_mm_ms": mm * 1e3, }, ).end(end_time=end_ns) + if loop is not None and loop_ctx is not None: + loop.recv_span_child = True except Exception: # Telemetry must never break the scheduler loop. pass @@ -970,7 +993,10 @@ def forward_span_enabled() -> bool: def kv_send_span_enabled() -> bool: - return _KV_SEND_SPAN_ENABLED + if not _KV_SEND_SPAN_ENABLED: + return False + lt = getattr(_LOOP_TLS, "current", None) + return lt is not None and lt.span is not None def loop_sample_decision(seq: int) -> bool: @@ -1003,6 +1029,8 @@ class LoopTimings: "span", "span_ctx", "pass_span_child", + "recv_span_child", + "launch_blocked_reason", "seq", "no_batch_reason", ) @@ -1034,10 +1062,37 @@ class LoopTimings: self.span = None self.span_ctx = None self.pass_span_child = False + self.recv_span_child = False + self.launch_blocked_reason = "" self.seq = 0 self.no_batch_reason = "" +def ensure_loop_span(lt: Optional[LoopTimings]) -> None: + """Materialize a sampled span once the loop is known to have work.""" + if ( + lt is None + or lt.span is not None + or not loop_sample_decision(lt.predicted_seq) + ): + return + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.prefill.loop") + lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns) + lt.span_ctx = otel_trace.set_span_in_context(lt.span) + except Exception: + lt.span = None + lt.span_ctx = None + + +def note_launch_blocked(lt: Optional[LoopTimings], reason: str) -> None: + """Preserve why this loop could not launch before queues are consumed.""" + if lt is not None and not lt.launch_blocked_reason: + lt.launch_blocked_reason = reason + + def begin_loop( phase: str = "prefill", *, @@ -1068,16 +1123,8 @@ def begin_loop( ) _LAST_MLP_OUTCOME = ("none", "none") _LOOP_TLS.current = lt - if _LOOP_SPAN_ENABLED and lt.has_work0 and loop_sample_decision(lt.predicted_seq): - try: - from opentelemetry import trace as otel_trace - - tracer = otel_trace.get_tracer("sglang.prefill.loop") - lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns) - lt.span_ctx = otel_trace.set_span_in_context(lt.span) - except Exception: - lt.span = None - lt.span_ctx = None + if lt.has_work0: + ensure_loop_span(lt) return lt @@ -1121,6 +1168,8 @@ def note_pass_snapshot(t: "PassTimings", total_s: float) -> None: "sglang.candidates": int(t.candidates), "sglang.skipped_prefetch": int(t.skipped_prefetch), "sglang.admitted": int(t.admitted), + "sglang.bootstrap_candidates": int(t.bootstrap_candidates), + "sglang.pending_bootstrap": int(t.pending_bootstrap), "sglang.resumed_chunked": int(bool(t.had_chunked)), "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), "sglang.mlp_sync_seq": int(t.seq), @@ -1154,8 +1203,14 @@ def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None: def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None: """Set by dp_attn.prepare_mlp_sync_batch_raw once per pass.""" + if not _LOOP_ENABLED: + return global _LAST_MLP_OUTCOME _LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome)) + if local_kind != "none" or global_outcome != "none": + # DP idle work is only known after the sync. Activate the span before + # end_pass so the get_next child can share the loop trace. + ensure_loop_span(getattr(_LOOP_TLS, "current", None)) def note_forward_device(t: float, category: str = "", **kwargs) -> None: @@ -1227,20 +1282,27 @@ def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str: if state.get("deferred", 0) > 0 or state.get("paused"): return "deferred_input" waiting = int(state.get("waiting", 0) or 0) - if waiting > 0: - if state.get("waiting_all_bootstrap"): + if waiting > 0 and pass_snapshot is not None: + bootstrap_candidates = int( + pass_snapshot.get("sglang.bootstrap_candidates", 0) or 0 + ) + pending_bootstrap = int( + pass_snapshot.get("sglang.pending_bootstrap", 0) or 0 + ) + if bootstrap_candidates > 0 and pending_bootstrap == bootstrap_candidates: return "waiting_bootstrap" - if pass_snapshot is not None: - candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0) - skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0) - admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0) - if candidates > 0 and skipped == candidates: - return "waiting_prefetch" - if admitted == 0: - return "admission_blocked" + candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0) + skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0) + admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0) + if candidates > 0 and skipped == candidates: + return "waiting_prefetch" + if admitted == 0: + return "admission_blocked" inflight = int(state.get("inflight", 0) or 0) bootstrap = int(state.get("bootstrap", 0) or 0) - if waiting == 0 and (inflight or bootstrap): + if waiting == 0 and bootstrap: + return "waiting_bootstrap" + if waiting == 0 and inflight: return "waiting_pd" if ( waiting == 0 @@ -1317,34 +1379,50 @@ def end_loop( ) local_kind, global_outcome = _LAST_MLP_OUTCOME pass_snapshot = lt.get_next[2] if lt.get_next is not None else None + pending = bool( + state.get("waiting", 0) + or state.get("result_queue", 0) + or state.get("inflight", 0) + or state.get("bootstrap", 0) + or state.get("deferred", 0) + or state.get("paused") + or state.get("waiting_for_result") + ) reason = "" - if not has_batch: - try: + try: + if local_kind == "none" and global_outcome == "idle": + # The DP sync replaces local None with a real idle batch, so this + # accounting must not depend on has_batch being False. + _ensure_loop_metrics()["lockstep_seconds"].labels( + dp_rank=str(dp_rank), forward_overlap=overlap + ).inc(loop_s) + elif not has_batch: + reason = lt.launch_blocked_reason or _classify_no_batch( + state, pass_snapshot + ) + global _LAST_NO_BATCH_REASON + _LAST_NO_BATCH_REASON = reason metrics = _ensure_loop_metrics() - if local_kind == "none" and global_outcome == "idle": - metrics["lockstep_seconds"].labels( - dp_rank=str(dp_rank), forward_overlap=overlap - ).inc(loop_s) - else: - reason = _classify_no_batch(state, pass_snapshot) - global _LAST_NO_BATCH_REASON - _LAST_NO_BATCH_REASON = reason - metrics["no_batch_seconds"].labels( - dp_rank=str(dp_rank), - reason=reason, - forward_overlap=overlap, - ).inc(loop_s) - metrics["no_batch_total"].labels( - dp_rank=str(dp_rank), - reason=reason, - forward_overlap=overlap, - ).inc() - except Exception: - # Telemetry must never break the scheduler loop. - pass + metrics["no_batch_seconds"].labels( + dp_rank=str(dp_rank), + reason=reason, + forward_overlap=overlap, + ).inc(loop_s) + metrics["no_batch_total"].labels( + dp_rank=str(dp_rank), + reason=reason, + forward_overlap=overlap, + ).inc() + except Exception: + # Telemetry must never break the scheduler loop. + pass lt.no_batch_reason = reason + if lt.span is None and (has_batch or pending): + # Covers work discovered after begin_loop, including first input and + # deferred-only loops. DP outcomes activate earlier for pass parenting. + ensure_loop_span(lt) if lt.span is None: return @@ -1352,13 +1430,6 @@ def end_loop( from opentelemetry import trace as otel_trace tracer = otel_trace.get_tracer("sglang.prefill.loop") - pending = bool( - state.get("waiting", 0) - or state.get("result_queue", 0) - or state.get("inflight", 0) - or state.get("bootstrap", 0) - or state.get("waiting_for_result") - ) attrs = { "sglang.phase": lt.phase, "sglang.dp_rank": int(dp_rank), @@ -1387,7 +1458,7 @@ def end_loop( except Exception: pass - if lt.recv is not None: + if lt.recv is not None and not lt.recv_span_child: start, end, parts = lt.recv child = {"sglang.dp_rank": int(dp_rank)} child.update(parts) diff --git a/test/registered/unit/observability/test_loop_span_accounting.py b/test/registered/unit/observability/test_loop_span_accounting.py index a416fc3bbf..b3162c6e5b 100644 --- a/test/registered/unit/observability/test_loop_span_accounting.py +++ b/test/registered/unit/observability/test_loop_span_accounting.py @@ -10,6 +10,8 @@ Covers the design's acceptance points: import importlib.util import os import pathlib +import sys +import types import unittest from unittest import mock @@ -63,6 +65,69 @@ def _fake_metrics(): return fakes +class _FakeHistogram: + def labels(self, **kw): + return self + + def observe(self, value): + pass + + +class _FakeSpan: + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs + self.attributes = {} + self.end_time = None + + def set_attributes(self, attrs): + self.attributes.update(attrs) + + def end(self, end_time=None): + self.end_time = end_time + + +class _FakeTracer: + def __init__(self): + self.spans = [] + + def start_span(self, name, **kwargs): + span = _FakeSpan(name, **kwargs) + self.spans.append(span) + return span + + +def _fake_otel(): + tracer = _FakeTracer() + trace = types.ModuleType("opentelemetry.trace") + trace.get_tracer = lambda _name: tracer + trace.set_span_in_context = lambda span: ("loop-context", span) + package = types.ModuleType("opentelemetry") + package.trace = trace + patcher = mock.patch.dict( + sys.modules, + {"opentelemetry": package, "opentelemetry.trace": trace}, + ) + return tracer, patcher + + +def _state(**overrides): + state = { + "waiting": 0, + "result_queue": 0, + "result_limit": 2, + "inflight": 0, + "bootstrap": 0, + "deferred": 0, + "paused": False, + "waiting_for_result": False, + "skip_batch_creation": False, + "forward_active": False, + } + state.update(overrides) + return state + + class LoopAccountingTest(unittest.TestCase): def test_sampling_is_deterministic_and_bounded(self): mod = _load() @@ -83,7 +148,15 @@ class LoopAccountingTest(unittest.TestCase): cases = [ ({"result_queue": 2}, None, "result_backpressure"), ({"deferred": 1}, None, "deferred_input"), - ({"waiting": 2, "waiting_all_bootstrap": True}, None, "waiting_bootstrap"), + ( + {"waiting": 2}, + { + "sglang.bootstrap_candidates": 2, + "sglang.pending_bootstrap": 2, + "sglang.admitted": 0, + }, + "waiting_bootstrap", + ), ( {"waiting": 2}, { @@ -102,6 +175,7 @@ class LoopAccountingTest(unittest.TestCase): }, "admission_blocked", ), + ({"waiting": 0, "bootstrap": 1}, None, "waiting_bootstrap"), ({"waiting": 0, "inflight": 1}, None, "waiting_pd"), ( { @@ -133,22 +207,13 @@ class LoopAccountingTest(unittest.TestCase): mod._ensure_loop_metrics = lambda: fakes def run_once(): - mod._LAST_MLP_OUTCOME = ("none", "none") lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) mod.note_mlp_sync_outcome("none", "none") mod.end_loop( lt, dp_rank=0, has_batch=False, - state={ - "waiting": 0, - "inflight": 0, - "bootstrap": 0, - "result_queue": 0, - "deferred": 0, - "waiting_for_result": False, - "forward_active": False, - }, + state=_state(), ) mod._LOOP_SPAN_ENABLED = False @@ -160,30 +225,121 @@ class LoopAccountingTest(unittest.TestCase): keys = list(fakes["no_batch_total"].data) self.assertTrue(any(("reason", "fully_idle") in k for k in keys)) - def test_dp_lockstep_idle_is_not_no_batch(self): + def test_dp_lockstep_idle_is_counted_with_real_idle_batch(self): mod = _load() fakes = _fake_metrics() mod._ensure_loop_metrics = lambda: fakes - mod._LAST_MLP_OUTCOME = ("none", "idle") - lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + lt = mod.begin_loop("prefill", pre_has_work=False, forward_active=False) mod.note_mlp_sync_outcome("none", "idle") + # prepare_mlp_sync_batch_raw replaces local None with a real idle batch. mod.end_loop( lt, dp_rank=1, - has_batch=False, - state={ - "waiting": 0, - "inflight": 0, - "bootstrap": 0, - "result_queue": 0, - "deferred": 0, - "waiting_for_result": False, - "forward_active": False, - }, + has_batch=True, + state=_state(), ) self.assertFalse(fakes["no_batch_seconds"].data) + self.assertFalse(fakes["no_batch_total"].data) self.assertTrue(fakes["lockstep_seconds"].data) + def test_launch_backpressure_survives_result_queue_popleft(self): + mod = _load() + fakes = _fake_metrics() + mod._ensure_loop_metrics = lambda: fakes + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.note_launch_blocked(lt, "result_backpressure") + mod.note_mlp_sync_outcome("none", "none") + # The regular overlap path consumes one result before end_loop. + mod.end_loop( + lt, + dp_rank=0, + has_batch=False, + state=_state(result_queue=1, waiting=2), + ) + keys = list(fakes["no_batch_total"].data) + self.assertTrue(any(("reason", "result_backpressure") in k for k in keys)) + self.assertFalse(any(("reason", "no_ready_other") in k for k in keys)) + + def test_late_work_activates_loop_before_pass_span(self): + mod = _load() + mod._LOOP_SPAN_ENABLED = True + mod._LOOP_SPAN_SAMPLE = 1.0 + mod._PASS_SPAN_ENABLED = True + mod._PASS_SPAN_SAMPLE = 1.0 + mod._ensure_loop_metrics = lambda: _fake_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=False, forward_active=False) + self.assertIsNone(lt.span) + # First input or DP idle is discovered only after begin_loop. + mod.note_mlp_sync_outcome("extend", "extend") + self.assertIsNotNone(lt.span) + pass_t = mod.PassTimings("prefill") + pass_t.seq = lt.predicted_seq + mod._emit_pass_span(0, pass_t, 0.001, 0.0) + self.assertTrue(lt.pass_span_child) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + + names = [span.name for span in tracer.spans] + self.assertEqual(names.count("scheduler.loop"), 1) + self.assertEqual(names.count("scheduler.prefill.pass"), 1) + self.assertNotIn("scheduler.loop.get_next", names) + pass_span = next( + span for span in tracer.spans if span.name == "scheduler.prefill.pass" + ) + self.assertEqual(pass_span.kwargs["context"], lt.span_ctx) + self.assertTrue(lt.span.attributes["sglang.has_batch"]) + + def test_deferred_work_activates_span_at_loop_end(self): + mod = _load() + mod._LOOP_SPAN_ENABLED = True + mod._LOOP_SPAN_SAMPLE = 1.0 + mod._ensure_loop_metrics = lambda: _fake_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=False, forward_active=False) + self.assertIsNone(lt.span) + mod.end_loop( + lt, + dp_rank=0, + has_batch=False, + state=_state(deferred=1), + ) + self.assertEqual([span.name for span in tracer.spans], ["scheduler.loop"]) + self.assertEqual(lt.no_batch_reason, "deferred_input") + + def test_recv_span_is_not_duplicated_under_loop(self): + mod = _load() + mod._LOOP_SPAN_ENABLED = True + mod._LOOP_SPAN_SAMPLE = 1.0 + mod._RECV_SPAN_ENABLED = True + mod._RECV_SPAN_SAMPLE = 1.0 + mod._ensure_loop_metrics = lambda: _fake_metrics() + mod._ensure_recv_metrics = lambda: ( + _FakeHistogram(), + {"msgs": _FakeCounter(), "prompt_tokens": _FakeCounter()}, + ) + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.observe_recv( + dp_rank=0, + total=0.001, + pull=0.0005, + broadcast=0.0, + unwrap=0.0005, + mm=0.0, + n_msgs=1, + n_tokens=16, + ) + self.assertTrue(lt.recv_span_child) + mod.note_mlp_sync_outcome("extend", "extend") + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + + names = [span.name for span in tracer.spans] + self.assertEqual(names.count("scheduler.recv_requests"), 1) + self.assertNotIn("scheduler.loop.recv", names) + if __name__ == "__main__": unittest.main()