From 5d5870f3eae3becd61ce69c05fe591fbdc545868 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Thu, 5 Nov 2026 15:00:23 +0800 Subject: [PATCH] observability: add scheduler loop span and no-batch accounting Adds an opt-in scheduler.loop span (SGLANG_TRACE_LOOP_SPAN) with recv / get_next / run_batch / process_result / kv_send children. The span is only created for sampled loops that have pending work; fully idle loops stay in the counters so the 714 pass/s idle spin never floods Jaeger. No-batch accounting is always on with SGLANG_BATCH_STAGE_TIMING: - sglang:scheduler_no_batch_{seconds,total}{dp_rank,reason,forward_overlap} - sglang:scheduler_dp_lockstep_idle_seconds_total{dp_rank,forward_overlap} with the design's reason priority (result_backpressure, deferred_input, waiting_bootstrap, waiting_prefetch, admission_blocked, waiting_pd, fully_idle, no_ready_other) and local_batch_kind/global_outcome from the DP sync. DP lockstep idle is counted separately from pure no-batch wait. loop gpu_ms reuses the existing DeviceTimer intervals (no new CUDA events); the forward.gpu lane is optional and approximate. Pass/recv spans become children of the loop span when it is active. Tests: test_loop_span_accounting (5) and the existing stage-timing suite (8) pass. --- python/sglang/srt/disaggregation/prefill.py | 113 ++++ python/sglang/srt/managers/scheduler.py | 2 + .../managers/scheduler_components/dp_attn.py | 16 + .../srt/observability/batch_telemetry.py | 553 +++++++++++++++++- .../test_loop_span_accounting.py | 189 ++++++ 5 files changed, 871 insertions(+), 2 deletions(-) create mode 100644 test/registered/unit/observability/test_loop_span_accounting.py diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 4ad4febe40..456c9c48a9 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -790,10 +790,90 @@ class SchedulerDisaggregationPrefillMixin: batch = None batch_result = None + def _forward_active() -> bool: + reporter = getattr(self, "metrics_reporter", None) + timer = ( + getattr(reporter, "forward_pass_device_timer", None) + if reporter is not None + else None + ) + return timer.is_active() if timer is not None else False + + def _pre_has_work() -> bool: + bootstrap_queue = getattr( + self.disagg_prefill_bootstrap_queue, "queue", None + ) + return bool( + self.last_batch is not None + or len(self.result_queue) > 0 + or len(self.waiting_queue) > 0 + or ( + self.disagg_prefill_inflight_queue is not None + and len(self.disagg_prefill_inflight_queue) > 0 + ) + or (bootstrap_queue is not None and len(bootstrap_queue) > 0) + ) + + def _loop_state( + waiting_for_result: bool = False, paused: bool = False + ) -> dict: + bootstrap_queue = getattr( + self.disagg_prefill_bootstrap_queue, "queue", None + ) + inflight = self.disagg_prefill_inflight_queue + return { + "waiting": len(self.waiting_queue), + "waiting_all_bootstrap": bool(self.waiting_queue) + and all( + getattr(r, "pending_bootstrap", False) for r in self.waiting_queue + ), + "result_queue": len(self.result_queue), + "result_limit": 2, + "inflight": len(inflight) if inflight is not None else 0, + "bootstrap": len(bootstrap_queue) if bootstrap_queue is not None else 0, + "deferred": len(self._deferred_input_requests), + "paused": bool(paused or self._engine_paused), + "waiting_for_result": bool(waiting_for_result), + "skip_batch_creation": bool(skip_batch_creation), + "forward_active": _forward_active(), + } + + def _end_loop( + lt, + has_batch: bool, + loop_batch, + waiting_for_result: bool = False, + paused: bool = False, + ) -> None: + if lt is None: + return + try: + batch_telemetry.end_loop( + lt, + dp_rank=get_parallel().attn_dp_rank, + has_batch=has_batch, + batch=loop_batch, + state=_loop_state( + waiting_for_result=waiting_for_result, paused=paused + ), + ) + except Exception: + # Telemetry must never break the scheduler loop. + pass + while True: if self.gracefully_exit: break + _lt = None + if batch_telemetry.loop_accounting_enabled(): + _lt = batch_telemetry.begin_loop( + "prefill", + pre_has_work=_pre_has_work(), + forward_active=_forward_active, + ) + loop_batch = None + # Stop dispatching at pause requests while waiting for a batch result. received_inputs = self.ingest_requests( stop_at_pause=waiting_for_batch_result @@ -801,6 +881,7 @@ class SchedulerDisaggregationPrefillMixin: if self._engine_paused: self._record_scheduler_state_for_paused_engine() self._yield_gil_if_needed() + _end_loop(_lt, False, None, paused=True) continue bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped() self.waiting_queue.extend(bootstrapped_reqs) @@ -827,6 +908,7 @@ class SchedulerDisaggregationPrefillMixin: batch = self.ngram_embedding_manager.prepare_for_forward( batch, chunked_req=self.chunked_req ) + loop_batch = batch self.cur_batch_for_debug = batch # Skip batch creation next round to avoid repeated preparation failures. skip_batch_creation = batch is None @@ -835,7 +917,23 @@ class SchedulerDisaggregationPrefillMixin: if batch: if self.enable_staging: self.maybe_prefetch_staging_for_batch(batch) + _rb_t0 = time.time_ns() batch_result = self.run_batch(batch) + if _lt is not None: + try: + batch_telemetry.mark_run_batch( + _rb_t0, + time.time_ns(), + { + "sglang.forward_mode": str(batch.forward_mode), + "sglang.bs": batch.batch_size(), + "sglang.num_tokens": int( + getattr(batch, "extend_num_tokens", 0) or 0 + ), + }, + ) + except Exception: + pass self._apply_war_barrier() self.result_queue.append((batch.copy(), batch_result)) else: @@ -853,10 +951,21 @@ class SchedulerDisaggregationPrefillMixin: ): waiting_for_batch_result = True self._yield_gil_if_needed() + _end_loop(_lt, False, None, waiting_for_result=True) continue # Process the oldest batch result. oldest_batch, oldest_batch_result = self.result_queue.popleft() + _pr_t0 = time.time_ns() self.process_batch_result(oldest_batch, oldest_batch_result) + if _lt is not None: + try: + batch_telemetry.mark_process( + _pr_t0, + time.time_ns(), + {"sglang.n_reqs": len(oldest_batch.reqs)}, + ) + except Exception: + pass elif batch is None: # When the server is idle, do self-check and re-init some states self.on_idle() @@ -871,6 +980,7 @@ class SchedulerDisaggregationPrefillMixin: self._sched_idled = batch is None self.last_batch = batch waiting_for_batch_result = False + _end_loop(_lt, loop_batch is not None, loop_batch) def is_disagg_prefill_batch_result_ready( self: Scheduler, @@ -1511,7 +1621,10 @@ class SchedulerDisaggregationPrefillMixin: state_indices = state_indices_for_request(self, req, end_idx) value = computer.compute(page_indices_gpu, state_indices) self.disagg_metadata_buffers.set_kv_checksum(req, value) + _kv_t0 = time.time_ns() if batch_telemetry.kv_send_span_enabled() else 0 self._send_kv_chunk(req, last_chunk=last_chunk, end_idx=end_idx) + if _kv_t0: + batch_telemetry.add_kv_send(_kv_t0, time.time_ns()) def _send_kv_chunk( self: Scheduler, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 9bf630348a..42f16f2662 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1417,6 +1417,8 @@ class Scheduler( metrics_collector_context=self.metrics_collector_context, metrics_collector=self.metrics_collector, ) + # Read the already-running DeviceTimer; adds no CUDA events. + batch_telemetry.install_forward_reporter(self) def init_schedule_policy(self): # Init schedule policy and new token estimation diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index f3d50a943a..88a5fcd449 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -675,6 +675,15 @@ def prepare_mlp_sync_batch_raw( else: need_idle_batch = max(mlp_sync_info.global_num_tokens) > 0 + # Capture the pre-sync local kind before the idle replacement below: the + # loop span's local_batch_kind describes what this rank brought in, not the + # idle batch it may be asked to run for DP lockstep. + _loop_local_kind = ( + "none" + if local_batch is None + else ("idle" if local_batch.forward_mode.is_idle() else "extend") + ) + batch_to_gather = local_batch if need_idle_batch: if local_batch is None: @@ -683,6 +692,13 @@ def prepare_mlp_sync_batch_raw( # NOTE: for prebuilt batch, we add an inner idle batch to run MLP sync batch_to_gather = local_batch.inner_idle_batch = get_idle_batch() + _loop_global_outcome = ( + "idle" + if (_loop_local_kind == "none" and need_idle_batch) + else ("none" if _loop_local_kind == "none" else _loop_local_kind) + ) + batch_telemetry.note_mlp_sync_outcome(_loop_local_kind, _loop_global_outcome) + if batch_to_gather is not None: _update_gather_batch( batch_to_gather, diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index 191f38c325..31ccdf3ac9 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -663,10 +663,13 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> from opentelemetry import trace as otel_trace tracer = otel_trace.get_tracer("sglang.prefill.pass") + loop = getattr(_LOOP_TLS, "current", None) + loop_ctx = loop.span_ctx if loop is not None else None end_ns = time.time_ns() start_ns = end_ns - int(total * 1e9) tracer.start_span( "scheduler.prefill.pass", + context=loop_ctx, start_time=start_ns, attributes={ "sglang.phase": t.phase, @@ -687,6 +690,8 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), }, ).end(end_time=end_ns) + if loop is not None and loop_ctx is not None: + loop.pass_span_child = True except Exception: # Tracing must never break the scheduler loop. pass @@ -736,6 +741,7 @@ def end_pass(dp_rank: int) -> None: keyed["resumed_chunk"].inc(1) elif t.new_chunk: keyed["new_chunk"].inc(1) + note_pass_snapshot(t, total) _emit_pass_span(dp_rank, t, total, other) except Exception: # Telemetry must never break the scheduler loop @@ -817,6 +823,22 @@ def observe_recv( counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs) if n_tokens: counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens) + end_ns = time.time_ns() + start_ns = end_ns - int(total * 1e9) + note_recv_phase( + start_wall_ns=start_ns, + end_wall_ns=end_ns, + parts={ + "sglang.recv_total_ms": total * 1e3, + "sglang.recv_pull_ms": pull * 1e3, + "sglang.recv_broadcast_ms": broadcast * 1e3, + "sglang.recv_unwrap_ms": unwrap * 1e3, + "sglang.recv_mm_ms": mm * 1e3, + "sglang.recv_msgs": int(n_msgs), + "sglang.recv_prompt_tokens": int(n_tokens), + }, + ) + if not _RECV_SPAN_ENABLED: return if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int( @@ -826,10 +848,10 @@ def observe_recv( from opentelemetry import trace as otel_trace tracer = otel_trace.get_tracer("sglang.recv") - end_ns = time.time_ns() tracer.start_span( "scheduler.recv_requests", - start_time=end_ns - int(total * 1e9), + context=current_loop_context(), + start_time=start_ns, attributes={ "sglang.phase": "recv", "sglang.dp_rank": int(dp_rank), @@ -878,3 +900,530 @@ def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None except Exception: # Telemetry must never break the scheduler loop. pass + +# --------------------------------------------------------------------------- +# Scheduler loop span + no-batch accounting (opt-in, same stage-timing gate). +# +# One ``scheduler.loop`` span per scheduler iteration (sampled), with phase +# children (recv / get_next / run_batch / process_result / kv_send). The +# no-batch counters are incremented on *every* loop (sampling only controls +# spans), split by a low-cardinality reason and by whether a forward was in +# flight (forward_overlap). Fully idle loops never emit a span. +# +# Enable with ``SGLANG_TRACE_LOOP_SPAN=1`` (requires SGLANG_BATCH_STAGE_TIMING). +# Knobs: +# SGLANG_TRACE_LOOP_SAMPLE=0.05 head-sample ratio (deterministic per +# mlp_sync_seq, identical on all ranks) +# SGLANG_TRACE_FORWARD_SPAN=0 emit approximate forward.gpu lanes +# SGLANG_TRACE_KV_SEND_SPAN=1 emit the kv_send child (default on +# when the loop span is on) +# --------------------------------------------------------------------------- + +from collections import deque as _deque + +_LOOP_ENABLED = _STAGE_TIMING +_LOOP_SPAN_ENABLED = _LOOP_ENABLED and os.getenv( + "SGLANG_TRACE_LOOP_SPAN", "0" +).strip().lower() not in ("", "0", "false", "no", "off") +_LOOP_SPAN_SAMPLE = float(os.getenv("SGLANG_TRACE_LOOP_SAMPLE", "0.05")) +_FORWARD_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( + "SGLANG_TRACE_FORWARD_SPAN", "0" +).strip().lower() not in ("", "0", "false", "no", "off") +_KV_SEND_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( + "SGLANG_TRACE_KV_SEND_SPAN", "1" +).strip().lower() not in ("", "0", "false", "no", "off") + +_NO_BATCH_REASONS = ( + "fully_idle", + "waiting_prefetch", + "waiting_bootstrap", + "result_backpressure", + "deferred_input", + "admission_blocked", + "waiting_pd", + "no_ready_other", +) + +_LOOP_METRICS = None +_LOOP_TLS = threading.local() + +# Monotonic total of completed forward device time (from the DeviceTimer), +# used to derive one loop's gpu_ms without adding a second timer. +_FORWARD_DEVICE_NS_TOTAL = 0 +_FORWARD_EVENTS = _deque(maxlen=128) # (completion_wall_ns, elapsed_ns, category) + +# Set by dp_attn.prepare_mlp_sync_batch_raw() once per pass. +_LAST_MLP_OUTCOME = ("none", "none") +_LAST_NO_BATCH_REASON = "no_ready_other" + + +def loop_accounting_enabled() -> bool: + return _LOOP_ENABLED + + +def loop_span_enabled() -> bool: + return _LOOP_SPAN_ENABLED + + +def forward_span_enabled() -> bool: + return _FORWARD_SPAN_ENABLED + + +def kv_send_span_enabled() -> bool: + return _KV_SEND_SPAN_ENABLED + + +def loop_sample_decision(seq: int) -> bool: + """Deterministic, rank-identical loop-span sampling decision.""" + if not _LOOP_SPAN_ENABLED or seq <= 0: + return False + if _LOOP_SPAN_SAMPLE >= 1.0: + return True + if _LOOP_SPAN_SAMPLE <= 0.0: + return False + return (seq * 2654435761) % 10000 < int(_LOOP_SPAN_SAMPLE * 10000) + + +class LoopTimings: + __slots__ = ( + "phase", + "t0_ns", + "t0_wall_ns", + "predicted_seq", + "active0", + "has_work0", + "forward_total0", + "recv", + "get_next", + "run_batch", + "process", + "kv_send_start", + "kv_send_end", + "kv_chunks", + "span", + "span_ctx", + "pass_span_child", + "seq", + "no_batch_reason", + ) + + def __init__( + self, + phase: str, + t0_ns: int, + t0_wall_ns: int, + predicted_seq: int, + active0: bool, + has_work0: bool, + forward_total0: int, + ): + self.phase = phase + self.t0_ns = t0_ns + self.t0_wall_ns = t0_wall_ns + self.predicted_seq = predicted_seq + self.active0 = bool(active0) + self.has_work0 = bool(has_work0) + self.forward_total0 = forward_total0 + self.recv = None + self.get_next = None + self.run_batch = None + self.process = None + self.kv_send_start = None + self.kv_send_end = None + self.kv_chunks = 0 + self.span = None + self.span_ctx = None + self.pass_span_child = False + self.seq = 0 + self.no_batch_reason = "" + + +def begin_loop( + phase: str = "prefill", + *, + pre_has_work: bool = True, + forward_active=None, +): + """Start one scheduler-loop accounting record. + + ``pre_has_work`` must be cheap to compute (queue lengths / last_batch): + fully idle loops are still counted, but never create a span. + ``forward_active`` is a callable (usually DeviceTimer.is_active) or a bool. + """ + global _LAST_MLP_OUTCOME + if not _LOOP_ENABLED: + return None + try: + active0 = bool(forward_active() if callable(forward_active) else forward_active) + except Exception: + active0 = False + lt = LoopTimings( + phase=phase, + t0_ns=time.monotonic_ns(), + t0_wall_ns=time.time_ns(), + predicted_seq=_DP_GATHER_SEQ + 1, + active0=active0, + has_work0=bool(pre_has_work), + forward_total0=_FORWARD_DEVICE_NS_TOTAL, + ) + _LAST_MLP_OUTCOME = ("none", "none") + _LOOP_TLS.current = lt + if _LOOP_SPAN_ENABLED and lt.has_work0 and loop_sample_decision(lt.predicted_seq): + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.prefill.loop") + lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns) + lt.span_ctx = otel_trace.set_span_in_context(lt.span) + except Exception: + lt.span = None + lt.span_ctx = None + return lt + + +def current_loop() -> Optional[LoopTimings]: + return getattr(_LOOP_TLS, "current", None) + + +def current_loop_context(): + lt = getattr(_LOOP_TLS, "current", None) + return lt.span_ctx if lt is not None else None + + +def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None: + """Record the recv_requests phase inside the current loop (if any).""" + lt = getattr(_LOOP_TLS, "current", None) + if lt is not None: + lt.recv = (int(start_wall_ns), int(end_wall_ns), dict(parts)) + + +def note_pass_snapshot(t: "PassTimings", total_s: float) -> None: + """Record get_next timings for the current loop (called from end_pass).""" + lt = getattr(_LOOP_TLS, "current", None) + if lt is None: + return + end_wall_ns = time.time_ns() + start_wall_ns = end_wall_ns - int(total_s * 1e9) + measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync + other = max(0.0, total_s - measured) + lt.get_next = ( + start_wall_ns, + end_wall_ns, + { + "sglang.get_next_ms": total_s * 1e3, + "sglang.match_ms": t.match * 1e3, + "sglang.adder_ms": t.adder * 1e3, + "sglang.hicache_ms": t.hicache * 1e3, + "sglang.bootstrap_ms": t.bootstrap * 1e3, + "sglang.mlp_sync_ms": t.mlp_sync * 1e3, + "sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3, + "sglang.other_ms": other * 1e3, + "sglang.candidates": int(t.candidates), + "sglang.skipped_prefetch": int(t.skipped_prefetch), + "sglang.admitted": int(t.admitted), + "sglang.resumed_chunked": int(bool(t.had_chunked)), + "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), + "sglang.mlp_sync_seq": int(t.seq), + }, + ) + lt.seq = int(t.seq) + + +def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): + lt = getattr(_LOOP_TLS, "current", None) + if lt is not None: + lt.run_batch = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {})) + + +def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): + lt = getattr(_LOOP_TLS, "current", None) + if lt is not None: + lt.process = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {})) + + +def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None: + lt = getattr(_LOOP_TLS, "current", None) + if lt is None: + return + lt.kv_chunks += 1 + if lt.kv_send_start is None or start_wall_ns < lt.kv_send_start: + lt.kv_send_start = int(start_wall_ns) + if lt.kv_send_end is None or end_wall_ns > lt.kv_send_end: + lt.kv_send_end = int(end_wall_ns) + + +def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None: + """Set by dp_attn.prepare_mlp_sync_batch_raw once per pass.""" + global _LAST_MLP_OUTCOME + _LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome)) + + +def note_forward_device(t: float, category: str = "", **kwargs) -> None: + """DeviceTimer reporter: accumulate completed forward device time.""" + global _FORWARD_DEVICE_NS_TOTAL + try: + ns = int(float(t) * 1e9) + except Exception: + return + _FORWARD_DEVICE_NS_TOTAL += ns + if _FORWARD_SPAN_ENABLED: + _FORWARD_EVENTS.append((time.time_ns(), ns, str(category))) + + +def install_forward_reporter(scheduler) -> None: + """Attach the forward device-time reporter to the existing DeviceTimer. + + No CUDA event is added here: SGLANG_ENABLE_METRICS_DEVICE_TIMER already + runs the timer; this only reads its completed intervals. + """ + if not _LOOP_ENABLED: + return + try: + reporter = getattr(scheduler, "metrics_reporter", None) + timer = getattr(reporter, "forward_pass_device_timer", None) + if timer is not None: + timer.add_reporter(note_forward_device) + except Exception: + # Telemetry must never break scheduler init. + pass + + +def _ensure_loop_metrics(): + global _LOOP_METRICS + if _LOOP_METRICS is None: + from prometheus_client import Counter + + _LOOP_METRICS = { + "no_batch_seconds": Counter( + "sglang:scheduler_no_batch_seconds_total", + "Wall time of scheduler loops that ended without a runnable batch, " + "by reason and forward_overlap", + ["dp_rank", "reason", "forward_overlap"], + ), + "no_batch_total": Counter( + "sglang:scheduler_no_batch_total", + "Count of scheduler loops that ended without a runnable batch, " + "by reason and forward_overlap", + ["dp_rank", "reason", "forward_overlap"], + ), + "lockstep_seconds": Counter( + "sglang:scheduler_dp_lockstep_idle_seconds_total", + "Wall time of DP-lockstep idle-forward loops (this rank had no " + "local batch but the DP group required an idle forward)", + ["dp_rank", "forward_overlap"], + ), + } + return _LOOP_METRICS + + +def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str: + """Low-cardinality no-batch reason, in the design's priority order. + + ``pass_snapshot`` is the get_next attribute dict (candidates / skipped / + admitted). It is None when the loop skipped batch preparation entirely. + """ + if state.get("result_queue", 0) >= state.get("result_limit", 2): + return "result_backpressure" + if state.get("deferred", 0) > 0 or state.get("paused"): + return "deferred_input" + waiting = int(state.get("waiting", 0) or 0) + if waiting > 0: + if state.get("waiting_all_bootstrap"): + return "waiting_bootstrap" + if pass_snapshot is not None: + candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0) + skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0) + admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0) + if candidates > 0 and skipped == candidates: + return "waiting_prefetch" + if admitted == 0: + return "admission_blocked" + inflight = int(state.get("inflight", 0) or 0) + bootstrap = int(state.get("bootstrap", 0) or 0) + if waiting == 0 and (inflight or bootstrap): + return "waiting_pd" + if ( + waiting == 0 + and inflight == 0 + and bootstrap == 0 + and int(state.get("result_queue", 0) or 0) == 0 + and int(state.get("deferred", 0) or 0) == 0 + and not state.get("waiting_for_result") + ): + return "fully_idle" + if state.get("skip_batch_creation") and pass_snapshot is None: + # A failed preparation defers retries; keep the last classified reason. + return _LAST_NO_BATCH_REASON + return "no_ready_other" + + +def _batch_attrs(batch) -> dict: + if batch is None: + return { + "sglang.n_reqs": 0, + "sglang.n_new": 0, + "sglang.n_cached": 0, + "sglang.chunked": False, + } + try: + n_reqs = int(batch.batch_size()) + except Exception: + n_reqs = len(getattr(batch, "reqs", ()) or ()) + try: + n_new = int(getattr(batch, "extend_num_tokens", 0) or 0) + except Exception: + n_new = 0 + try: + n_cached = int(sum(getattr(batch, "prefix_lens", ()) or ())) + except Exception: + n_cached = 0 + try: + chunked = getattr(batch, "chunked_req", None) is not None + except Exception: + chunked = False + return { + "sglang.n_reqs": n_reqs, + "sglang.n_new": n_new, + "sglang.n_cached": n_cached, + "sglang.chunked": bool(chunked), + } + + +def end_loop( + lt: Optional[LoopTimings], + *, + dp_rank: int, + has_batch: bool, + batch=None, + state: Optional[dict] = None, +) -> None: + """Close one loop: counters (always), loop span + phases (if sampled).""" + if lt is None: + return + _LOOP_TLS.current = None + state = state or {} + try: + now_ns = time.monotonic_ns() + now_wall_ns = time.time_ns() + loop_s = max(0.0, (now_ns - lt.t0_ns) / 1e9) + gpu_ns = max(0, _FORWARD_DEVICE_NS_TOTAL - lt.forward_total0) + active1 = bool(state.get("forward_active", False)) + overlap = ( + "full" + if lt.active0 and active1 + else "partial" + if (lt.active0 or active1) + else "none" + ) + local_kind, global_outcome = _LAST_MLP_OUTCOME + pass_snapshot = lt.get_next[2] if lt.get_next is not None else None + reason = "" + + if not has_batch: + try: + metrics = _ensure_loop_metrics() + if local_kind == "none" and global_outcome == "idle": + metrics["lockstep_seconds"].labels( + dp_rank=str(dp_rank), forward_overlap=overlap + ).inc(loop_s) + else: + reason = _classify_no_batch(state, pass_snapshot) + global _LAST_NO_BATCH_REASON + _LAST_NO_BATCH_REASON = reason + metrics["no_batch_seconds"].labels( + dp_rank=str(dp_rank), + reason=reason, + forward_overlap=overlap, + ).inc(loop_s) + metrics["no_batch_total"].labels( + dp_rank=str(dp_rank), + reason=reason, + forward_overlap=overlap, + ).inc() + except Exception: + # Telemetry must never break the scheduler loop. + pass + lt.no_batch_reason = reason + + if lt.span is None: + return + + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.prefill.loop") + pending = bool( + state.get("waiting", 0) + or state.get("result_queue", 0) + or state.get("inflight", 0) + or state.get("bootstrap", 0) + or state.get("waiting_for_result") + ) + attrs = { + "sglang.phase": lt.phase, + "sglang.dp_rank": int(dp_rank), + "sglang.loop_seq": int(lt.seq or lt.predicted_seq), + "sglang.loop_ms": loop_s * 1e3, + "sglang.gpu_ms": gpu_ns / 1e6, + "sglang.gpu_idle_ms": max(0.0, loop_s * 1e3 - gpu_ns / 1e6), + "sglang.has_batch": bool(has_batch), + "sglang.engine_has_pending_work": pending, + "sglang.local_batch_kind": local_kind, + "sglang.global_outcome": global_outcome, + "sglang.no_batch_reason": reason, + "sglang.forward_overlap": overlap, + } + attrs.update(_batch_attrs(batch)) + lt.span.set_attributes(attrs) + + def _child(name, start_ns, end_ns, child_attrs): + try: + tracer.start_span( + name, + context=lt.span_ctx, + start_time=int(start_ns), + attributes=child_attrs, + ).end(end_time=int(end_ns)) + except Exception: + pass + + if lt.recv is not None: + start, end, parts = lt.recv + child = {"sglang.dp_rank": int(dp_rank)} + child.update(parts) + _child("scheduler.loop.recv", start, end, child) + if lt.get_next is not None and not lt.pass_span_child: + start, end, ng_attrs = lt.get_next + _child("scheduler.loop.get_next", start, end, ng_attrs) + if lt.run_batch is not None: + start, end, rb_attrs = lt.run_batch + _child("scheduler.loop.run_batch", start, end, rb_attrs) + if lt.process is not None: + start, end, pr_attrs = lt.process + _child("scheduler.loop.process_result", start, end, pr_attrs) + if _KV_SEND_SPAN_ENABLED and lt.kv_send_start is not None: + _child( + "scheduler.loop.kv_send", + lt.kv_send_start, + lt.kv_send_end, + {"sglang.n_chunks": int(lt.kv_chunks)}, + ) + if _FORWARD_SPAN_ENABLED: + for completion_wall, elapsed_ns, category in list(_FORWARD_EVENTS): + if completion_wall < lt.t0_wall_ns or completion_wall > now_wall_ns: + continue + _child( + "scheduler.loop.forward.gpu", + completion_wall - elapsed_ns, + completion_wall, + { + "sglang.gpu_ms": elapsed_ns / 1e6, + "sglang.category": category, + }, + ) + lt.span.end(end_time=now_wall_ns) + except Exception: + # Telemetry must never break the scheduler loop. + pass + finally: + _LOOP_TLS.current = None diff --git a/test/registered/unit/observability/test_loop_span_accounting.py b/test/registered/unit/observability/test_loop_span_accounting.py new file mode 100644 index 0000000000..a416fc3bbf --- /dev/null +++ b/test/registered/unit/observability/test_loop_span_accounting.py @@ -0,0 +1,189 @@ +"""Unit tests for the scheduler loop span + no-batch accounting. + +Covers the design's acceptance points: + * deterministic, rank-identical loop-span sampling by mlp_sync_seq; + * no-batch reason priority and the no_ready_other fallback; + * the no-batch counters are independent of span sampling; + * DP lockstep idle is counted separately from pure no-batch wait; + * forward device-time accumulation feeds the loop gpu_ms source. +""" +import importlib.util +import os +import pathlib +import unittest +from unittest import mock + + +def _module_path(): + p = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/observability/batch_telemetry.py" + ) + if p.exists(): + return p + spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry") + return pathlib.Path(spec.origin) + + +def _load(**env): + base = { + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_TRACE_LOOP_SPAN": "0", + "SGLANG_TRACE_LOOP_SAMPLE": "0.05", + "SGLANG_TRACE_FORWARD_SPAN": "0", + } + base.update(env) + with mock.patch.dict(os.environ, base): + spec = importlib.util.spec_from_file_location( + "batch_telemetry_loop_under_test", _module_path() + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class _FakeCounter: + def __init__(self): + self.data = {} + + def labels(self, **kw): + self._key = tuple(sorted(kw.items())) + return self + + def inc(self, value=1.0): + self.data[self._key] = self.data.get(self._key, 0.0) + value + + +def _fake_metrics(): + fakes = { + "no_batch_seconds": _FakeCounter(), + "no_batch_total": _FakeCounter(), + "lockstep_seconds": _FakeCounter(), + } + return fakes + + +class LoopAccountingTest(unittest.TestCase): + def test_sampling_is_deterministic_and_bounded(self): + mod = _load() + mod._LOOP_SPAN_ENABLED = True + mod._LOOP_SPAN_SAMPLE = 1.0 + self.assertTrue(mod.loop_sample_decision(12345)) + self.assertTrue(mod.loop_sample_decision(12345)) + self.assertFalse(mod.loop_sample_decision(0)) + mod._LOOP_SPAN_SAMPLE = 0.0 + self.assertFalse(mod.loop_sample_decision(1)) + mod._LOOP_SPAN_SAMPLE = 0.05 + a = [mod.loop_sample_decision(i) for i in range(200)] + b = [mod.loop_sample_decision(i) for i in range(200)] + self.assertEqual(a, b) + + def test_no_batch_reason_priority(self): + mod = _load() + cases = [ + ({"result_queue": 2}, None, "result_backpressure"), + ({"deferred": 1}, None, "deferred_input"), + ({"waiting": 2, "waiting_all_bootstrap": True}, None, "waiting_bootstrap"), + ( + {"waiting": 2}, + { + "sglang.candidates": 5, + "sglang.skipped_prefetch": 5, + "sglang.admitted": 0, + }, + "waiting_prefetch", + ), + ( + {"waiting": 2}, + { + "sglang.candidates": 5, + "sglang.skipped_prefetch": 1, + "sglang.admitted": 0, + }, + "admission_blocked", + ), + ({"waiting": 0, "inflight": 1}, None, "waiting_pd"), + ( + { + "waiting": 0, + "inflight": 0, + "bootstrap": 0, + "result_queue": 0, + "deferred": 0, + }, + None, + "fully_idle", + ), + ] + for state, snapshot, expected in cases: + self.assertEqual( + mod._classify_no_batch(state, snapshot), expected, (state, snapshot) + ) + + def test_forward_device_time_accumulates(self): + mod = _load() + before = mod._FORWARD_DEVICE_NS_TOTAL + mod.note_forward_device(0.0025, "extend") + mod.note_forward_device(0.0075, "extend") + self.assertGreaterEqual(mod._FORWARD_DEVICE_NS_TOTAL - before, 10_000_000) + + def test_no_batch_counters_do_not_depend_on_span_sampling(self): + mod = _load() + fakes = _fake_metrics() + mod._ensure_loop_metrics = lambda: fakes + + def run_once(): + mod._LAST_MLP_OUTCOME = ("none", "none") + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.note_mlp_sync_outcome("none", "none") + mod.end_loop( + lt, + dp_rank=0, + has_batch=False, + state={ + "waiting": 0, + "inflight": 0, + "bootstrap": 0, + "result_queue": 0, + "deferred": 0, + "waiting_for_result": False, + "forward_active": False, + }, + ) + + mod._LOOP_SPAN_ENABLED = False + run_once() + mod._LOOP_SPAN_ENABLED = True + mod._LOOP_SPAN_SAMPLE = 0.0 + run_once() + self.assertEqual(sum(fakes["no_batch_total"].data.values()), 2.0) + keys = list(fakes["no_batch_total"].data) + self.assertTrue(any(("reason", "fully_idle") in k for k in keys)) + + def test_dp_lockstep_idle_is_not_no_batch(self): + mod = _load() + fakes = _fake_metrics() + mod._ensure_loop_metrics = lambda: fakes + mod._LAST_MLP_OUTCOME = ("none", "idle") + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.note_mlp_sync_outcome("none", "idle") + mod.end_loop( + lt, + dp_rank=1, + has_batch=False, + state={ + "waiting": 0, + "inflight": 0, + "bootstrap": 0, + "result_queue": 0, + "deferred": 0, + "waiting_for_result": False, + "forward_active": False, + }, + ) + self.assertFalse(fakes["no_batch_seconds"].data) + self.assertTrue(fakes["lockstep_seconds"].data) + + +if __name__ == "__main__": + unittest.main()