From 2372edd73d65e7c5a840b201ccb0d69a9769cb0b Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Tue, 17 Nov 2026 22:27:55 +0800 Subject: [PATCH] observability: skip batch-span links for TraceNullContext requests TraceNullContext forwards every attribute to itself, so the previous root_span.get_span_context() probe returned the null context itself and otel's exporter failed with 'expected bytes, TraceNullContext found'. Gate on tracing_enable before probing. --- .../srt/observability/batch_telemetry.py | 5490 +++++++++-------- 1 file changed, 2748 insertions(+), 2742 deletions(-) diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index 61df34f93d..d092f51843 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -1,2742 +1,2748 @@ -"""Per-batch scheduler telemetry (opt-in). - -Adds, at the *batch* granularity (one record per scheduler batch per DP rank): - - * Prometheus histograms (safe for the multiprocess collector): - sglang:batch_new_tokens{phase,dp_rank} - sglang:batch_cached_tokens{phase,dp_rank} - sglang:batch_duration_seconds{phase,dp_rank} - sglang:batch_queue_depth{phase,dp_rank} - sglang:batch_pending_tokens{phase,dp_rank} - sglang:batch_n_reqs{phase,dp_rank} - * An OTLP span ``scheduler.batch.{phase}`` with ``Link``s to the request - traces that took part in the batch (batch -> requests navigation), and a - ``sglang.batch_id`` attribute for reverse lookups / log correlation. - -Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs: - - SGLANG_BATCH_SPAN=0 histograms only (no spans) - SGLANG_BATCH_SPAN_MIN_MS=50 only emit spans for batches slower than this - SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0) - SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span - - SGLANG_TRACE_LOOP_DETAIL=1 split run_batch/process_result into host - phases (requires SGLANG_BATCH_STAGE_TIMING=1 - and SGLANG_TRACE_LOOP_SPAN=1; only sampled - loops are recorded) - SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown - (queue/source/indices/transfer/join) - SGLANG_TRACE_BACKUP_DETAIL=1 per-flush HiCache backup sub-phases - (one ``scheduler.cache_flush_backup`` span; - requires SGLANG_TRACE_LOOP_DETAIL=1) - -Cost model (measured on our 4xDP4 / 4-GPU instances): - * histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU. - * span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s. - Everything is behind a single module-level bool, so when disabled the hot - path pays one ``if not _ENABLED: return``. - -NOTE: sglang runs prometheus_client in multiprocess mode, which does not -carry exemplars, so the span trace id is returned to the caller and appended -to the batch log line instead (see ``metrics_reporter``). -""" - -from __future__ import annotations - -import os -import random -import threading -import time -from typing import Any, Iterable, Optional - -_ENABLED = os.getenv("SGLANG_BATCH_TELEMETRY", "0").strip().lower() not in ( - "", - "0", - "false", - "no", - "off", -) -_SPAN_ENABLED = _ENABLED and os.getenv("SGLANG_BATCH_SPAN", "1").strip().lower() not in ( - "0", - "false", - "no", - "off", -) -_SPAN_MIN_S = float(os.getenv("SGLANG_BATCH_SPAN_MIN_MS", "0")) / 1000.0 -_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_SPAN_SAMPLE", "1.0")) -_MAX_LINKS = int(os.getenv("SGLANG_BATCH_SPAN_MAX_LINKS", "32")) - -_HISTOGRAMS = None -_CHILDREN: dict[tuple[str, str, int], Any] = {} -_TRACER = None - - -def enabled() -> bool: - return _ENABLED - - -_REQUEST_COUNTER = None -_DP_RANK = None -_COMPLETION_RANK = None - - -def _should_report_completion() -> bool: - """Whether this scheduler is the canonical reporter of request completions. - - Every TP/CP rank runs a scheduler and observes the same requests, and a PP - stage only sees its own microbatches, so an ungated counter would be - multiplied across replicas once the multiprocess registry aggregates them. - Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same - shape the existing scheduler metrics use for rank-unique series. - """ - global _COMPLETION_RANK - if _COMPLETION_RANK is None: - try: - from sglang.srt.runtime_context import get_parallel - - parallel = get_parallel() - _COMPLETION_RANK = bool( - parallel.attn_tp_rank == 0 - and parallel.attn_cp_rank == 0 - and parallel.pp_group.is_last_rank - ) - except Exception: - # Unknown topology (e.g. a bare unit-test process): keep counting. - _COMPLETION_RANK = True - return _COMPLETION_RANK - - -def _current_dp_rank() -> int: - """This process's attention-DP rank (0 for non-DP engines).""" - global _DP_RANK - if _DP_RANK is None: - try: - from sglang.srt.runtime_context import get_parallel - - _DP_RANK = int(get_parallel().attn_dp_rank or 0) - except Exception: - _DP_RANK = 0 - return _DP_RANK - - -def _ensure_request_counter(): - global _REQUEST_COUNTER - if _REQUEST_COUNTER is None: - from prometheus_client import Counter - - _REQUEST_COUNTER = Counter( - "sglang:batch_requests_total", - "Requests completed by the canonical scheduler rank, by phase. " - "phase=prefill: prefill compute finished and the request entered the " - "PD transfer queue (not a guarantee that KV transfer succeeded). " - "phase=decode: request finished decoding.", - ["phase", "dp_rank"], - ) - return _REQUEST_COUNTER - - -def note_request_completed(phase: str, n: int = 1) -> None: - """Count finished requests per DP rank. - - This is the per-rank RPS source: the frontend ``num_requests_total`` has no - rank label, while the scheduler knows exactly which DP shard completed the - request. Cheap (one counter increment per finished request) and gated by - SGLANG_BATCH_TELEMETRY like the other batch metrics. - """ - if not _ENABLED or n <= 0 or not _should_report_completion(): - return - try: - _ensure_request_counter().labels( - phase=phase, dp_rank=str(_current_dp_rank()) - ).inc(int(n)) - except Exception: - # Telemetry must never fail a request. - pass - - -def _ensure_histograms(): - global _HISTOGRAMS - if _HISTOGRAMS is None: - from prometheus_client import Histogram - - labels = ["phase", "dp_rank"] - _HISTOGRAMS = { - "batch_new_tokens": Histogram( - "sglang:batch_new_tokens", - "New (uncached) prompt tokens processed by one scheduler batch", - labels, - buckets=(128, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536), - ), - "batch_cached_tokens": Histogram( - "sglang:batch_cached_tokens", - "Cached prompt tokens carried by one scheduler batch", - labels, - buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576), - ), - "batch_duration_seconds": Histogram( - "sglang:batch_duration_seconds", - "Wall time spent in one scheduler batch", - labels, - buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0), - ), - "batch_queue_depth": Histogram( - "sglang:batch_queue_depth", - "Waiting-queue depth observed by one scheduler batch", - labels, - buckets=(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512), - ), - "batch_pending_tokens": Histogram( - "sglang:batch_pending_tokens", - "Prefill backlog (pending tokens) observed by one scheduler batch", - labels, - buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576), - ), - "batch_n_reqs": Histogram( - "sglang:batch_n_reqs", - "Requests carried by one scheduler batch", - labels, - buckets=(1, 2, 4, 8, 16, 32, 64, 128, 256), - ), - } - return _HISTOGRAMS - - -def _child(name: str, phase: str, dp_rank: int): - key = (name, phase, dp_rank) - child = _CHILDREN.get(key) - if child is None: - child = _ensure_histograms()[name].labels(phase=phase, dp_rank=str(dp_rank)) - _CHILDREN[key] = child - return child - - -def _request_span_context(req: Any): - """SpanContext of a request, from either a live span or a serialized copy. - - Scheduler processes receive a *copy* of the request trace context where - ``root_span`` is None (see ``trace_async.copy_for_thread``) and only - ``root_span_context`` (an OTel ``Context``) survives. Handle both. - """ - trace_ctx = getattr(getattr(req, "time_stats", None), "trace_ctx", None) - if trace_ctx is None: - return None - try: - root_span = getattr(trace_ctx, "root_span", None) - if root_span is not None: - span_ctx = root_span.get_span_context() - if span_ctx.is_valid: - return span_ctx - root_ctx = getattr(trace_ctx, "root_span_context", None) - if root_ctx is not None: - from opentelemetry import trace as otel_trace - - span_ctx = otel_trace.get_current_span(root_ctx).get_span_context() - if span_ctx.is_valid: - return span_ctx - except Exception: - return None - return None - - -def _emit_span( - phase: str, - dp_rank: int, - iter_no: int, - n_reqs: int, - n_new: int, - n_cached: int, - queue_depth: int, - pending_tokens: int, - duration_s: float, - reqs: Optional[Iterable[Any]], -) -> Optional[str]: - """Create one batch span with links to the request traces. Returns trace id.""" - if not _SPAN_ENABLED: - return None - if _SPAN_MIN_S > 0 and duration_s < _SPAN_MIN_S: - return None - if _SPAN_SAMPLE < 1.0 and random.random() > _SPAN_SAMPLE: - return None - global _TRACER - try: - from opentelemetry import trace as otel_trace - from opentelemetry.trace import Link - - if _TRACER is None: - _TRACER = otel_trace.get_tracer("sglang.batch") - links = [] - if reqs: - for req in reqs: - if len(links) >= _MAX_LINKS: - break - ctx = _request_span_context(req) - if ctx is not None: - links.append(Link(ctx)) - end_ns = time.time_ns() - start_ns = end_ns - int(duration_s * 1e9) - span = _TRACER.start_span( - f"scheduler.batch.{phase}", - start_time=start_ns, - attributes={ - "sglang.batch_id": f"{dp_rank}:{iter_no}", - "sglang.dp_rank": int(dp_rank), - "sglang.phase": phase, - "sglang.n_reqs": int(n_reqs), - "sglang.n_new_tokens": int(n_new), - "sglang.n_cached_tokens": int(n_cached), - "sglang.queue_depth": int(queue_depth), - "sglang.pending_tokens": int(pending_tokens), - "sglang.batch_duration_s": float(duration_s), - }, - links=links, - ) - span.end(end_time=end_ns) - span_ctx = span.get_span_context() - if not span_ctx.is_valid: - # No TracerProvider configured (tracing disabled) -> NoOp span. - return None - return format(span_ctx.trace_id, "032x") - except Exception: - return None - - -def _observe( - phase: str, - dp_rank: int, - iter_no: int, - n_reqs: int, - n_new: int, - n_cached: int, - queue_depth: int, - pending_tokens: int, - duration_s: float, - reqs: Optional[Iterable[Any]], -) -> Optional[str]: - if not _ENABLED: - return None - try: - trace_id = _emit_span( - phase, - dp_rank, - iter_no, - n_reqs, - n_new, - n_cached, - queue_depth, - pending_tokens, - duration_s, - reqs, - ) - _child("batch_new_tokens", phase, dp_rank).observe(n_new) - _child("batch_cached_tokens", phase, dp_rank).observe(n_cached) - _child("batch_duration_seconds", phase, dp_rank).observe(duration_s) - _child("batch_queue_depth", phase, dp_rank).observe(queue_depth) - _child("batch_pending_tokens", phase, dp_rank).observe(pending_tokens) - _child("batch_n_reqs", phase, dp_rank).observe(n_reqs) - return trace_id - except Exception: - # Telemetry must never break the scheduler loop. - return None - - -def observe_prefill( - *, - dp_rank: int, - iter_no: int, - n_reqs: int, - n_new: int, - n_cached: int, - queue_depth: int, - pending_tokens: int, - duration_s: float, - reqs: Optional[Iterable[Any]] = None, -) -> Optional[str]: - return _observe( - "prefill", - dp_rank, - iter_no, - n_reqs, - n_new, - n_cached, - queue_depth, - pending_tokens, - duration_s, - reqs, - ) - - -def observe_decode( - *, - dp_rank: int, - iter_no: int, - n_reqs: int, - n_new: int, - n_cached: int, - queue_depth: int, - pending_tokens: int, - duration_s: float, - reqs: Optional[Iterable[Any]] = None, -) -> Optional[str]: - return _observe( - "decode", - dp_rank, - iter_no, - n_reqs, - n_new, - n_cached, - queue_depth, - pending_tokens, - duration_s, - reqs, - ) - -_DP_HISTS = None - - -def _ensure_dp_histograms(): - global _DP_HISTS - if _DP_HISTS is None: - from prometheus_client import Histogram - - _DP_HISTS = { - "dp_step_tokens": Histogram( - "sglang:dp_step_tokens", - "Per-rank token count in one DP-attention step", - ["dp_rank", "phase"], - buckets=(0, 256, 1024, 2048, 4096, 8192, 16384, 32768), - ), - "dp_step_padding_ratio": Histogram( - "sglang:dp_step_padding_ratio", - "1 - mean/max tokens across active DP ranks in one step (padding waste)", - ["phase", "n_active"], - buckets=(0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8), - ), - "dp_step_spread": Histogram( - "sglang:dp_step_spread", - "(max-min)/mean tokens across active DP ranks in one step", - ["phase", "n_active"], - buckets=(0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0), - ), - } - return _DP_HISTS - - -_DP_IDLE_MODE = 4 # ForwardMode.IDLE - - -def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = "unknown") -> None: - """Record per-step shape alignment across *active* DP ranks. - - ``tokens``/``modes``/``health`` are the per-rank vectors from the DP - all-gather (info tensor columns 0, 5, 9). IDLE ranks and health-check - steps are excluded so the padding ratio reflects real batch-shape - misalignment rather than idle-slot padding. - """ - if not _ENABLED: - return - try: - if health and any(bool(h) for h in health): - return - sel = [] - for i, t in enumerate(tokens): - if t is None: - continue - t = int(t) - if t <= 1: - continue - if modes is not None and i < len(modes) and int(modes[i]) == _DP_IDLE_MODE: - continue - sel.append(t) - if len(sel) < 2: - return - hist = _ensure_dp_histograms() - mx = max(sel) - mean = sum(sel) / len(sel) - n_active = str(len(sel)) - key = ("dp_step_tokens", phase, dp_rank) - child = _CHILDREN.get(key) - if child is None: - child = hist["dp_step_tokens"].labels(dp_rank=str(dp_rank), phase=phase) - _CHILDREN[key] = child - child.observe(int(tokens[dp_rank]) if 0 <= dp_rank < len(tokens) else int(mean)) - if mx > 0: - hist["dp_step_padding_ratio"].labels(phase=phase, n_active=n_active).observe( - 1.0 - mean / mx - ) - if mean > 0: - hist["dp_step_spread"].labels(phase=phase, n_active=n_active).observe( - (mx - min(sel)) / mean - ) - except Exception: - pass - - -# --------------------------------------------------------------------------- -# Per-pass stage timing (SGLANG_BATCH_STAGE_TIMING=1) -- P (disagg prefill). -# -# One sample per get_next_disagg_prefill_batch_to_run() pass, including passes -# that produce no batch. Disabled by default; when off every hook below is a -# single module-level bool check. No CUDA calls, no per-pass logging. -# -# Cross-rank analysis uses a dedicated, independently sampled span -# ``scheduler.prefill.pass`` emitted from end_pass(): the sampling decision is a -# pure function of ``mlp_sync_seq`` (identical on every DP rank that took part -# in the same gather), so a sampled gather round is sampled on *all* ranks -- -# unlike the per-rank random sampling of the batch span. Idle ranks and ranks -# with no local batch therefore still emit, which is exactly the case the -# skew analysis needs. -# --------------------------------------------------------------------------- - -_STAGE_TIMING = os.getenv("SGLANG_BATCH_STAGE_TIMING", "0").strip().lower() not in ( - "", - "0", - "false", - "no", - "off", -) - -_STAGE_LABELS = ["phase", "dp_rank"] -_STAGE_BUCKETS = ( - 1e-5, - 3e-5, - 1e-4, - 3e-4, - 1e-3, - 3e-3, - 1e-2, - 3e-2, - 1e-1, - 3e-1, - 1.0, - 3.0, -) - -_STAGE_HISTS = None -_STAGE_COUNTERS = None -_STAGE_CHILDREN: dict = {} -_STAGE_TLS = threading.local() - -# Set by the DP all-gather path (dp_attn.prepare_mlp_sync_batch_raw). -_LAST_DP_GATHER = None -_DP_GATHER_SEQ = 0 - -_PASS_SPAN_ENABLED = _STAGE_TIMING and os.getenv( - "SGLANG_BATCH_PASS_SPAN", "1" -).strip().lower() not in ("", "0", "false", "no", "off") -_PASS_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_PASS_SPAN_SAMPLE", "0.05")) - -_STAGE_HIST_DOCS = { - "get_next": "Wall time of one get_next_disagg_prefill_batch_to_run pass", - "match": "Per-pass sum of init_next_round_input (prefix match) over candidates", - "adder": "Per-pass sum of PrefillAdder.add_one_req (budget/admission)", - "hicache_events": "Per-pass time in _process_hicache_events (includes its collectives)", - "bootstrap": "Per-pass time in resolve_waiting_queue_bootstrap (poll + all-reduce)", - "mlp_sync": "Per-pass outer maybe_prepare_mlp_sync_batch time (not pure gather wait)", - "other": "get_next total minus the instrumented segments", -} - - -def stage_timing_enabled() -> bool: - return _STAGE_TIMING - - -def pass_span_enabled() -> bool: - return _PASS_SPAN_ENABLED - - -def pass_span_decision(seq: int) -> bool: - """Deterministic, rank-identical sampling decision for one gather round. - - ``seq`` is the DP all-gather sequence number, which is the same on every - rank that participated in that round. Therefore one decision covers the - whole round, instead of four independent ``random.random()`` draws. - """ - if not _PASS_SPAN_ENABLED or seq <= 0: - return False - if _PASS_SPAN_SAMPLE >= 1.0: - return True - if _PASS_SPAN_SAMPLE <= 0.0: - return False - return (seq * 2654435761) % 10000 < int(_PASS_SPAN_SAMPLE * 10000) - - -def _ensure_stage_metrics(): - global _STAGE_HISTS, _STAGE_COUNTERS - if _STAGE_HISTS is None: - from prometheus_client import Counter, Histogram - - _STAGE_HISTS = { - name: Histogram( - f"sglang:batch_prefill_{name}_seconds", - doc, - _STAGE_LABELS, - buckets=_STAGE_BUCKETS, - ) - for name, doc in _STAGE_HIST_DOCS.items() - } - _STAGE_COUNTERS = { - "candidates": Counter( - "sglang:batch_prefill_candidates_total", - "Requests examined by the per-pass admission candidate loop", - _STAGE_LABELS, - ), - "skipped_prefetch": Counter( - "sglang:batch_prefill_skipped_prefetch_total", - "Candidates skipped because their L2/L3 prefetch was not ready", - _STAGE_LABELS, - ), - "admitted": Counter( - "sglang:batch_prefill_admitted_total", - "Requests admitted into can_run_list by one pass", - _STAGE_LABELS, - ), - "resumed_chunk": Counter( - "sglang:batch_prefill_resumed_chunk_total", - "Passes whose batch continues a chunked-prefill request " - "(including its last chunk)", - _STAGE_LABELS, - ), - "new_chunk": Counter( - "sglang:batch_prefill_new_chunk_total", - "Passes that start a new chunked-prefill request", - _STAGE_LABELS, - ), - } - return _STAGE_HISTS, _STAGE_COUNTERS - - -class PassTimings: - __slots__ = ( - "phase", - "t0", - "match", - "adder", - "hicache", - "bootstrap", - "mlp_sync", - "pending_abort", - "chunk", - "new_batch", - "set_sched", - "chunk_count", - "dp_gather", - "candidates", - "skipped_prefetch", - "admitted", - "bootstrap_candidates", - "pending_bootstrap", - "had_chunked", - "new_chunk", - "seq", - ) - - def __init__(self, phase: str): - self.phase = phase - self.t0 = time.monotonic() - self.match = 0.0 - self.adder = 0.0 - self.hicache = 0.0 - self.bootstrap = 0.0 - self.mlp_sync = 0.0 - self.pending_abort = 0.0 - self.chunk = 0.0 - self.new_batch = 0.0 - self.set_sched = 0.0 - self.chunk_count = 0 - self.dp_gather = None - self.candidates = 0 - self.skipped_prefetch = 0 - self.admitted = 0 - self.bootstrap_candidates = 0 - self.pending_bootstrap = 0 - self.had_chunked = False - self.new_chunk = False - self.seq = 0 - - -def begin_pass(phase: str = "prefill") -> Optional[PassTimings]: - """Start a new pass. Returns None when stage timing is disabled.""" - if not _STAGE_TIMING: - return None - timings = PassTimings(phase) - _STAGE_TLS.current = timings - return timings - - -def _current() -> Optional[PassTimings]: - if not _STAGE_TIMING: - return None - return getattr(_STAGE_TLS, "current", None) - - -def add_match(seconds: float) -> None: - t = _current() - if t is not None: - t.match += seconds - - -def add_adder(seconds: float) -> None: - t = _current() - if t is not None: - t.adder += seconds - - -def add_hicache(seconds: float) -> None: - t = _current() - if t is not None: - t.hicache += seconds - - -def add_bootstrap(seconds: float) -> None: - t = _current() - if t is not None: - t.bootstrap += seconds - - -def add_pending_abort(seconds: float) -> None: - t = _current() - if t is not None: - t.pending_abort += seconds - - -def add_chunk(seconds: float, count: int = 0) -> None: - t = _current() - if t is not None: - t.chunk += seconds - t.chunk_count += int(count) - - -def add_new_batch(seconds: float) -> None: - t = _current() - if t is not None: - t.new_batch += seconds - - -def add_set_sched(seconds: float) -> None: - t = _current() - if t is not None: - t.set_sched += seconds - - -def mark_candidate(n: int = 1) -> None: - t = _current() - if t is not None: - t.candidates += n - - -def mark_skipped_prefetch(n: int = 1) -> None: - t = _current() - if t is not None: - t.skipped_prefetch += n - - -def set_admitted(n: int) -> None: - t = _current() - if t is not None: - t.admitted = int(n) - - -def set_bootstrap_pending(pending: int, candidates: int) -> None: - """Reuse the bootstrap resolver's existing pass; do not rescan the queue.""" - t = _current() - if t is not None: - t.pending_bootstrap = int(pending) - t.bootstrap_candidates = int(candidates) - - -def note_had_chunked(flag: bool) -> None: - """Record whether a chunked request was already in flight when the pass began. - - Must be called at pass start, before process_prefill_chunk()/admission can - consume the chunked request (the last chunk clears self.chunked_req). - """ - t = _current() - if t is not None: - t.had_chunked = bool(flag) - - -def note_new_chunk(flag: bool) -> None: - """Record whether this pass started a new chunked request (first chunk).""" - t = _current() - if t is not None: - t.new_chunk = bool(flag) - - -def note_dp_gather(seconds: Optional[float]) -> None: - """Called by the DP all-gather path; None means no gather ran this pass.""" - global _LAST_DP_GATHER, _DP_GATHER_SEQ - if not _STAGE_TIMING: - return - if seconds is None: - _LAST_DP_GATHER = None - return - _LAST_DP_GATHER = float(seconds) - _DP_GATHER_SEQ += 1 - - -def _take_dp_gather(): - global _LAST_DP_GATHER - value = _LAST_DP_GATHER - _LAST_DP_GATHER = None - return value - - -def add_mlp_sync(seconds: float) -> None: - t = _current() - if t is None: - return - t.mlp_sync += seconds - gather = _take_dp_gather() - if gather is not None: - t.dp_gather = gather - t.seq = _DP_GATHER_SEQ - - -def _emit_pass_span( - dp_rank: int, - t: PassTimings, - total: float, - other: float, - residual: float = 0.0, -) -> None: - if not pass_span_decision(t.seq): - return - loop = getattr(_LOOP_TLS, "current", None) - loop_ctx = loop.span_ctx if loop is not None else None - if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None: - # An unsampled loop owns this pass; do not leak an independent pass trace. - # If work is discovered later, end_loop emits the snapshot as its child. - return - try: - from opentelemetry import trace as otel_trace - - tracer = otel_trace.get_tracer("sglang.prefill.pass") - end_ns = time.time_ns() - start_ns = end_ns - int(total * 1e9) - tracer.start_span( - "scheduler.prefill.pass", - context=loop_ctx, - start_time=start_ns, - attributes={ - "sglang.phase": t.phase, - "sglang.dp_rank": int(dp_rank), - "sglang.mlp_sync_seq": int(t.seq), - "sglang.get_next_ms": total * 1e3, - "sglang.match_ms": t.match * 1e3, - "sglang.adder_ms": t.adder * 1e3, - "sglang.hicache_ms": t.hicache * 1e3, - "sglang.bootstrap_ms": t.bootstrap * 1e3, - "sglang.mlp_sync_ms": t.mlp_sync * 1e3, - "sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3, - "sglang.other_ms": other * 1e3, - "sglang.pending_abort_ms": t.pending_abort * 1e3, - "sglang.chunk_ms": t.chunk * 1e3, - "sglang.new_batch_ms": t.new_batch * 1e3, - "sglang.set_sched_ms": t.set_sched * 1e3, - "sglang.get_next_residual_ms": residual * 1e3, - "sglang.chunk_count": int(t.chunk_count), - "sglang.candidates": int(t.candidates), - "sglang.skipped_prefetch": int(t.skipped_prefetch), - "sglang.admitted": int(t.admitted), - "sglang.resumed_chunked": int(bool(t.had_chunked)), - "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), - }, - ).end(end_time=end_ns) - if loop is not None and loop_ctx is not None: - loop.pass_span_child = True - except Exception: - # Tracing must never break the scheduler loop. - pass - - -def end_pass(dp_rank: int) -> None: - """Close the current pass: observe histograms/counters and emit the pass span.""" - t = _current() - if t is None: - return - _STAGE_TLS.current = None - try: - total = time.monotonic() - t.t0 - measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync - other = total - measured - if other < 0.0: - other = 0.0 - # Residual after also charging the get_next sub-phases. - measured2 = measured + ( - t.pending_abort + t.chunk + t.new_batch + t.set_sched - ) - residual = total - measured2 - if residual < 0.0: - residual = 0.0 - hist, counters = _ensure_stage_metrics() - child_key = (t.phase, dp_rank) - keyed = _STAGE_CHILDREN.get(child_key) - if keyed is None: - keyed = { - name: h.labels(phase=t.phase, dp_rank=str(dp_rank)) - for name, h in hist.items() - } - keyed.update( - { - name: c.labels(phase=t.phase, dp_rank=str(dp_rank)) - for name, c in counters.items() - } - ) - _STAGE_CHILDREN[child_key] = keyed - keyed["get_next"].observe(total) - keyed["match"].observe(t.match) - keyed["adder"].observe(t.adder) - keyed["hicache_events"].observe(t.hicache) - keyed["bootstrap"].observe(t.bootstrap) - keyed["mlp_sync"].observe(t.mlp_sync) - keyed["other"].observe(other) - if t.candidates: - keyed["candidates"].inc(t.candidates) - if t.skipped_prefetch: - keyed["skipped_prefetch"].inc(t.skipped_prefetch) - if t.admitted: - keyed["admitted"].inc(t.admitted) - if t.had_chunked: - keyed["resumed_chunk"].inc(1) - elif t.new_chunk: - keyed["new_chunk"].inc(1) - note_pass_snapshot(t, total) - _emit_pass_span(dp_rank, t, total, other, residual) - except Exception: - # Telemetry must never break the scheduler loop - _STAGE_TLS.current = None - -# --------------------------------------------------------------------------- -# Scheduler receive path (same SGLANG_BATCH_STAGE_TIMING gate): split the -# recv_requests stage into pull / broadcast / unwrap / mm. -# -# NOTE: tokenization itself runs in the tokenizer worker processes (HTTP side). -# What is measured here is the scheduler-side IPC: ZMQ pull + decode of the -# tokenized request objects, the cross-rank broadcast of those objects, and -# the unpickling of their token-id buffers. Large prompts make all three -# visible (e.g. 14.6k tokens per request). -# --------------------------------------------------------------------------- - -_RECV_SPAN_ENABLED = _STAGE_TIMING and os.getenv( - "SGLANG_RECV_SPAN", "1" -).strip().lower() not in ("", "0", "false", "no", "off") -_RECV_SPAN_SAMPLE = float(os.getenv("SGLANG_RECV_SPAN_SAMPLE", "0.05")) - -_RECV_PARTS = ("total", "pull", "broadcast", "unwrap", "mm") -_RECV_HISTS = None -_RECV_COUNTERS = None -_RECV_SEQ = 0 - - -def recv_timing_enabled() -> bool: - return _STAGE_TIMING - - -def _ensure_recv_metrics(): - global _RECV_HISTS, _RECV_COUNTERS - if _RECV_HISTS is None: - from prometheus_client import Counter, Histogram - - _RECV_HISTS = Histogram( - "sglang:recv_requests_seconds", - "Scheduler recv_requests wall time by part " - "(total/pull/broadcast/unwrap/mm)", - ["dp_rank", "part"], - buckets=_STAGE_BUCKETS, - ) - _RECV_COUNTERS = { - "msgs": Counter( - "sglang:recv_requests_msgs_total", - "Messages received by the scheduler recv path", - ["dp_rank"], - ), - "prompt_tokens": Counter( - "sglang:recv_requests_prompt_tokens_total", - "Prompt tokens carried by messages received in the recv path", - ["dp_rank"], - ), - } - return _RECV_HISTS, _RECV_COUNTERS - - -def observe_recv( - *, - dp_rank: int, - total: float, - pull: float, - broadcast: float, - unwrap: float, - mm: float, - n_msgs: int, - n_tokens: int, -) -> None: - if not _STAGE_TIMING: - return - global _RECV_SEQ - _RECV_SEQ += 1 - try: - hist, counters = _ensure_recv_metrics() - for part, value in zip(_RECV_PARTS, (total, pull, broadcast, unwrap, mm)): - hist.labels(dp_rank=str(dp_rank), part=part).observe(value) - if n_msgs: - counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs) - if n_tokens: - counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens) - end_ns = time.time_ns() - start_ns = end_ns - int(total * 1e9) - note_recv_phase( - start_wall_ns=start_ns, - end_wall_ns=end_ns, - parts={ - "sglang.recv_total_ms": total * 1e3, - "sglang.recv_pull_ms": pull * 1e3, - "sglang.recv_broadcast_ms": broadcast * 1e3, - "sglang.recv_unwrap_ms": unwrap * 1e3, - "sglang.recv_mm_ms": mm * 1e3, - "sglang.recv_msgs": int(n_msgs), - "sglang.recv_prompt_tokens": int(n_tokens), - }, - ) - - if not _RECV_SPAN_ENABLED: - return - if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int( - _RECV_SPAN_SAMPLE * 10000 - ): - return - loop = current_loop() - loop_ctx = current_loop_context() - if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None: - # Keep the snapshot only; end_loop emits it if this loop has work. - return - from opentelemetry import trace as otel_trace - - tracer = otel_trace.get_tracer("sglang.recv") - tracer.start_span( - "scheduler.recv_requests", - context=loop_ctx, - start_time=start_ns, - attributes={ - "sglang.phase": "recv", - "sglang.dp_rank": int(dp_rank), - "sglang.recv_msgs": int(n_msgs), - "sglang.recv_prompt_tokens": int(n_tokens), - "sglang.recv_total_ms": total * 1e3, - "sglang.recv_pull_ms": pull * 1e3, - "sglang.recv_broadcast_ms": broadcast * 1e3, - "sglang.recv_unwrap_ms": unwrap * 1e3, - "sglang.recv_mm_ms": mm * 1e3, - }, - ).end(end_time=end_ns) - if loop is not None and loop_ctx is not None: - loop.recv_span_child = True - except Exception: - # Telemetry must never break the scheduler loop. - pass - - -# --------------------------------------------------------------------------- -# Storage prefetch ready latency (same SGLANG_BATCH_STAGE_TIMING gate). -# -# Request-level wait from "prefetch in flight" to "usable" (the scheduler -# check returns True), split by L3 hit/miss. Under wait_complete this is the -# admission delay a request pays for its L3 lookup. -# --------------------------------------------------------------------------- - -_PREFETCH_READY_HIST = None - - -def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None: - if not _STAGE_TIMING: - return - global _PREFETCH_READY_HIST - try: - if _PREFETCH_READY_HIST is None: - from prometheus_client import Histogram - - _PREFETCH_READY_HIST = Histogram( - "sglang:storage_prefetch_ready_seconds", - "Prefetch in-flight -> usable latency, split by L3 hit/miss", - ["dp_rank", "result"], - buckets=(1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0, 3.0, 10.0, 30.0), - ) - _PREFETCH_READY_HIST.labels(dp_rank=str(dp_rank), result=result).observe( - seconds - ) - except Exception: - # Telemetry must never break the scheduler loop. - pass - -# --------------------------------------------------------------------------- -# Scheduler loop span + no-batch accounting (opt-in, same stage-timing gate). -# -# One ``scheduler.loop`` span per scheduler iteration (sampled), with phase -# children (recv / get_next / run_batch / process_result / kv_send). The -# no-batch counters are incremented on *every* loop (sampling only controls -# spans), split by a low-cardinality reason and by whether a forward was in -# flight (forward_overlap). Fully idle loops never emit a span. -# -# Enable with ``SGLANG_TRACE_LOOP_SPAN=1`` (requires SGLANG_BATCH_STAGE_TIMING). -# Knobs: -# SGLANG_TRACE_LOOP_SAMPLE=0.05 head-sample ratio (deterministic per -# mlp_sync_seq, identical on all ranks) -# SGLANG_TRACE_FORWARD_SPAN=0 emit approximate forward.gpu lanes -# SGLANG_TRACE_KV_SEND_SPAN=1 emit the kv_send child (default on -# when the loop span is on) -# --------------------------------------------------------------------------- - -from collections import deque as _deque - -_LOOP_ENABLED = _STAGE_TIMING -_LOOP_SPAN_ENABLED = _LOOP_ENABLED and os.getenv( - "SGLANG_TRACE_LOOP_SPAN", "0" -).strip().lower() not in ("", "0", "false", "no", "off") -_LOOP_SPAN_SAMPLE = float(os.getenv("SGLANG_TRACE_LOOP_SAMPLE", "0.05")) -_FORWARD_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( - "SGLANG_TRACE_FORWARD_SPAN", "0" -).strip().lower() not in ("", "0", "false", "no", "off") -_KV_SEND_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( - "SGLANG_TRACE_KV_SEND_SPAN", "1" -).strip().lower() not in ("", "0", "false", "no", "off") - -_NO_BATCH_REASONS = ( - "fully_idle", - "waiting_prefetch", - "waiting_bootstrap", - "result_backpressure", - "deferred_input", - "admission_blocked", - "waiting_pd", - "no_ready_other", -) - -_LOOP_METRICS = None -_LOOP_TLS = threading.local() - -# Monotonic total of completed forward device time (from the DeviceTimer), -# used to derive one loop's gpu_ms without adding a second timer. -_FORWARD_DEVICE_NS_TOTAL = 0 -_FORWARD_EVENTS = _deque(maxlen=128) # (completion_wall_ns, elapsed_ns, category) - -# Set by dp_attn.prepare_mlp_sync_batch_raw() once per pass. -_LAST_MLP_OUTCOME = ("none", "none") -_LAST_NO_BATCH_REASON = "no_ready_other" - - -def loop_accounting_enabled() -> bool: - return _LOOP_ENABLED - - -def loop_span_enabled() -> bool: - return _LOOP_SPAN_ENABLED - - -def forward_span_enabled() -> bool: - return _FORWARD_SPAN_ENABLED - - -def kv_send_span_enabled() -> bool: - if not _KV_SEND_SPAN_ENABLED: - return False - lt = getattr(_LOOP_TLS, "current", None) - return lt is not None and lt.span is not None - - -def loop_sample_decision(seq: int) -> bool: - """Deterministic, rank-identical loop-span sampling decision.""" - if not _LOOP_SPAN_ENABLED or seq <= 0: - return False - if _LOOP_SPAN_SAMPLE >= 1.0: - return True - if _LOOP_SPAN_SAMPLE <= 0.0: - return False - return (seq * 2654435761) % 10000 < int(_LOOP_SPAN_SAMPLE * 10000) - - -class LoopTimings: - __slots__ = ( - "phase", - "t0_ns", - "t0_wall_ns", - "predicted_seq", - "active0", - "has_work0", - "forward_total0", - "recv", - "get_next", - "run_batch", - "process", - "kv_send_start", - "kv_send_end", - "kv_chunks", - "span", - "span_ctx", - "pass_span_child", - "recv_span_child", - "launch_blocked_reason", - "seq", - "no_batch_reason", - "pass_end_wall_ns", - "arrival_est_wall_ns", - "detail", - ) - - def __init__( - self, - phase: str, - t0_ns: int, - t0_wall_ns: int, - predicted_seq: int, - active0: bool, - has_work0: bool, - forward_total0: int, - ): - self.phase = phase - self.t0_ns = t0_ns - self.t0_wall_ns = t0_wall_ns - self.predicted_seq = predicted_seq - self.active0 = bool(active0) - self.has_work0 = bool(has_work0) - self.forward_total0 = forward_total0 - self.recv = None - self.get_next = None - self.run_batch = None - self.process = None - self.kv_send_start = None - self.kv_send_end = None - self.kv_chunks = 0 - self.span = None - self.span_ctx = None - self.pass_span_child = False - self.recv_span_child = False - self.launch_blocked_reason = "" - self.seq = 0 - self.no_batch_reason = "" - self.pass_end_wall_ns = 0 - self.arrival_est_wall_ns = 0 - # Only populated when SGLANG_TRACE_LOOP_DETAIL=1 and this loop is - # sampled; see the "Loop detail phases" section below. - self.detail = None - - -def ensure_loop_span(lt: Optional[LoopTimings]) -> None: - """Materialize a sampled span once the loop is known to have work.""" - if ( - lt is None - or lt.span is not None - or not loop_sample_decision(lt.predicted_seq) - ): - return - try: - from opentelemetry import trace as otel_trace - - tracer = otel_trace.get_tracer("sglang.prefill.loop") - lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns) - lt.span_ctx = otel_trace.set_span_in_context(lt.span) - except Exception: - lt.span = None - lt.span_ctx = None - - -def note_launch_blocked(lt: Optional[LoopTimings], reason: str) -> None: - """Preserve why this loop could not launch before queues are consumed.""" - if lt is not None and not lt.launch_blocked_reason: - lt.launch_blocked_reason = reason - - -def begin_loop( - phase: str = "prefill", - *, - pre_has_work: bool = True, - forward_active=None, -): - """Start one scheduler-loop accounting record. - - ``pre_has_work`` must be cheap to compute (queue lengths / last_batch): - fully idle loops are still counted, but never create a span. - ``forward_active`` is a callable (usually DeviceTimer.is_active) or a bool. - """ - global _LAST_MLP_OUTCOME - if not _LOOP_ENABLED: - return None - try: - active0 = bool(forward_active() if callable(forward_active) else forward_active) - except Exception: - active0 = False - lt = LoopTimings( - phase=phase, - t0_ns=time.monotonic_ns(), - t0_wall_ns=time.time_ns(), - predicted_seq=_DP_GATHER_SEQ + 1, - active0=active0, - has_work0=bool(pre_has_work), - forward_total0=_FORWARD_DEVICE_NS_TOTAL, - ) - _LAST_MLP_OUTCOME = ("none", "none") - _LOOP_TLS.current = lt - if lt.has_work0: - ensure_loop_span(lt) - return lt - - -def current_loop() -> Optional[LoopTimings]: - return getattr(_LOOP_TLS, "current", None) - - -def current_loop_context(): - lt = getattr(_LOOP_TLS, "current", None) - return lt.span_ctx if lt is not None else None - - -# --------------------------------------------------------------------------- -# Loop detail phases (SGLANG_TRACE_LOOP_DETAIL=1). -# -# Splits run_batch / process_result into host phases. Gated twice: the module -# switch must be on AND the current loop must be sampled (same deterministic -# decision as the loop span), so detail data never covers an unsampled loop. -# -# Accounting rules (see handoff/prefill-cpu-kv-overlap-design-20261106.md §3.2): -# * single phases -> one child span (``scheduler.loop.``) -# * repeated phases -> summed attributes only (``sglang._ms`` + count) -# * durations use time.monotonic_ns(); OTLP start/end use time.time_ns() -# * residuals are clamped >= 0 and reported, never folded into a phase -# --------------------------------------------------------------------------- - -_DETAIL_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( - "SGLANG_TRACE_LOOP_DETAIL", "0" -).strip().lower() not in ("", "0", "false", "no", "off") - -_DETAIL_TLS = threading.local() - -# Chunk-level (Mooncake transfer worker) detail tracing. Independent switch: -# the worker runs on other threads and has no scheduler loop TLS. -_MOONCAKE_DETAIL_ENABLED = os.getenv( - "SGLANG_TRACE_MOONCAKE_DETAIL", "0" -).strip().lower() not in ("", "0", "false", "no", "off") - -# Previous loop summary per scheduler thread, used for DP-skew correlation -# (previous loop's phases vs this loop's arrival skew). -_LAST_LOOP_SUMMARY: dict = {} - -# Phases that are inherently per-request / per-segment: always attributes, -# even when a loop happens to have exactly one occurrence, so span count never -# grows with n_reqs / n_segments (design §4.2). -_DETAIL_ATTRIBUTE_ONLY_PREFIXES = ( - "per_request_result", - "prefix_device_indices", - "kv_state_payload", - "kv_page_indices", - "kv_sender_send", -) - -_DETAIL_PREV_KEYS = ( - "run_batch_ms", - "process_result_ms", - "copy_done_wait_ms", - "kv_state_payload_ms", - "kv_page_indices_ms", - "kv_sender_send_ms", -) - - -def loop_detail_enabled() -> bool: - """True when detail is on and the *current* loop is sampled.""" - if not _DETAIL_ENABLED: - return False - lt = getattr(_LOOP_TLS, "current", None) - return lt is not None and loop_sample_decision(lt.predicted_seq) - - -def mooncake_detail_enabled() -> bool: - return _MOONCAKE_DETAIL_ENABLED - - -def _detail_state(): - """Detail state of the current loop, only when that loop is sampled.""" - if not _DETAIL_ENABLED: - return None - lt = getattr(_LOOP_TLS, "current", None) - if lt is None or not loop_sample_decision(lt.predicted_seq): - return None - if lt.detail is None: - lt.detail = {"phases": {}, "counts": {}, "sums": {}} - return lt.detail - - -def detail_begin(name: str) -> Optional[int]: - """Start a single-occurrence phase. Returns a monotonic-ns token.""" - if not _DETAIL_ENABLED: - return None - state = _detail_state() - if state is None: - return None - now_ns = time.monotonic_ns() - opens = getattr(_DETAIL_TLS, "open", None) - if opens is None: - opens = {} - _DETAIL_TLS.open = opens - opens[name] = (now_ns, time.time_ns(), _thread_time_ns()) - return now_ns - - -def detail_end(name: str, token: Optional[int] = None) -> None: - """Close a phase opened by ``detail_begin`` (or started at ``token``).""" - if not _DETAIL_ENABLED: - return - state = _detail_state() - if state is None: - return - now_ns = time.monotonic_ns() - opens = getattr(_DETAIL_TLS, "open", None) or {} - entry = opens.pop(name, None) - start_wall_ns = entry[1] if entry is not None else None - thread_cpu_ns = None - if entry is not None and len(entry) > 2 and entry[2] is not None: - cpu_now = _thread_time_ns() - if cpu_now is not None: - thread_cpu_ns = max(0, cpu_now - int(entry[2])) - if token is None: - if entry is None: - return - token = entry[0] - if start_wall_ns is None: - start_wall_ns = time.time_ns() - max(0, now_ns - int(token)) - _detail_record( - state, - name, - int(token), - now_ns, - start_wall_ns, - time.time_ns(), - thread_cpu_ns=thread_cpu_ns, - ) - - -def detail_add( - name: str, - start_ns: int, - end_ns: int, - thread_cpu_ns: Optional[int] = None, -) -> None: - """Accumulate one occurrence of a phase (no span per occurrence). - - ``thread_cpu_ns`` is optional: callers that did not measure CPU time must - leave it as None so reports can tell "not measured" apart from "zero". - """ - if not _DETAIL_ENABLED or start_ns is None or end_ns is None: - return - state = _detail_state() - if state is None: - return - start_ns = int(start_ns) - end_ns = int(end_ns) - if end_ns < start_ns: - end_ns = start_ns - now_wall_ns = time.time_ns() - _detail_record( - state, - name, - start_ns, - end_ns, - now_wall_ns - (end_ns - start_ns), - now_wall_ns, - thread_cpu_ns=thread_cpu_ns, - ) - - -def _detail_record( - state, - name, - start_ns, - end_ns, - start_wall_ns, - end_wall_ns, - thread_cpu_ns: Optional[int] = None, -) -> None: - rec = state["phases"].get(name) - if rec is None: - rec = [end_ns - start_ns, 1, int(start_wall_ns), int(end_wall_ns), 0, 0] - state["phases"][name] = rec - else: - rec[0] += end_ns - start_ns - rec[1] += 1 - rec[3] = int(end_wall_ns) - if thread_cpu_ns is not None: - rec[4] += int(thread_cpu_ns) - rec[5] += 1 - - -def thread_cpu_ns() -> Optional[int]: - """Thread CPU clock, or None when the platform has no such clock. - - Callers outside this module (the scheduler's parent span) must use this - wrapper so an unsupported or failing clock degrades to "not measured" - instead of raising on the scheduling path. - """ - try: - return time.thread_time_ns() - except (AttributeError, OSError, ValueError): - return None - - -# Backwards-compatible private alias for module-local call sites. -_thread_time_ns = thread_cpu_ns - - -def detail_attr_setdefault(name: str, value) -> None: - """Set a loop attribute only if this loop has not recorded it yet.""" - if not _DETAIL_ENABLED: - return - state = _detail_state() - if state is None: - return - attrs = state.setdefault("attrs", {}) - if name not in attrs: - attrs[name] = value - - -def detail_attr_sum(name: str, value: int) -> None: - """Accumulate a numeric loop attribute (e.g. evicted tokens).""" - if not _DETAIL_ENABLED: - return - state = _detail_state() - if state is None: - return - sums = state.setdefault("sums", {}) - sums[name] = int(sums.get(name, 0)) + int(value) - - -def detail_count(name: str, n: int = 1) -> None: - """Accumulate a low-cardinality counter for this loop.""" - if not _DETAIL_ENABLED: - return - state = _detail_state() - if state is None: - return - state["counts"][name] = int(state["counts"].get(name, 0)) + int(n) - - -def detail_attr(name: str, value) -> None: - """Set a low-cardinality loop attribute (last write wins).""" - if not _DETAIL_ENABLED: - return - state = _detail_state() - if state is None: - return - state.setdefault("attrs", {})[name] = value - - -def detail_phase_total_ns(names) -> int: - """Sum of accumulated ns for the given phases in the current loop. - - ``names`` are *base* names: a phase recorded as ``kv_sender_send.last`` - (per send kind) is included when ``kv_sender_send`` is requested. Phase - names are only generated by this module, so prefix matching cannot pick up - an unrelated phase that merely shares a leading string. - """ - state = _detail_state() - if state is None: - return 0 - phases = state.get("phases") or {} - total = 0 - for name, rec in phases.items(): - if rec is None: - continue - for want in names: - if name == want or name.startswith(want + "."): - total += int(rec[0]) - break - return total - - -class detail_phase: - """Context manager for one single-occurrence phase (cheap when disabled).""" - - __slots__ = ("name", "token") - - def __init__(self, name: str): - self.name = name - self.token = None - - def __enter__(self): - self.token = detail_begin(self.name) - return self - - def __exit__(self, exc_type, exc, tb): - detail_end(self.name, self.token) - return False - - -# --------------------------------------------------------------------------- -# HiCache flush micro timing (SGLANG_TRACE_BACKUP_DETAIL=1). -# -# One tracer per ``cache_controller.start_writing()`` call (i.e. per -# ``flush_pending_backups``). Sub-phases are host-wall timings on the caller -# thread; a synchronous ``.cpu()`` therefore shows up as a long host phase and -# must not be read as device-transfer time. The tracer keeps per-flush -# resolution by emitting ONE child span of the current loop -# (``scheduler.cache_flush_backup``), because the loop-level detail only keeps -# per-loop sums (design: handoff/prefill-flush-backups-validation-20261107.md). -# -# Disabled by default: every entry point is a single module-level bool check, -# and when off no object, dict or span is created. -# --------------------------------------------------------------------------- - -_BACKUP_DETAIL_ENABLED = _DETAIL_ENABLED and os.getenv( - "SGLANG_TRACE_BACKUP_DETAIL", "0" -).strip().lower() not in ("", "0", "false", "no", "off") - -_FLUSH_TLS = threading.local() - - -def backup_detail_enabled() -> bool: - """True when per-flush micro timing is compiled in (``*_DETAIL`` is on).""" - return _BACKUP_DETAIL_ENABLED - - -def set_flush_origin(origin: Optional[str]) -> Optional[str]: - """Tag the next flushes with the caller's scheduler context. - - Only used to tell ``process_batch_result`` flushes (the current suspect) - from the ones issued by ``check_hicache_events`` / ``write``. Returns the - previous value so the caller can restore it. - """ - if not _BACKUP_DETAIL_ENABLED: - return None - previous = getattr(_FLUSH_TLS, "origin", None) - _FLUSH_TLS.origin = origin - return previous - - -def current_flush_origin() -> str: - return getattr(_FLUSH_TLS, "origin", None) or "other" - - -class BackupFlushTracer: - """Sub-phase timings for one flush; emits a single span on ``end``.""" - - __slots__ = ( - "origin", - "start_ns", - "loop_seq", - "loop_ctx", - "phases", - "attrs", - "counts", - "_open", - "_prev", - ) - - def __init__(self, origin: str): - self.origin = origin - # Captured on the scheduler thread: the flushed span may be emitted by - # the async submit worker, which has no scheduler loop TLS. - self.loop_seq = current_loop_seq() - self.loop_ctx = current_loop_context_for_spans() - self.start_ns = time.monotonic_ns() - self.phases: dict[str, int] = {} - self.attrs: dict[str, Any] = {} - self.counts: dict[str, int] = {} - self._open: dict[str, int] = {} - self._prev = None - - # -- phase helpers ----------------------------------------------------- - def begin(self, name: str) -> int: - start = time.monotonic_ns() - self._open[name] = start - return start - - def end(self, name: str, start_ns: Optional[int] = None) -> None: - start = self._open.pop(name, None) - if start is None: - start = start_ns - if start is None: - return - self.add(name, start) - - def add(self, name: str, start_ns: int) -> None: - self.record(name, int(start_ns), time.monotonic_ns()) - - def record(self, name: str, start_ns: int, end_ns: int) -> None: - self.phases[name] = self.phases.get(name, 0) + max( - 0, int(end_ns) - int(start_ns) - ) - - def set(self, key: str, value: Any) -> None: - self.attrs[key] = value - - def inc(self, key: str, n: int = 1) -> None: - self.counts[key] = self.counts.get(key, 0) + int(n) - - def total_ns(self) -> int: - return max(0, time.monotonic_ns() - self.start_ns) - - # -- emission ---------------------------------------------------------- - def end_and_emit(self) -> None: - now_ns = time.monotonic_ns() - total_ns = max(0, now_ns - self.start_ns) - attrs = { - "sglang.flush_origin": self.origin, - "sglang.flush_total_ms": total_ns / 1e6, - "sglang.loop_seq": int(self.loop_seq or 0), - } - for name, ns in self.phases.items(): - attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6 - for name, value in self.counts.items(): - attrs["sglang.flush_" + name] = int(value) - attrs.update(self.attrs) - # OTel rejects None; keep a single malformed dimension from dropping the - # whole span. - attrs = {k: v for k, v in attrs.items() if v is not None} - try: - from opentelemetry import trace as otel_trace - - ctx = self.loop_ctx - start_wall_ns = time.time_ns() - total_ns - span = otel_trace.get_tracer("sglang.cache.flush").start_span( - "scheduler.cache_flush_backup", - context=ctx, - start_time=int(start_wall_ns), - attributes=attrs, - ) - span.end(end_time=int(time.time_ns())) - except Exception: - # Telemetry must never break the cache controller. - pass - - -class backup_phase: - """Context manager for one sub-phase of the active flush (no-op when off).""" - - __slots__ = ("name", "tracer", "start_ns") - - def __init__(self, name: str): - self.name = name - self.tracer = None - self.start_ns = None - - def __enter__(self): - if not _BACKUP_DETAIL_ENABLED: - return self - self.tracer = getattr(_FLUSH_TLS, "tracer", None) - if self.tracer is not None: - self.start_ns = self.tracer.begin(self.name) - return self - - def __exit__(self, exc_type, exc, tb): - if self.tracer is not None: - self.tracer.end(self.name, self.start_ns) - return False - - -def backup_flush_begin(origin: Optional[str] = None) -> Optional[BackupFlushTracer]: - """Start tracing one flush; returns None when disabled / not sampled.""" - if not _BACKUP_DETAIL_ENABLED: - return None - if not loop_detail_enabled(): - return None - tracer = BackupFlushTracer(origin or current_flush_origin()) - tracer._prev = getattr(_FLUSH_TLS, "tracer", None) - _FLUSH_TLS.tracer = tracer - return tracer - - -def backup_flush_end(tracer: Optional[BackupFlushTracer]) -> None: - if tracer is None: - return - if getattr(_FLUSH_TLS, "tracer", None) is tracer: - _FLUSH_TLS.tracer = tracer._prev - try: - tracer.end_and_emit() - except Exception: - pass - - -def current_flush_tracer() -> Optional[BackupFlushTracer]: - if not _BACKUP_DETAIL_ENABLED: - return None - return getattr(_FLUSH_TLS, "tracer", None) - - -def backup_flush_attach(tracer: Optional[BackupFlushTracer]): - """Bind ``tracer`` to this thread so sub-phases land on the same span. - - Used by the async submit worker, which executes phases on a different - thread than the one that created the tracer. Returns the previous binding. - """ - if not _BACKUP_DETAIL_ENABLED: - return None - previous = getattr(_FLUSH_TLS, "tracer", None) - _FLUSH_TLS.tracer = tracer - return previous - - -def backup_flush_detach(previous) -> None: - if not _BACKUP_DETAIL_ENABLED: - return - _FLUSH_TLS.tracer = previous - - -def flush_note(key: str, value: Any) -> None: - """Attach a dimension to the active flush (no-op when disabled).""" - if not _BACKUP_DETAIL_ENABLED: - return - tracer = getattr(_FLUSH_TLS, "tracer", None) - if tracer is not None: - tracer.set(key, value) - - -def flush_count(key: str, n: int = 1) -> None: - """Accumulate a counter on the active flush (no-op when disabled).""" - if not _BACKUP_DETAIL_ENABLED: - return - tracer = getattr(_FLUSH_TLS, "tracer", None) - if tracer is not None: - tracer.inc(key, n) - - -def flush_mark() -> Optional[int]: - """Timestamp for a manual phase (None when no flush is being traced).""" - if not _BACKUP_DETAIL_ENABLED: - return None - if getattr(_FLUSH_TLS, "tracer", None) is None: - return None - return time.monotonic_ns() - - -def flush_phase_since(name: str, start_ns: Optional[int]) -> None: - """Record ``name`` from ``start_ns`` to now (no-op when untraced).""" - if start_ns is None: - return - tracer = getattr(_FLUSH_TLS, "tracer", None) - if tracer is not None: - tracer.add(name, start_ns) - - -def current_loop_seq() -> int: - """Loop sequence for cross-record correlation (0 when unavailable).""" - lt = getattr(_LOOP_TLS, "current", None) - if lt is None: - return 0 - return int(lt.seq or lt.predicted_seq or 0) - - -def current_loop_context_for_spans(): - """OTel context of the current loop span (None if unsampled).""" - lt = getattr(_LOOP_TLS, "current", None) - return lt.span_ctx if lt is not None else None - - -def _chunk_parent_context(chunk): - trace_ctx = getattr(chunk, "trace_ctx", None) - return getattr(trace_ctx, "root_span_context", None) - - -def split_queue_wait( - queue_wait_accum_ns: int, last_enqueue_mono_ns: int, dequeue_mono_ns: int -) -> tuple: - """Return ``(attempt_queue_wait_ns, total_queue_wait_ns)``. - - The attempt value matches the last_enqueue->dequeue wall window used by the - ``mooncake.queue_wait`` child span; the total value additionally includes - the queue waits of earlier deferred attempts (design §5.3 metric). - Keeping them separate prevents a span whose wall length disagrees with its - ``wait_ms`` attribute (review finding). - """ - attempt = max(0, int(dequeue_mono_ns) - int(last_enqueue_mono_ns)) - return attempt, max(0, int(queue_wait_accum_ns)) + attempt - - -class RoomDetailState: - """Thread-safe per-room detail bookkeeping (scheduler vs worker threads). - - The scheduler aborts/retracts a room while the transfer worker may still be - finishing an attempt. A reset therefore records a cancellation mark, and - the worker's join update re-checks under the same lock before writing, so a - late worker can never resurrect stats after abort (review finding). - """ - - # Bounded: a marker only needs to survive until the in-flight worker - # attempt finishes (milliseconds). The cap prevents unbounded growth if an - # aborted room never gets a worker teardown. - _MAX_CANCELLED = 8192 - - def __init__(self): - self._lock = threading.Lock() - self._join: dict = {} - self._seq: dict = {} - self._first: dict = {} - self._cancelled: dict = {} - - def _mark_cancelled(self, room: int) -> None: - self._cancelled[int(room)] = time.monotonic_ns() - while len(self._cancelled) > self._MAX_CANCELLED: - self._cancelled.pop(next(iter(self._cancelled))) - - def begin_chunk(self, room: int, now_mono_ns: int, now_wall_ns: int): - """Stamp a real enqueue. Returns ``(chunk_seq, first_enqueue_wall_ns)``. - - A new chunk clears a previous cancellation mark: it is the start of a - fresh flow (a retry may land on the same room id). - """ - with self._lock: - self._cancelled.pop(int(room), None) - seq = int(self._seq.get(int(room), 0)) - self._seq[int(room)] = seq + 1 - first = self._first.get(int(room)) - if first is None: - first = (int(now_mono_ns), int(now_wall_ns)) - self._first[int(room)] = first - return seq, int(first[1]) - - def record_join( - self, - room: int, - *, - status: str, - room_failed: bool, - now_wall_ns: int, - queue_wait_total_ns: int, - worker_busy_ns: int, - enqueue_wall_ns: int, - is_last_chunk: bool, - chunk_seq: int, - ) -> bool: - """Update the per-room join summary; False when the write was refused.""" - if status != "ok": - return False - with self._lock: - if room_failed or int(room) in self._cancelled: - return False - stats = self._join.get(int(room)) - if stats is None: - stats = {"chunk_count": 0} - self._join[int(room)] = stats - stats["chunk_count"] = int(stats.get("chunk_count", 0)) + 1 - first = self._first.get(int(room)) - if first is not None: - stats["first_enqueue_wall_ns"] = int(first[1]) - if is_last_chunk: - stats["last_chunk_enqueue_wall_ns"] = int(enqueue_wall_ns) - stats["last_chunk_complete_wall_ns"] = int(now_wall_ns) - stats["last_chunk_queue_wait_ms"] = queue_wait_total_ns / 1e6 - stats["last_chunk_worker_ms"] = worker_busy_ns / 1e6 - stats["last_chunk_chunk_seq"] = int(chunk_seq) - return True - - def reset(self, room: int) -> None: - with self._lock: - self._join.pop(int(room), None) - self._seq.pop(int(room), None) - self._first.pop(int(room), None) - self._mark_cancelled(room) - - def teardown(self, room: int, keep_join: bool) -> None: - with self._lock: - if not keep_join: - self._join.pop(int(room), None) - self._seq.pop(int(room), None) - self._first.pop(int(room), None) - self._cancelled.pop(int(room), None) - - def pop_join(self, room: int): - with self._lock: - return self._join.pop(int(room), None) - - -def note_chunk_requeue(chunk, now_mono_ns: int, now_wall_ns: int) -> None: - """Accumulate only the *queue* part of a finished attempt before re-enqueue. - - The previous implementation added ``requeue - last_enqueue``, which also - contains the worker's own execution and the staging watermark wait, thus - inflating queue wait on every defer (review finding). Only the interval - from the last enqueue to this attempt's dequeue is queue wait. - """ - if not _MOONCAKE_DETAIL_ENABLED or not getattr(chunk, "observe", False): - return - last_enqueue = int(getattr(chunk, "last_enqueue_mono_ns", 0) or 0) - dequeue = int(getattr(chunk, "dequeue_mono_ns", 0) or 0) - if last_enqueue and dequeue: - chunk.queue_wait_accum_ns = int( - getattr(chunk, "queue_wait_accum_ns", 0) or 0 - ) + max(0, dequeue - last_enqueue) - chunk.last_enqueue_mono_ns = int(now_mono_ns) - chunk.last_enqueue_wall_ns = int(now_wall_ns) - chunk.dequeue_mono_ns = 0 - chunk.requeue_count = int(getattr(chunk, "requeue_count", 0) or 0) + 1 - - -def emit_mooncake_chunk( - *, - chunk, - status: str, - queue_wait_ns: int, - queue_wait_total_ns: int, - source_wait_ns: int, - indices_wait_ns: int, - kv_transfer_ns: int, - state_transfer_ns: int, - aux_transfer_ns: int, - attempt_wall_ns: int, - dequeue_wall_ns: int, - end_wall_ns: int, - n_destinations: int, - source_wait_wall=None, - indices_wait_wall=None, - dp_rank: int = -1, - tp_rank: int = -1, -) -> None: - """One span per transfer-worker attempt, attributed by room/chunk_seq. - - ``worker_busy_ms`` follows the design definition: attempt wall time minus - the source/indices event waits (those are blocking waits, not worker work). - ``attempt_wall_ms`` is the raw dequeue->end time. - - Wait child spans use the real event timestamps captured by the worker. - A wait without a recorded window is attributes-only; no synthetic - ``end - duration`` placement is drawn (review finding). - """ - if not _MOONCAKE_DETAIL_ENABLED: - return - try: - from opentelemetry import trace as otel_trace - - tracer = otel_trace.get_tracer("sglang.mooncake.chunk") - # Attempt-scoped start: the last enqueue of this attempt, not the - # first enqueue of the chunk, so deferred retries do not overlap. - start_wall_ns = int( - getattr(chunk, "last_enqueue_wall_ns", 0) - or chunk.enqueue_wall_ns - or dequeue_wall_ns - ) or end_wall_ns - worker_busy_ns = max( - 0, int(attempt_wall_ns) - int(source_wait_ns) - int(indices_wait_ns) - ) - residual_ns = max( - 0, - worker_busy_ns - - int(kv_transfer_ns) - - int(state_transfer_ns) - - int(aux_transfer_ns), - ) - span = tracer.start_span( - "mooncake.kv_chunk", - context=_chunk_parent_context(chunk), - start_time=start_wall_ns, - attributes={ - "sglang.bootstrap_room": int(chunk.room), - "sglang.chunk_seq": int(chunk.chunk_seq), - "sglang.loop_seq": int(chunk.loop_seq), - "sglang.send_kind": str(chunk.send_kind or "unknown"), - "sglang.is_last_chunk": bool(chunk.is_last_chunk), - "sglang.queue_id": int(chunk.queue_id), - "sglang.requeue_count": int(chunk.requeue_count), - "sglang.num_kv_tokens": int(chunk.num_kv_tokens or 0), - "sglang.num_pages": int(chunk.num_pages), - "sglang.approx_bytes": int(chunk.approx_bytes), - "sglang.n_destinations": int(n_destinations), - "sglang.chunk_status": str(status), - "sglang.dp_rank": int(dp_rank), - "sglang.tp_rank": int(tp_rank), - # Attempt window (matches the child span) vs cumulative wait - # across deferred attempts (design §5.3 metric). - "sglang.queue_wait_ms": queue_wait_ns / 1e6, - "sglang.queue_wait_total_ms": queue_wait_total_ns / 1e6, - "sglang.source_wait_ms": source_wait_ns / 1e6, - "sglang.indices_wait_ms": indices_wait_ns / 1e6, - "sglang.kv_transfer_ms": kv_transfer_ns / 1e6, - "sglang.state_transfer_ms": state_transfer_ns / 1e6, - "sglang.aux_transfer_ms": aux_transfer_ns / 1e6, - "sglang.attempt_wall_ms": attempt_wall_ns / 1e6, - "sglang.worker_busy_ms": worker_busy_ns / 1e6, - "sglang.worker_residual_ms": residual_ns / 1e6, - }, - ) - span_ctx = otel_trace.set_span_in_context(span) - - def _child(name, start_ns, stop_ns, wait_ns): - start_ns = int(start_ns or 0) - stop_ns = int(stop_ns or 0) - if start_ns <= 0 or stop_ns <= start_ns: - return - try: - tracer.start_span( - name, - context=span_ctx, - start_time=start_ns, - attributes={"sglang.wait_ms": max(0, int(wait_ns)) / 1e6}, - ).end(end_time=stop_ns) - except Exception: - pass - - # Real event windows only; queue wait ends at dequeue. Zero-wait - # attempts (immediate dequeue) get no span. - if queue_wait_ns > 0: - _child( - "mooncake.queue_wait", - getattr(chunk, "last_enqueue_wall_ns", 0), - dequeue_wall_ns, - queue_wait_ns, - ) - if source_wait_wall is not None: - _child( - "mooncake.wait_source", - source_wait_wall[0], - source_wait_wall[1], - source_wait_ns, - ) - if indices_wait_wall is not None: - _child( - "mooncake.wait_indices", - indices_wait_wall[0], - indices_wait_wall[1], - indices_wait_ns, - ) - span.end(end_time=int(end_wall_ns)) - except Exception: - # Telemetry must never break the transfer worker. - pass - - -def emit_request_kv_join( - *, - req, - stats: dict, - dp_rank: int = -1, - completion_wall_ns: Optional[int] = None, -) -> None: - """Request-level join summary: last-chunk exposure + completion poll.""" - if not _MOONCAKE_DETAIL_ENABLED or not stats: - return - try: - from opentelemetry import trace as otel_trace - from opentelemetry.trace import Link - - now_wall_ns = int(completion_wall_ns or time.time_ns()) - enqueue_wall_ns = int(stats.get("last_chunk_enqueue_wall_ns", 0) or 0) - complete_wall_ns = int(stats.get("last_chunk_complete_wall_ns", 0) or 0) - final_exposed_ns = ( - max(0, complete_wall_ns - enqueue_wall_ns) - if enqueue_wall_ns and complete_wall_ns - else 0 - ) - completion_poll_ns = ( - max(0, now_wall_ns - complete_wall_ns) if complete_wall_ns else 0 - ) - # First enqueue comes from the manager's real queue.put() event - # (wall clock), not from pre-send preparation (review finding). - first_enqueue_wall_ns = int(stats.get("first_enqueue_wall_ns", 0) or 0) - first_enqueue_to_completion_ms = None - if first_enqueue_wall_ns: - first_enqueue_to_completion_ms = max( - 0.0, (now_wall_ns - first_enqueue_wall_ns) / 1e6 - ) - req_ctx = _request_span_context(req) - attrs = { - "sglang.bootstrap_room": int(getattr(req, "bootstrap_room", 0) or 0), - "sglang.rid": str(getattr(req, "rid", "")), - "sglang.dp_rank": int(dp_rank), - "sglang.chunk_count": int(stats.get("chunk_count", 0)), - "sglang.last_chunk_queue_wait_ms": float( - stats.get("last_chunk_queue_wait_ms", 0.0) - ), - "sglang.last_chunk_worker_ms": float( - stats.get("last_chunk_worker_ms", 0.0) - ), - "sglang.final_exposed_ms": final_exposed_ns / 1e6, - "sglang.completion_poll_ms": completion_poll_ns / 1e6, - # Per-rank request join. rank_join_skew_ms (max-min over TP/DP - # ranks) is computed offline by joining this span by room. - "sglang.request_join_ms": (final_exposed_ns + completion_poll_ns) / 1e6, - } - if first_enqueue_to_completion_ms is not None: - attrs["sglang.first_enqueue_to_completion_ms"] = ( - first_enqueue_to_completion_ms - ) - tracer = otel_trace.get_tracer("sglang.kv.request") - tracer.start_span( - "kv.request_join", - # Cover the whole request transfer when first enqueue is known. - start_time=first_enqueue_wall_ns or enqueue_wall_ns or now_wall_ns, - links=[Link(req_ctx)] if req_ctx is not None else [], - attributes=attrs, - ).end(end_time=now_wall_ns) - except Exception: - pass - - -def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None: - """Record the recv_requests phase inside the current loop (if any).""" - lt = getattr(_LOOP_TLS, "current", None) - if lt is not None: - lt.recv = (int(start_wall_ns), int(end_wall_ns), dict(parts)) - - -def note_pass_snapshot(t: "PassTimings", total_s: float) -> None: - """Record get_next timings for the current loop (called from end_pass).""" - lt = getattr(_LOOP_TLS, "current", None) - if lt is None: - return - end_wall_ns = time.time_ns() - start_wall_ns = end_wall_ns - int(total_s * 1e9) - measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync - other = max(0.0, total_s - measured) - lt.get_next = ( - start_wall_ns, - end_wall_ns, - { - "sglang.get_next_ms": total_s * 1e3, - "sglang.match_ms": t.match * 1e3, - "sglang.adder_ms": t.adder * 1e3, - "sglang.hicache_ms": t.hicache * 1e3, - "sglang.bootstrap_ms": t.bootstrap * 1e3, - "sglang.mlp_sync_ms": t.mlp_sync * 1e3, - "sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3, - "sglang.other_ms": other * 1e3, - "sglang.candidates": int(t.candidates), - "sglang.skipped_prefetch": int(t.skipped_prefetch), - "sglang.admitted": int(t.admitted), - "sglang.bootstrap_candidates": int(t.bootstrap_candidates), - "sglang.pending_bootstrap": int(t.pending_bootstrap), - "sglang.resumed_chunked": int(bool(t.had_chunked)), - "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), - "sglang.mlp_sync_seq": int(t.seq), - }, - ) - lt.seq = int(t.seq) - lt.pass_end_wall_ns = end_wall_ns - # Estimated DP-sync arrival: the pass ended, then mlp_sync waited. - # See design §1.2: arrival_est = pass_end - mlp_sync. - lt.arrival_est_wall_ns = end_wall_ns - int(t.mlp_sync * 1e9) - - -def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): - lt = getattr(_LOOP_TLS, "current", None) - if lt is not None: - attrs = dict(attrs or {}) - attrs.setdefault( - "sglang.run_batch_ms", max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6 - ) - lt.run_batch = (int(start_wall_ns), int(end_wall_ns), attrs) - - -def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): - lt = getattr(_LOOP_TLS, "current", None) - if lt is not None: - attrs = dict(attrs or {}) - attrs.setdefault( - "sglang.process_result_ms", - max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6, - ) - lt.process = (int(start_wall_ns), int(end_wall_ns), attrs) - - -def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None: - lt = getattr(_LOOP_TLS, "current", None) - if lt is None: - return - lt.kv_chunks += 1 - if lt.kv_send_start is None or start_wall_ns < lt.kv_send_start: - lt.kv_send_start = int(start_wall_ns) - if lt.kv_send_end is None or end_wall_ns > lt.kv_send_end: - lt.kv_send_end = int(end_wall_ns) - - -def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None: - """Set by dp_attn.prepare_mlp_sync_batch_raw once per pass.""" - if not _LOOP_ENABLED: - return - global _LAST_MLP_OUTCOME - _LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome)) - if local_kind != "none" or global_outcome != "none": - # DP idle work is only known after the sync. Activate the span before - # end_pass so the get_next child can share the loop trace. - ensure_loop_span(getattr(_LOOP_TLS, "current", None)) - - -def note_forward_device(t: float, category: str = "", **kwargs) -> None: - """DeviceTimer reporter: accumulate completed forward device time.""" - global _FORWARD_DEVICE_NS_TOTAL - try: - ns = int(float(t) * 1e9) - except Exception: - return - _FORWARD_DEVICE_NS_TOTAL += ns - if _FORWARD_SPAN_ENABLED: - _FORWARD_EVENTS.append((time.time_ns(), ns, str(category))) - - -def install_forward_reporter(scheduler) -> None: - """Attach the forward device-time reporter to the existing DeviceTimer. - - No CUDA event is added here: SGLANG_ENABLE_METRICS_DEVICE_TIMER already - runs the timer; this only reads its completed intervals. - """ - if not _LOOP_ENABLED: - return - try: - reporter = getattr(scheduler, "metrics_reporter", None) - timer = getattr(reporter, "forward_pass_device_timer", None) - if timer is not None: - timer.add_reporter(note_forward_device) - except Exception: - # Telemetry must never break scheduler init. - pass - - -def _ensure_loop_metrics(): - global _LOOP_METRICS - if _LOOP_METRICS is None: - from prometheus_client import Counter - - _LOOP_METRICS = { - "no_batch_seconds": Counter( - "sglang:scheduler_no_batch_seconds_total", - "Wall time of scheduler loops that ended without a runnable batch, " - "by reason and forward_overlap", - ["dp_rank", "reason", "forward_overlap"], - ), - "no_batch_total": Counter( - "sglang:scheduler_no_batch_total", - "Count of scheduler loops that ended without a runnable batch, " - "by reason and forward_overlap", - ["dp_rank", "reason", "forward_overlap"], - ), - "lockstep_seconds": Counter( - "sglang:scheduler_dp_lockstep_idle_seconds_total", - "Wall time of DP-lockstep idle-forward loops (this rank had no " - "local batch but the DP group required an idle forward)", - ["dp_rank", "forward_overlap"], - ), - } - return _LOOP_METRICS - - -def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str: - """Low-cardinality no-batch reason, in the design's priority order. - - ``pass_snapshot`` is the get_next attribute dict (candidates / skipped / - admitted). It is None when the loop skipped batch preparation entirely. - """ - if state.get("result_queue", 0) >= state.get("result_limit", 2): - return "result_backpressure" - if state.get("deferred", 0) > 0 or state.get("paused"): - return "deferred_input" - waiting = int(state.get("waiting", 0) or 0) - if waiting > 0 and pass_snapshot is not None: - bootstrap_candidates = int( - pass_snapshot.get("sglang.bootstrap_candidates", 0) or 0 - ) - pending_bootstrap = int( - pass_snapshot.get("sglang.pending_bootstrap", 0) or 0 - ) - if bootstrap_candidates > 0 and pending_bootstrap == bootstrap_candidates: - return "waiting_bootstrap" - candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0) - skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0) - admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0) - if candidates > 0 and skipped == candidates: - return "waiting_prefetch" - if admitted == 0: - return "admission_blocked" - inflight = int(state.get("inflight", 0) or 0) - bootstrap = int(state.get("bootstrap", 0) or 0) - if waiting == 0 and bootstrap: - return "waiting_bootstrap" - if waiting == 0 and inflight: - return "waiting_pd" - if ( - waiting == 0 - and inflight == 0 - and bootstrap == 0 - and int(state.get("result_queue", 0) or 0) == 0 - and int(state.get("deferred", 0) or 0) == 0 - and not state.get("waiting_for_result") - ): - return "fully_idle" - if state.get("skip_batch_creation") and pass_snapshot is None: - # A failed preparation defers retries; keep the last classified reason. - return _LAST_NO_BATCH_REASON - return "no_ready_other" - - -def _batch_attrs(batch) -> dict: - if batch is None: - return { - "sglang.n_reqs": 0, - "sglang.n_new": 0, - "sglang.n_cached": 0, - "sglang.chunked": False, - } - try: - n_reqs = int(batch.batch_size()) - except Exception: - n_reqs = len(getattr(batch, "reqs", ()) or ()) - try: - n_new = int(getattr(batch, "extend_num_tokens", 0) or 0) - except Exception: - n_new = 0 - try: - n_cached = int(sum(getattr(batch, "prefix_lens", ()) or ())) - except Exception: - n_cached = 0 - try: - chunked = getattr(batch, "chunked_req", None) is not None - except Exception: - chunked = False - return { - "sglang.n_reqs": n_reqs, - "sglang.n_new": n_new, - "sglang.n_cached": n_cached, - "sglang.chunked": bool(chunked), - } - - -_LAST_LOOP_SEQ = 0 -_LAST_LOOP_HAS_DETAIL = False - - -def _total_ms(ms: dict, base: str) -> float: - return sum( - value for name, value in ms.items() if name == base or name.startswith(base + ".") - ) - - -def _update_prev_state(lt: LoopTimings, loop_seq: int): - """Capture the previous loop summary and store this loop for the next one. - - Returns ``(prev_seq, prev_has_detail, prev_summary, consecutive)``. Only a - loop that actually recorded detail becomes a valid predecessor, so a - sampled loop after an unsampled one is marked non-consecutive instead of - silently publishing a stale window (design §4.5). - """ - global _LAST_LOOP_SEQ, _LAST_LOOP_HAS_DETAIL - prev_seq = int(_LAST_LOOP_SEQ) - prev_has = bool(_LAST_LOOP_HAS_DETAIL) - prev_summary = dict(_LAST_LOOP_SUMMARY) - consecutive = prev_has and prev_seq == loop_seq - 1 - - ms = { - name: rec[0] / 1e6 - for name, rec in ((lt.detail or {}).get("phases") or {}).items() - if rec - } - _LAST_LOOP_SUMMARY.clear() - if lt.detail is not None: - _LAST_LOOP_SUMMARY.update( - { - "run_batch_ms": float( - lt.run_batch[2].get("sglang.run_batch_ms", 0.0) - ) - if lt.run_batch is not None - else 0.0, - "process_result_ms": float( - lt.process[2].get("sglang.process_result_ms", 0.0) - ) - if lt.process is not None - else 0.0, - "copy_done_wait_ms": _total_ms(ms, "copy_done_wait"), - "kv_state_payload_ms": _total_ms(ms, "kv_state_payload"), - "kv_page_indices_ms": _total_ms(ms, "kv_page_indices"), - "kv_sender_send_ms": _total_ms(ms, "kv_sender_send"), - } - ) - _LAST_LOOP_SEQ = int(loop_seq) - _LAST_LOOP_HAS_DETAIL = lt.detail is not None - return prev_seq, prev_has, prev_summary, consecutive - - -def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None: - """Expose detail phases on the loop span and its child spans. - - Single-occurrence phases become children (``scheduler.loop.``); - repeated phases only contribute summed attributes, so span count never - grows with n_reqs / n_segments. - """ - detail = lt.detail or {} - phases = detail.get("phases") or {} - ms = {} - accounting_anomalies = 0 - for name, rec in phases.items(): - try: - total_ns, count, start_wall, end_wall = rec[:4] - except Exception: - continue - key = "sglang." + name.replace(".", "_") + "_ms" - ms[name] = total_ns / 1e6 - attrs[key] = ms[name] - cpu_ns, cpu_count = (rec[4], rec[5]) if len(rec) > 5 else (0, 0) - prefix = "sglang." + name.replace(".", "_") - non_cpu_ms = None - if cpu_count: - attrs[prefix + "_thread_cpu_ms"] = cpu_ns / 1e6 - attrs[prefix + "_thread_cpu_measured_count"] = int(cpu_count) - # Only a fully measured phase may report the wall - CPU remainder; - # a partial measurement would silently attribute unmeasured time to - # "non-CPU". - if cpu_count == count: - non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6) - attrs[prefix + "_non_cpu_ms"] = non_cpu_ms - # Only a deficit beyond clock granularity is an accounting - # anomaly; a few hundred ns of skew on a 2 ms phase is noise. - deficit_ns = cpu_ns - total_ns - if deficit_ns > max(100_000, total_ns // 100): - attrs[prefix + "_non_cpu_raw_ms"] = -deficit_ns / 1e6 - attrs[prefix + "_accounting_anomaly"] = True - accounting_anomalies += 1 - if count > 1: - attrs[prefix + "_count"] = int(count) - continue - if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES: - continue - child_attrs = { - "sglang.dp_rank": int(dp_rank), - "sglang.phase_ms": total_ns / 1e6, - } - if cpu_count: - child_attrs["sglang.phase_thread_cpu_ms"] = cpu_ns / 1e6 - child_attrs["sglang.phase_cpu_measured_count"] = int(cpu_count) - if non_cpu_ms is not None: - child_attrs["sglang.phase_non_cpu_ms"] = non_cpu_ms - child( - "scheduler.loop." + name.replace(".", "_"), - start_wall, - end_wall, - child_attrs, - ) - - if accounting_anomalies: - attrs["sglang.accounting_anomaly_count"] = int(accounting_anomalies) - for name, value in (detail.get("counts") or {}).items(): - attrs["sglang." + name.replace(".", "_") + "_count"] = int(value) - for name, value in (detail.get("sums") or {}).items(): - attrs["sglang." + name.replace(".", "_")] = int(value) - for name, value in (detail.get("attrs") or {}).items(): - attrs["sglang." + name.replace(".", "_")] = value - - # Accounting residuals (design §3.2): never negative, never folded in. - def _total(*bases): - total = 0.0 - for base in bases: - for name, value in ms.items(): - if name == base or name.startswith(base + "."): - total += value - return total - - # Roll per-kind variants (kv_page_indices.last, ...) up to one base - # attribute so analysis can use the total without knowing the kinds. - bases = {name.split(".", 1)[0] for name in ms} - for base in bases: - if base not in ms: - attrs["sglang." + base + "_ms"] = _total(base) - - if lt.run_batch is not None: - run_ms = float(lt.run_batch[2].get("sglang.run_batch_ms", 0.0)) - parts = _total( - "early_send_host", - "resolve_inputs", - "model_forward_host", - "copy_result_submit", - ) - attrs["sglang.run_residual_ms"] = max(0.0, run_ms - parts) - if lt.process is not None: - proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0)) - parts = _total( - # Outer phases of process_batch_result (scheduler.py). - "flush_trace", - "load_snapshot", - "flush_backups", - "step_counters", - "metrics_report", - # Inner disagg-prefill phases (prefill.py). - "copy_done_wait", - "auxiliary_and_logprob", - "per_request_result", - "report_stats", - ) - if "sglang.process_kv_ms" in lt.process[2]: - # KV phases charged to this call only (sends issued later in the - # loop, e.g. pending-bootstrap, are not part of process_result). - parts += float(lt.process[2]["sglang.process_kv_ms"]) - else: - parts += _total( - "kv_state_payload", "kv_page_indices", "kv_sender_send" - ) - attrs["sglang.process_residual_ms"] = max(0.0, proc_ms - parts) - - - -def end_loop( - lt: Optional[LoopTimings], - *, - dp_rank: int, - has_batch: bool, - batch=None, - state: Optional[dict] = None, -) -> None: - """Close one loop: counters (always), loop span + phases (if sampled).""" - if lt is None: - return - _LOOP_TLS.current = None - state = state or {} - try: - now_ns = time.monotonic_ns() - now_wall_ns = time.time_ns() - loop_s = max(0.0, (now_ns - lt.t0_ns) / 1e9) - gpu_ns = max(0, _FORWARD_DEVICE_NS_TOTAL - lt.forward_total0) - active1 = bool(state.get("forward_active", False)) - overlap = ( - "full" - if lt.active0 and active1 - else "partial" - if (lt.active0 or active1) - else "none" - ) - local_kind, global_outcome = _LAST_MLP_OUTCOME - pass_snapshot = lt.get_next[2] if lt.get_next is not None else None - pending = bool( - state.get("waiting", 0) - or state.get("result_queue", 0) - or state.get("inflight", 0) - or state.get("bootstrap", 0) - or state.get("deferred", 0) - or state.get("paused") - or state.get("waiting_for_result") - ) - reason = "" - - try: - if local_kind == "none" and global_outcome == "idle": - # The DP sync replaces local None with a real idle batch, so this - # accounting must not depend on has_batch being False. - _ensure_loop_metrics()["lockstep_seconds"].labels( - dp_rank=str(dp_rank), forward_overlap=overlap - ).inc(loop_s) - elif not has_batch: - reason = lt.launch_blocked_reason or _classify_no_batch( - state, pass_snapshot - ) - global _LAST_NO_BATCH_REASON - _LAST_NO_BATCH_REASON = reason - metrics = _ensure_loop_metrics() - metrics["no_batch_seconds"].labels( - dp_rank=str(dp_rank), - reason=reason, - forward_overlap=overlap, - ).inc(loop_s) - metrics["no_batch_total"].labels( - dp_rank=str(dp_rank), - reason=reason, - forward_overlap=overlap, - ).inc() - except Exception: - # Telemetry must never break the scheduler loop. - pass - lt.no_batch_reason = reason - - # Update the prev-loop chain on every loop (sampled or not) so a - # sampled loop after an unsampled one is reported as non-consecutive - # instead of silently carrying a stale window (review finding). - loop_seq_cur = int(lt.seq or lt.predicted_seq) - prev_state = None - if _DETAIL_ENABLED: - prev_state = _update_prev_state(lt, loop_seq_cur) - - if lt.span is None and (has_batch or pending): - # Covers work discovered after begin_loop, including first input and - # deferred-only loops. DP outcomes activate earlier for pass parenting. - ensure_loop_span(lt) - if lt.span is None: - return - - try: - from opentelemetry import trace as otel_trace - - tracer = otel_trace.get_tracer("sglang.prefill.loop") - attrs = { - "sglang.phase": lt.phase, - "sglang.dp_rank": int(dp_rank), - "sglang.loop_seq": int(lt.seq or lt.predicted_seq), - "sglang.loop_ms": loop_s * 1e3, - "sglang.gpu_ms": gpu_ns / 1e6, - "sglang.gpu_idle_ms": max(0.0, loop_s * 1e3 - gpu_ns / 1e6), - "sglang.has_batch": bool(has_batch), - "sglang.engine_has_pending_work": pending, - "sglang.local_batch_kind": local_kind, - "sglang.global_outcome": global_outcome, - "sglang.no_batch_reason": reason, - "sglang.forward_overlap": overlap, - } - attrs.update(_batch_attrs(batch)) - - def _child(name, start_ns, end_ns, child_attrs): - try: - tracer.start_span( - name, - context=lt.span_ctx, - start_time=int(start_ns), - attributes=child_attrs, - ).end(end_time=int(end_ns)) - except Exception: - pass - - if lt.detail is not None: - _emit_loop_detail(lt, attrs, _child, int(dp_rank)) - if _DETAIL_ENABLED: - if lt.pass_end_wall_ns: - # DP-skew join fields (design §4.5); independent of phases. - attrs["sglang.pass_end_wall_ns"] = int(lt.pass_end_wall_ns) - attrs["sglang.arrival_est_wall_ns"] = int(lt.arrival_est_wall_ns) - if prev_state is not None: - prev_seq, prev_has, prev_summary, consecutive = prev_state - attrs["sglang.prev_loop_seq"] = int(prev_seq) - if consecutive: - for key in _DETAIL_PREV_KEYS: - if key in prev_summary: - attrs["sglang.prev_" + key] = prev_summary[key] - elif prev_seq: - # The previous loop was unsampled or had no detail; - # its phases must not be treated as seq-1 values. - attrs["sglang.prev_detail_missing"] = True - lt.span.set_attributes(attrs) - - if lt.recv is not None and not lt.recv_span_child: - start, end, parts = lt.recv - child = {"sglang.dp_rank": int(dp_rank)} - child.update(parts) - _child("scheduler.loop.recv", start, end, child) - if lt.get_next is not None and not lt.pass_span_child: - start, end, ng_attrs = lt.get_next - _child("scheduler.loop.get_next", start, end, ng_attrs) - if lt.run_batch is not None: - start, end, rb_attrs = lt.run_batch - _child("scheduler.loop.run_batch", start, end, rb_attrs) - if lt.process is not None: - start, end, pr_attrs = lt.process - _child("scheduler.loop.process_result", start, end, pr_attrs) - if _KV_SEND_SPAN_ENABLED and lt.kv_send_start is not None: - _child( - "scheduler.loop.kv_send", - lt.kv_send_start, - lt.kv_send_end, - {"sglang.n_chunks": int(lt.kv_chunks)}, - ) - if _FORWARD_SPAN_ENABLED: - for completion_wall, elapsed_ns, category in list(_FORWARD_EVENTS): - if completion_wall < lt.t0_wall_ns or completion_wall > now_wall_ns: - continue - _child( - "scheduler.loop.forward.gpu", - completion_wall - elapsed_ns, - completion_wall, - { - "sglang.gpu_ms": elapsed_ns / 1e6, - "sglang.category": category, - }, - ) - lt.span.end(end_time=now_wall_ns) - except Exception: - # Telemetry must never break the scheduler loop. - pass - finally: - _LOOP_TLS.current = None +"""Per-batch scheduler telemetry (opt-in). + +Adds, at the *batch* granularity (one record per scheduler batch per DP rank): + + * Prometheus histograms (safe for the multiprocess collector): + sglang:batch_new_tokens{phase,dp_rank} + sglang:batch_cached_tokens{phase,dp_rank} + sglang:batch_duration_seconds{phase,dp_rank} + sglang:batch_queue_depth{phase,dp_rank} + sglang:batch_pending_tokens{phase,dp_rank} + sglang:batch_n_reqs{phase,dp_rank} + * An OTLP span ``scheduler.batch.{phase}`` with ``Link``s to the request + traces that took part in the batch (batch -> requests navigation), and a + ``sglang.batch_id`` attribute for reverse lookups / log correlation. + +Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs: + + SGLANG_BATCH_SPAN=0 histograms only (no spans) + SGLANG_BATCH_SPAN_MIN_MS=50 only emit spans for batches slower than this + SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0) + SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span + + SGLANG_TRACE_LOOP_DETAIL=1 split run_batch/process_result into host + phases (requires SGLANG_BATCH_STAGE_TIMING=1 + and SGLANG_TRACE_LOOP_SPAN=1; only sampled + loops are recorded) + SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown + (queue/source/indices/transfer/join) + SGLANG_TRACE_BACKUP_DETAIL=1 per-flush HiCache backup sub-phases + (one ``scheduler.cache_flush_backup`` span; + requires SGLANG_TRACE_LOOP_DETAIL=1) + +Cost model (measured on our 4xDP4 / 4-GPU instances): + * histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU. + * span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s. + Everything is behind a single module-level bool, so when disabled the hot + path pays one ``if not _ENABLED: return``. + +NOTE: sglang runs prometheus_client in multiprocess mode, which does not +carry exemplars, so the span trace id is returned to the caller and appended +to the batch log line instead (see ``metrics_reporter``). +""" + +from __future__ import annotations + +import os +import random +import threading +import time +from typing import Any, Iterable, Optional + +_ENABLED = os.getenv("SGLANG_BATCH_TELEMETRY", "0").strip().lower() not in ( + "", + "0", + "false", + "no", + "off", +) +_SPAN_ENABLED = _ENABLED and os.getenv("SGLANG_BATCH_SPAN", "1").strip().lower() not in ( + "0", + "false", + "no", + "off", +) +_SPAN_MIN_S = float(os.getenv("SGLANG_BATCH_SPAN_MIN_MS", "0")) / 1000.0 +_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_SPAN_SAMPLE", "1.0")) +_MAX_LINKS = int(os.getenv("SGLANG_BATCH_SPAN_MAX_LINKS", "32")) + +_HISTOGRAMS = None +_CHILDREN: dict[tuple[str, str, int], Any] = {} +_TRACER = None + + +def enabled() -> bool: + return _ENABLED + + +_REQUEST_COUNTER = None +_DP_RANK = None +_COMPLETION_RANK = None + + +def _should_report_completion() -> bool: + """Whether this scheduler is the canonical reporter of request completions. + + Every TP/CP rank runs a scheduler and observes the same requests, and a PP + stage only sees its own microbatches, so an ungated counter would be + multiplied across replicas once the multiprocess registry aggregates them. + Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same + shape the existing scheduler metrics use for rank-unique series. + """ + global _COMPLETION_RANK + if _COMPLETION_RANK is None: + try: + from sglang.srt.runtime_context import get_parallel + + parallel = get_parallel() + _COMPLETION_RANK = bool( + parallel.attn_tp_rank == 0 + and parallel.attn_cp_rank == 0 + and parallel.pp_group.is_last_rank + ) + except Exception: + # Unknown topology (e.g. a bare unit-test process): keep counting. + _COMPLETION_RANK = True + return _COMPLETION_RANK + + +def _current_dp_rank() -> int: + """This process's attention-DP rank (0 for non-DP engines).""" + global _DP_RANK + if _DP_RANK is None: + try: + from sglang.srt.runtime_context import get_parallel + + _DP_RANK = int(get_parallel().attn_dp_rank or 0) + except Exception: + _DP_RANK = 0 + return _DP_RANK + + +def _ensure_request_counter(): + global _REQUEST_COUNTER + if _REQUEST_COUNTER is None: + from prometheus_client import Counter + + _REQUEST_COUNTER = Counter( + "sglang:batch_requests_total", + "Requests completed by the canonical scheduler rank, by phase. " + "phase=prefill: prefill compute finished and the request entered the " + "PD transfer queue (not a guarantee that KV transfer succeeded). " + "phase=decode: request finished decoding.", + ["phase", "dp_rank"], + ) + return _REQUEST_COUNTER + + +def note_request_completed(phase: str, n: int = 1) -> None: + """Count finished requests per DP rank. + + This is the per-rank RPS source: the frontend ``num_requests_total`` has no + rank label, while the scheduler knows exactly which DP shard completed the + request. Cheap (one counter increment per finished request) and gated by + SGLANG_BATCH_TELEMETRY like the other batch metrics. + """ + if not _ENABLED or n <= 0 or not _should_report_completion(): + return + try: + _ensure_request_counter().labels( + phase=phase, dp_rank=str(_current_dp_rank()) + ).inc(int(n)) + except Exception: + # Telemetry must never fail a request. + pass + + +def _ensure_histograms(): + global _HISTOGRAMS + if _HISTOGRAMS is None: + from prometheus_client import Histogram + + labels = ["phase", "dp_rank"] + _HISTOGRAMS = { + "batch_new_tokens": Histogram( + "sglang:batch_new_tokens", + "New (uncached) prompt tokens processed by one scheduler batch", + labels, + buckets=(128, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536), + ), + "batch_cached_tokens": Histogram( + "sglang:batch_cached_tokens", + "Cached prompt tokens carried by one scheduler batch", + labels, + buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576), + ), + "batch_duration_seconds": Histogram( + "sglang:batch_duration_seconds", + "Wall time spent in one scheduler batch", + labels, + buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0), + ), + "batch_queue_depth": Histogram( + "sglang:batch_queue_depth", + "Waiting-queue depth observed by one scheduler batch", + labels, + buckets=(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512), + ), + "batch_pending_tokens": Histogram( + "sglang:batch_pending_tokens", + "Prefill backlog (pending tokens) observed by one scheduler batch", + labels, + buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576), + ), + "batch_n_reqs": Histogram( + "sglang:batch_n_reqs", + "Requests carried by one scheduler batch", + labels, + buckets=(1, 2, 4, 8, 16, 32, 64, 128, 256), + ), + } + return _HISTOGRAMS + + +def _child(name: str, phase: str, dp_rank: int): + key = (name, phase, dp_rank) + child = _CHILDREN.get(key) + if child is None: + child = _ensure_histograms()[name].labels(phase=phase, dp_rank=str(dp_rank)) + _CHILDREN[key] = child + return child + + +def _request_span_context(req: Any): + """SpanContext of a request, from either a live span or a serialized copy. + + Scheduler processes receive a *copy* of the request trace context where + ``root_span`` is None (see ``trace_async.copy_for_thread``) and only + ``root_span_context`` (an OTel ``Context``) survives. Handle both. + """ + trace_ctx = getattr(getattr(req, "time_stats", None), "trace_ctx", None) + if trace_ctx is None: + return None + # TraceNullContext forwards every attribute to itself (``__getattr__``), so + # the span-context probes below would "succeed" and hand a non-SpanContext + # object to otel's Link(), which fails at export time with + # "expected bytes, TraceNullContext found". + if not getattr(trace_ctx, "tracing_enable", False): + return None + try: + root_span = getattr(trace_ctx, "root_span", None) + if root_span is not None: + span_ctx = root_span.get_span_context() + if span_ctx.is_valid: + return span_ctx + root_ctx = getattr(trace_ctx, "root_span_context", None) + if root_ctx is not None: + from opentelemetry import trace as otel_trace + + span_ctx = otel_trace.get_current_span(root_ctx).get_span_context() + if span_ctx.is_valid: + return span_ctx + except Exception: + return None + return None + + +def _emit_span( + phase: str, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]], +) -> Optional[str]: + """Create one batch span with links to the request traces. Returns trace id.""" + if not _SPAN_ENABLED: + return None + if _SPAN_MIN_S > 0 and duration_s < _SPAN_MIN_S: + return None + if _SPAN_SAMPLE < 1.0 and random.random() > _SPAN_SAMPLE: + return None + global _TRACER + try: + from opentelemetry import trace as otel_trace + from opentelemetry.trace import Link + + if _TRACER is None: + _TRACER = otel_trace.get_tracer("sglang.batch") + links = [] + if reqs: + for req in reqs: + if len(links) >= _MAX_LINKS: + break + ctx = _request_span_context(req) + if ctx is not None: + links.append(Link(ctx)) + end_ns = time.time_ns() + start_ns = end_ns - int(duration_s * 1e9) + span = _TRACER.start_span( + f"scheduler.batch.{phase}", + start_time=start_ns, + attributes={ + "sglang.batch_id": f"{dp_rank}:{iter_no}", + "sglang.dp_rank": int(dp_rank), + "sglang.phase": phase, + "sglang.n_reqs": int(n_reqs), + "sglang.n_new_tokens": int(n_new), + "sglang.n_cached_tokens": int(n_cached), + "sglang.queue_depth": int(queue_depth), + "sglang.pending_tokens": int(pending_tokens), + "sglang.batch_duration_s": float(duration_s), + }, + links=links, + ) + span.end(end_time=end_ns) + span_ctx = span.get_span_context() + if not span_ctx.is_valid: + # No TracerProvider configured (tracing disabled) -> NoOp span. + return None + return format(span_ctx.trace_id, "032x") + except Exception: + return None + + +def _observe( + phase: str, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]], +) -> Optional[str]: + if not _ENABLED: + return None + try: + trace_id = _emit_span( + phase, + dp_rank, + iter_no, + n_reqs, + n_new, + n_cached, + queue_depth, + pending_tokens, + duration_s, + reqs, + ) + _child("batch_new_tokens", phase, dp_rank).observe(n_new) + _child("batch_cached_tokens", phase, dp_rank).observe(n_cached) + _child("batch_duration_seconds", phase, dp_rank).observe(duration_s) + _child("batch_queue_depth", phase, dp_rank).observe(queue_depth) + _child("batch_pending_tokens", phase, dp_rank).observe(pending_tokens) + _child("batch_n_reqs", phase, dp_rank).observe(n_reqs) + return trace_id + except Exception: + # Telemetry must never break the scheduler loop. + return None + + +def observe_prefill( + *, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]] = None, +) -> Optional[str]: + return _observe( + "prefill", + dp_rank, + iter_no, + n_reqs, + n_new, + n_cached, + queue_depth, + pending_tokens, + duration_s, + reqs, + ) + + +def observe_decode( + *, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]] = None, +) -> Optional[str]: + return _observe( + "decode", + dp_rank, + iter_no, + n_reqs, + n_new, + n_cached, + queue_depth, + pending_tokens, + duration_s, + reqs, + ) + +_DP_HISTS = None + + +def _ensure_dp_histograms(): + global _DP_HISTS + if _DP_HISTS is None: + from prometheus_client import Histogram + + _DP_HISTS = { + "dp_step_tokens": Histogram( + "sglang:dp_step_tokens", + "Per-rank token count in one DP-attention step", + ["dp_rank", "phase"], + buckets=(0, 256, 1024, 2048, 4096, 8192, 16384, 32768), + ), + "dp_step_padding_ratio": Histogram( + "sglang:dp_step_padding_ratio", + "1 - mean/max tokens across active DP ranks in one step (padding waste)", + ["phase", "n_active"], + buckets=(0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8), + ), + "dp_step_spread": Histogram( + "sglang:dp_step_spread", + "(max-min)/mean tokens across active DP ranks in one step", + ["phase", "n_active"], + buckets=(0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0), + ), + } + return _DP_HISTS + + +_DP_IDLE_MODE = 4 # ForwardMode.IDLE + + +def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = "unknown") -> None: + """Record per-step shape alignment across *active* DP ranks. + + ``tokens``/``modes``/``health`` are the per-rank vectors from the DP + all-gather (info tensor columns 0, 5, 9). IDLE ranks and health-check + steps are excluded so the padding ratio reflects real batch-shape + misalignment rather than idle-slot padding. + """ + if not _ENABLED: + return + try: + if health and any(bool(h) for h in health): + return + sel = [] + for i, t in enumerate(tokens): + if t is None: + continue + t = int(t) + if t <= 1: + continue + if modes is not None and i < len(modes) and int(modes[i]) == _DP_IDLE_MODE: + continue + sel.append(t) + if len(sel) < 2: + return + hist = _ensure_dp_histograms() + mx = max(sel) + mean = sum(sel) / len(sel) + n_active = str(len(sel)) + key = ("dp_step_tokens", phase, dp_rank) + child = _CHILDREN.get(key) + if child is None: + child = hist["dp_step_tokens"].labels(dp_rank=str(dp_rank), phase=phase) + _CHILDREN[key] = child + child.observe(int(tokens[dp_rank]) if 0 <= dp_rank < len(tokens) else int(mean)) + if mx > 0: + hist["dp_step_padding_ratio"].labels(phase=phase, n_active=n_active).observe( + 1.0 - mean / mx + ) + if mean > 0: + hist["dp_step_spread"].labels(phase=phase, n_active=n_active).observe( + (mx - min(sel)) / mean + ) + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Per-pass stage timing (SGLANG_BATCH_STAGE_TIMING=1) -- P (disagg prefill). +# +# One sample per get_next_disagg_prefill_batch_to_run() pass, including passes +# that produce no batch. Disabled by default; when off every hook below is a +# single module-level bool check. No CUDA calls, no per-pass logging. +# +# Cross-rank analysis uses a dedicated, independently sampled span +# ``scheduler.prefill.pass`` emitted from end_pass(): the sampling decision is a +# pure function of ``mlp_sync_seq`` (identical on every DP rank that took part +# in the same gather), so a sampled gather round is sampled on *all* ranks -- +# unlike the per-rank random sampling of the batch span. Idle ranks and ranks +# with no local batch therefore still emit, which is exactly the case the +# skew analysis needs. +# --------------------------------------------------------------------------- + +_STAGE_TIMING = os.getenv("SGLANG_BATCH_STAGE_TIMING", "0").strip().lower() not in ( + "", + "0", + "false", + "no", + "off", +) + +_STAGE_LABELS = ["phase", "dp_rank"] +_STAGE_BUCKETS = ( + 1e-5, + 3e-5, + 1e-4, + 3e-4, + 1e-3, + 3e-3, + 1e-2, + 3e-2, + 1e-1, + 3e-1, + 1.0, + 3.0, +) + +_STAGE_HISTS = None +_STAGE_COUNTERS = None +_STAGE_CHILDREN: dict = {} +_STAGE_TLS = threading.local() + +# Set by the DP all-gather path (dp_attn.prepare_mlp_sync_batch_raw). +_LAST_DP_GATHER = None +_DP_GATHER_SEQ = 0 + +_PASS_SPAN_ENABLED = _STAGE_TIMING and os.getenv( + "SGLANG_BATCH_PASS_SPAN", "1" +).strip().lower() not in ("", "0", "false", "no", "off") +_PASS_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_PASS_SPAN_SAMPLE", "0.05")) + +_STAGE_HIST_DOCS = { + "get_next": "Wall time of one get_next_disagg_prefill_batch_to_run pass", + "match": "Per-pass sum of init_next_round_input (prefix match) over candidates", + "adder": "Per-pass sum of PrefillAdder.add_one_req (budget/admission)", + "hicache_events": "Per-pass time in _process_hicache_events (includes its collectives)", + "bootstrap": "Per-pass time in resolve_waiting_queue_bootstrap (poll + all-reduce)", + "mlp_sync": "Per-pass outer maybe_prepare_mlp_sync_batch time (not pure gather wait)", + "other": "get_next total minus the instrumented segments", +} + + +def stage_timing_enabled() -> bool: + return _STAGE_TIMING + + +def pass_span_enabled() -> bool: + return _PASS_SPAN_ENABLED + + +def pass_span_decision(seq: int) -> bool: + """Deterministic, rank-identical sampling decision for one gather round. + + ``seq`` is the DP all-gather sequence number, which is the same on every + rank that participated in that round. Therefore one decision covers the + whole round, instead of four independent ``random.random()`` draws. + """ + if not _PASS_SPAN_ENABLED or seq <= 0: + return False + if _PASS_SPAN_SAMPLE >= 1.0: + return True + if _PASS_SPAN_SAMPLE <= 0.0: + return False + return (seq * 2654435761) % 10000 < int(_PASS_SPAN_SAMPLE * 10000) + + +def _ensure_stage_metrics(): + global _STAGE_HISTS, _STAGE_COUNTERS + if _STAGE_HISTS is None: + from prometheus_client import Counter, Histogram + + _STAGE_HISTS = { + name: Histogram( + f"sglang:batch_prefill_{name}_seconds", + doc, + _STAGE_LABELS, + buckets=_STAGE_BUCKETS, + ) + for name, doc in _STAGE_HIST_DOCS.items() + } + _STAGE_COUNTERS = { + "candidates": Counter( + "sglang:batch_prefill_candidates_total", + "Requests examined by the per-pass admission candidate loop", + _STAGE_LABELS, + ), + "skipped_prefetch": Counter( + "sglang:batch_prefill_skipped_prefetch_total", + "Candidates skipped because their L2/L3 prefetch was not ready", + _STAGE_LABELS, + ), + "admitted": Counter( + "sglang:batch_prefill_admitted_total", + "Requests admitted into can_run_list by one pass", + _STAGE_LABELS, + ), + "resumed_chunk": Counter( + "sglang:batch_prefill_resumed_chunk_total", + "Passes whose batch continues a chunked-prefill request " + "(including its last chunk)", + _STAGE_LABELS, + ), + "new_chunk": Counter( + "sglang:batch_prefill_new_chunk_total", + "Passes that start a new chunked-prefill request", + _STAGE_LABELS, + ), + } + return _STAGE_HISTS, _STAGE_COUNTERS + + +class PassTimings: + __slots__ = ( + "phase", + "t0", + "match", + "adder", + "hicache", + "bootstrap", + "mlp_sync", + "pending_abort", + "chunk", + "new_batch", + "set_sched", + "chunk_count", + "dp_gather", + "candidates", + "skipped_prefetch", + "admitted", + "bootstrap_candidates", + "pending_bootstrap", + "had_chunked", + "new_chunk", + "seq", + ) + + def __init__(self, phase: str): + self.phase = phase + self.t0 = time.monotonic() + self.match = 0.0 + self.adder = 0.0 + self.hicache = 0.0 + self.bootstrap = 0.0 + self.mlp_sync = 0.0 + self.pending_abort = 0.0 + self.chunk = 0.0 + self.new_batch = 0.0 + self.set_sched = 0.0 + self.chunk_count = 0 + self.dp_gather = None + self.candidates = 0 + self.skipped_prefetch = 0 + self.admitted = 0 + self.bootstrap_candidates = 0 + self.pending_bootstrap = 0 + self.had_chunked = False + self.new_chunk = False + self.seq = 0 + + +def begin_pass(phase: str = "prefill") -> Optional[PassTimings]: + """Start a new pass. Returns None when stage timing is disabled.""" + if not _STAGE_TIMING: + return None + timings = PassTimings(phase) + _STAGE_TLS.current = timings + return timings + + +def _current() -> Optional[PassTimings]: + if not _STAGE_TIMING: + return None + return getattr(_STAGE_TLS, "current", None) + + +def add_match(seconds: float) -> None: + t = _current() + if t is not None: + t.match += seconds + + +def add_adder(seconds: float) -> None: + t = _current() + if t is not None: + t.adder += seconds + + +def add_hicache(seconds: float) -> None: + t = _current() + if t is not None: + t.hicache += seconds + + +def add_bootstrap(seconds: float) -> None: + t = _current() + if t is not None: + t.bootstrap += seconds + + +def add_pending_abort(seconds: float) -> None: + t = _current() + if t is not None: + t.pending_abort += seconds + + +def add_chunk(seconds: float, count: int = 0) -> None: + t = _current() + if t is not None: + t.chunk += seconds + t.chunk_count += int(count) + + +def add_new_batch(seconds: float) -> None: + t = _current() + if t is not None: + t.new_batch += seconds + + +def add_set_sched(seconds: float) -> None: + t = _current() + if t is not None: + t.set_sched += seconds + + +def mark_candidate(n: int = 1) -> None: + t = _current() + if t is not None: + t.candidates += n + + +def mark_skipped_prefetch(n: int = 1) -> None: + t = _current() + if t is not None: + t.skipped_prefetch += n + + +def set_admitted(n: int) -> None: + t = _current() + if t is not None: + t.admitted = int(n) + + +def set_bootstrap_pending(pending: int, candidates: int) -> None: + """Reuse the bootstrap resolver's existing pass; do not rescan the queue.""" + t = _current() + if t is not None: + t.pending_bootstrap = int(pending) + t.bootstrap_candidates = int(candidates) + + +def note_had_chunked(flag: bool) -> None: + """Record whether a chunked request was already in flight when the pass began. + + Must be called at pass start, before process_prefill_chunk()/admission can + consume the chunked request (the last chunk clears self.chunked_req). + """ + t = _current() + if t is not None: + t.had_chunked = bool(flag) + + +def note_new_chunk(flag: bool) -> None: + """Record whether this pass started a new chunked request (first chunk).""" + t = _current() + if t is not None: + t.new_chunk = bool(flag) + + +def note_dp_gather(seconds: Optional[float]) -> None: + """Called by the DP all-gather path; None means no gather ran this pass.""" + global _LAST_DP_GATHER, _DP_GATHER_SEQ + if not _STAGE_TIMING: + return + if seconds is None: + _LAST_DP_GATHER = None + return + _LAST_DP_GATHER = float(seconds) + _DP_GATHER_SEQ += 1 + + +def _take_dp_gather(): + global _LAST_DP_GATHER + value = _LAST_DP_GATHER + _LAST_DP_GATHER = None + return value + + +def add_mlp_sync(seconds: float) -> None: + t = _current() + if t is None: + return + t.mlp_sync += seconds + gather = _take_dp_gather() + if gather is not None: + t.dp_gather = gather + t.seq = _DP_GATHER_SEQ + + +def _emit_pass_span( + dp_rank: int, + t: PassTimings, + total: float, + other: float, + residual: float = 0.0, +) -> None: + if not pass_span_decision(t.seq): + return + loop = getattr(_LOOP_TLS, "current", None) + loop_ctx = loop.span_ctx if loop is not None else None + if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None: + # An unsampled loop owns this pass; do not leak an independent pass trace. + # If work is discovered later, end_loop emits the snapshot as its child. + return + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.prefill.pass") + end_ns = time.time_ns() + start_ns = end_ns - int(total * 1e9) + tracer.start_span( + "scheduler.prefill.pass", + context=loop_ctx, + start_time=start_ns, + attributes={ + "sglang.phase": t.phase, + "sglang.dp_rank": int(dp_rank), + "sglang.mlp_sync_seq": int(t.seq), + "sglang.get_next_ms": total * 1e3, + "sglang.match_ms": t.match * 1e3, + "sglang.adder_ms": t.adder * 1e3, + "sglang.hicache_ms": t.hicache * 1e3, + "sglang.bootstrap_ms": t.bootstrap * 1e3, + "sglang.mlp_sync_ms": t.mlp_sync * 1e3, + "sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3, + "sglang.other_ms": other * 1e3, + "sglang.pending_abort_ms": t.pending_abort * 1e3, + "sglang.chunk_ms": t.chunk * 1e3, + "sglang.new_batch_ms": t.new_batch * 1e3, + "sglang.set_sched_ms": t.set_sched * 1e3, + "sglang.get_next_residual_ms": residual * 1e3, + "sglang.chunk_count": int(t.chunk_count), + "sglang.candidates": int(t.candidates), + "sglang.skipped_prefetch": int(t.skipped_prefetch), + "sglang.admitted": int(t.admitted), + "sglang.resumed_chunked": int(bool(t.had_chunked)), + "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), + }, + ).end(end_time=end_ns) + if loop is not None and loop_ctx is not None: + loop.pass_span_child = True + except Exception: + # Tracing must never break the scheduler loop. + pass + + +def end_pass(dp_rank: int) -> None: + """Close the current pass: observe histograms/counters and emit the pass span.""" + t = _current() + if t is None: + return + _STAGE_TLS.current = None + try: + total = time.monotonic() - t.t0 + measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync + other = total - measured + if other < 0.0: + other = 0.0 + # Residual after also charging the get_next sub-phases. + measured2 = measured + ( + t.pending_abort + t.chunk + t.new_batch + t.set_sched + ) + residual = total - measured2 + if residual < 0.0: + residual = 0.0 + hist, counters = _ensure_stage_metrics() + child_key = (t.phase, dp_rank) + keyed = _STAGE_CHILDREN.get(child_key) + if keyed is None: + keyed = { + name: h.labels(phase=t.phase, dp_rank=str(dp_rank)) + for name, h in hist.items() + } + keyed.update( + { + name: c.labels(phase=t.phase, dp_rank=str(dp_rank)) + for name, c in counters.items() + } + ) + _STAGE_CHILDREN[child_key] = keyed + keyed["get_next"].observe(total) + keyed["match"].observe(t.match) + keyed["adder"].observe(t.adder) + keyed["hicache_events"].observe(t.hicache) + keyed["bootstrap"].observe(t.bootstrap) + keyed["mlp_sync"].observe(t.mlp_sync) + keyed["other"].observe(other) + if t.candidates: + keyed["candidates"].inc(t.candidates) + if t.skipped_prefetch: + keyed["skipped_prefetch"].inc(t.skipped_prefetch) + if t.admitted: + keyed["admitted"].inc(t.admitted) + if t.had_chunked: + keyed["resumed_chunk"].inc(1) + elif t.new_chunk: + keyed["new_chunk"].inc(1) + note_pass_snapshot(t, total) + _emit_pass_span(dp_rank, t, total, other, residual) + except Exception: + # Telemetry must never break the scheduler loop + _STAGE_TLS.current = None + +# --------------------------------------------------------------------------- +# Scheduler receive path (same SGLANG_BATCH_STAGE_TIMING gate): split the +# recv_requests stage into pull / broadcast / unwrap / mm. +# +# NOTE: tokenization itself runs in the tokenizer worker processes (HTTP side). +# What is measured here is the scheduler-side IPC: ZMQ pull + decode of the +# tokenized request objects, the cross-rank broadcast of those objects, and +# the unpickling of their token-id buffers. Large prompts make all three +# visible (e.g. 14.6k tokens per request). +# --------------------------------------------------------------------------- + +_RECV_SPAN_ENABLED = _STAGE_TIMING and os.getenv( + "SGLANG_RECV_SPAN", "1" +).strip().lower() not in ("", "0", "false", "no", "off") +_RECV_SPAN_SAMPLE = float(os.getenv("SGLANG_RECV_SPAN_SAMPLE", "0.05")) + +_RECV_PARTS = ("total", "pull", "broadcast", "unwrap", "mm") +_RECV_HISTS = None +_RECV_COUNTERS = None +_RECV_SEQ = 0 + + +def recv_timing_enabled() -> bool: + return _STAGE_TIMING + + +def _ensure_recv_metrics(): + global _RECV_HISTS, _RECV_COUNTERS + if _RECV_HISTS is None: + from prometheus_client import Counter, Histogram + + _RECV_HISTS = Histogram( + "sglang:recv_requests_seconds", + "Scheduler recv_requests wall time by part " + "(total/pull/broadcast/unwrap/mm)", + ["dp_rank", "part"], + buckets=_STAGE_BUCKETS, + ) + _RECV_COUNTERS = { + "msgs": Counter( + "sglang:recv_requests_msgs_total", + "Messages received by the scheduler recv path", + ["dp_rank"], + ), + "prompt_tokens": Counter( + "sglang:recv_requests_prompt_tokens_total", + "Prompt tokens carried by messages received in the recv path", + ["dp_rank"], + ), + } + return _RECV_HISTS, _RECV_COUNTERS + + +def observe_recv( + *, + dp_rank: int, + total: float, + pull: float, + broadcast: float, + unwrap: float, + mm: float, + n_msgs: int, + n_tokens: int, +) -> None: + if not _STAGE_TIMING: + return + global _RECV_SEQ + _RECV_SEQ += 1 + try: + hist, counters = _ensure_recv_metrics() + for part, value in zip(_RECV_PARTS, (total, pull, broadcast, unwrap, mm)): + hist.labels(dp_rank=str(dp_rank), part=part).observe(value) + if n_msgs: + counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs) + if n_tokens: + counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens) + end_ns = time.time_ns() + start_ns = end_ns - int(total * 1e9) + note_recv_phase( + start_wall_ns=start_ns, + end_wall_ns=end_ns, + parts={ + "sglang.recv_total_ms": total * 1e3, + "sglang.recv_pull_ms": pull * 1e3, + "sglang.recv_broadcast_ms": broadcast * 1e3, + "sglang.recv_unwrap_ms": unwrap * 1e3, + "sglang.recv_mm_ms": mm * 1e3, + "sglang.recv_msgs": int(n_msgs), + "sglang.recv_prompt_tokens": int(n_tokens), + }, + ) + + if not _RECV_SPAN_ENABLED: + return + if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int( + _RECV_SPAN_SAMPLE * 10000 + ): + return + loop = current_loop() + loop_ctx = current_loop_context() + if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None: + # Keep the snapshot only; end_loop emits it if this loop has work. + return + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.recv") + tracer.start_span( + "scheduler.recv_requests", + context=loop_ctx, + start_time=start_ns, + attributes={ + "sglang.phase": "recv", + "sglang.dp_rank": int(dp_rank), + "sglang.recv_msgs": int(n_msgs), + "sglang.recv_prompt_tokens": int(n_tokens), + "sglang.recv_total_ms": total * 1e3, + "sglang.recv_pull_ms": pull * 1e3, + "sglang.recv_broadcast_ms": broadcast * 1e3, + "sglang.recv_unwrap_ms": unwrap * 1e3, + "sglang.recv_mm_ms": mm * 1e3, + }, + ).end(end_time=end_ns) + if loop is not None and loop_ctx is not None: + loop.recv_span_child = True + except Exception: + # Telemetry must never break the scheduler loop. + pass + + +# --------------------------------------------------------------------------- +# Storage prefetch ready latency (same SGLANG_BATCH_STAGE_TIMING gate). +# +# Request-level wait from "prefetch in flight" to "usable" (the scheduler +# check returns True), split by L3 hit/miss. Under wait_complete this is the +# admission delay a request pays for its L3 lookup. +# --------------------------------------------------------------------------- + +_PREFETCH_READY_HIST = None + + +def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None: + if not _STAGE_TIMING: + return + global _PREFETCH_READY_HIST + try: + if _PREFETCH_READY_HIST is None: + from prometheus_client import Histogram + + _PREFETCH_READY_HIST = Histogram( + "sglang:storage_prefetch_ready_seconds", + "Prefetch in-flight -> usable latency, split by L3 hit/miss", + ["dp_rank", "result"], + buckets=(1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0, 3.0, 10.0, 30.0), + ) + _PREFETCH_READY_HIST.labels(dp_rank=str(dp_rank), result=result).observe( + seconds + ) + except Exception: + # Telemetry must never break the scheduler loop. + pass + +# --------------------------------------------------------------------------- +# Scheduler loop span + no-batch accounting (opt-in, same stage-timing gate). +# +# One ``scheduler.loop`` span per scheduler iteration (sampled), with phase +# children (recv / get_next / run_batch / process_result / kv_send). The +# no-batch counters are incremented on *every* loop (sampling only controls +# spans), split by a low-cardinality reason and by whether a forward was in +# flight (forward_overlap). Fully idle loops never emit a span. +# +# Enable with ``SGLANG_TRACE_LOOP_SPAN=1`` (requires SGLANG_BATCH_STAGE_TIMING). +# Knobs: +# SGLANG_TRACE_LOOP_SAMPLE=0.05 head-sample ratio (deterministic per +# mlp_sync_seq, identical on all ranks) +# SGLANG_TRACE_FORWARD_SPAN=0 emit approximate forward.gpu lanes +# SGLANG_TRACE_KV_SEND_SPAN=1 emit the kv_send child (default on +# when the loop span is on) +# --------------------------------------------------------------------------- + +from collections import deque as _deque + +_LOOP_ENABLED = _STAGE_TIMING +_LOOP_SPAN_ENABLED = _LOOP_ENABLED and os.getenv( + "SGLANG_TRACE_LOOP_SPAN", "0" +).strip().lower() not in ("", "0", "false", "no", "off") +_LOOP_SPAN_SAMPLE = float(os.getenv("SGLANG_TRACE_LOOP_SAMPLE", "0.05")) +_FORWARD_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( + "SGLANG_TRACE_FORWARD_SPAN", "0" +).strip().lower() not in ("", "0", "false", "no", "off") +_KV_SEND_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( + "SGLANG_TRACE_KV_SEND_SPAN", "1" +).strip().lower() not in ("", "0", "false", "no", "off") + +_NO_BATCH_REASONS = ( + "fully_idle", + "waiting_prefetch", + "waiting_bootstrap", + "result_backpressure", + "deferred_input", + "admission_blocked", + "waiting_pd", + "no_ready_other", +) + +_LOOP_METRICS = None +_LOOP_TLS = threading.local() + +# Monotonic total of completed forward device time (from the DeviceTimer), +# used to derive one loop's gpu_ms without adding a second timer. +_FORWARD_DEVICE_NS_TOTAL = 0 +_FORWARD_EVENTS = _deque(maxlen=128) # (completion_wall_ns, elapsed_ns, category) + +# Set by dp_attn.prepare_mlp_sync_batch_raw() once per pass. +_LAST_MLP_OUTCOME = ("none", "none") +_LAST_NO_BATCH_REASON = "no_ready_other" + + +def loop_accounting_enabled() -> bool: + return _LOOP_ENABLED + + +def loop_span_enabled() -> bool: + return _LOOP_SPAN_ENABLED + + +def forward_span_enabled() -> bool: + return _FORWARD_SPAN_ENABLED + + +def kv_send_span_enabled() -> bool: + if not _KV_SEND_SPAN_ENABLED: + return False + lt = getattr(_LOOP_TLS, "current", None) + return lt is not None and lt.span is not None + + +def loop_sample_decision(seq: int) -> bool: + """Deterministic, rank-identical loop-span sampling decision.""" + if not _LOOP_SPAN_ENABLED or seq <= 0: + return False + if _LOOP_SPAN_SAMPLE >= 1.0: + return True + if _LOOP_SPAN_SAMPLE <= 0.0: + return False + return (seq * 2654435761) % 10000 < int(_LOOP_SPAN_SAMPLE * 10000) + + +class LoopTimings: + __slots__ = ( + "phase", + "t0_ns", + "t0_wall_ns", + "predicted_seq", + "active0", + "has_work0", + "forward_total0", + "recv", + "get_next", + "run_batch", + "process", + "kv_send_start", + "kv_send_end", + "kv_chunks", + "span", + "span_ctx", + "pass_span_child", + "recv_span_child", + "launch_blocked_reason", + "seq", + "no_batch_reason", + "pass_end_wall_ns", + "arrival_est_wall_ns", + "detail", + ) + + def __init__( + self, + phase: str, + t0_ns: int, + t0_wall_ns: int, + predicted_seq: int, + active0: bool, + has_work0: bool, + forward_total0: int, + ): + self.phase = phase + self.t0_ns = t0_ns + self.t0_wall_ns = t0_wall_ns + self.predicted_seq = predicted_seq + self.active0 = bool(active0) + self.has_work0 = bool(has_work0) + self.forward_total0 = forward_total0 + self.recv = None + self.get_next = None + self.run_batch = None + self.process = None + self.kv_send_start = None + self.kv_send_end = None + self.kv_chunks = 0 + self.span = None + self.span_ctx = None + self.pass_span_child = False + self.recv_span_child = False + self.launch_blocked_reason = "" + self.seq = 0 + self.no_batch_reason = "" + self.pass_end_wall_ns = 0 + self.arrival_est_wall_ns = 0 + # Only populated when SGLANG_TRACE_LOOP_DETAIL=1 and this loop is + # sampled; see the "Loop detail phases" section below. + self.detail = None + + +def ensure_loop_span(lt: Optional[LoopTimings]) -> None: + """Materialize a sampled span once the loop is known to have work.""" + if ( + lt is None + or lt.span is not None + or not loop_sample_decision(lt.predicted_seq) + ): + return + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.prefill.loop") + lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns) + lt.span_ctx = otel_trace.set_span_in_context(lt.span) + except Exception: + lt.span = None + lt.span_ctx = None + + +def note_launch_blocked(lt: Optional[LoopTimings], reason: str) -> None: + """Preserve why this loop could not launch before queues are consumed.""" + if lt is not None and not lt.launch_blocked_reason: + lt.launch_blocked_reason = reason + + +def begin_loop( + phase: str = "prefill", + *, + pre_has_work: bool = True, + forward_active=None, +): + """Start one scheduler-loop accounting record. + + ``pre_has_work`` must be cheap to compute (queue lengths / last_batch): + fully idle loops are still counted, but never create a span. + ``forward_active`` is a callable (usually DeviceTimer.is_active) or a bool. + """ + global _LAST_MLP_OUTCOME + if not _LOOP_ENABLED: + return None + try: + active0 = bool(forward_active() if callable(forward_active) else forward_active) + except Exception: + active0 = False + lt = LoopTimings( + phase=phase, + t0_ns=time.monotonic_ns(), + t0_wall_ns=time.time_ns(), + predicted_seq=_DP_GATHER_SEQ + 1, + active0=active0, + has_work0=bool(pre_has_work), + forward_total0=_FORWARD_DEVICE_NS_TOTAL, + ) + _LAST_MLP_OUTCOME = ("none", "none") + _LOOP_TLS.current = lt + if lt.has_work0: + ensure_loop_span(lt) + return lt + + +def current_loop() -> Optional[LoopTimings]: + return getattr(_LOOP_TLS, "current", None) + + +def current_loop_context(): + lt = getattr(_LOOP_TLS, "current", None) + return lt.span_ctx if lt is not None else None + + +# --------------------------------------------------------------------------- +# Loop detail phases (SGLANG_TRACE_LOOP_DETAIL=1). +# +# Splits run_batch / process_result into host phases. Gated twice: the module +# switch must be on AND the current loop must be sampled (same deterministic +# decision as the loop span), so detail data never covers an unsampled loop. +# +# Accounting rules (see handoff/prefill-cpu-kv-overlap-design-20261106.md §3.2): +# * single phases -> one child span (``scheduler.loop.``) +# * repeated phases -> summed attributes only (``sglang._ms`` + count) +# * durations use time.monotonic_ns(); OTLP start/end use time.time_ns() +# * residuals are clamped >= 0 and reported, never folded into a phase +# --------------------------------------------------------------------------- + +_DETAIL_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( + "SGLANG_TRACE_LOOP_DETAIL", "0" +).strip().lower() not in ("", "0", "false", "no", "off") + +_DETAIL_TLS = threading.local() + +# Chunk-level (Mooncake transfer worker) detail tracing. Independent switch: +# the worker runs on other threads and has no scheduler loop TLS. +_MOONCAKE_DETAIL_ENABLED = os.getenv( + "SGLANG_TRACE_MOONCAKE_DETAIL", "0" +).strip().lower() not in ("", "0", "false", "no", "off") + +# Previous loop summary per scheduler thread, used for DP-skew correlation +# (previous loop's phases vs this loop's arrival skew). +_LAST_LOOP_SUMMARY: dict = {} + +# Phases that are inherently per-request / per-segment: always attributes, +# even when a loop happens to have exactly one occurrence, so span count never +# grows with n_reqs / n_segments (design §4.2). +_DETAIL_ATTRIBUTE_ONLY_PREFIXES = ( + "per_request_result", + "prefix_device_indices", + "kv_state_payload", + "kv_page_indices", + "kv_sender_send", +) + +_DETAIL_PREV_KEYS = ( + "run_batch_ms", + "process_result_ms", + "copy_done_wait_ms", + "kv_state_payload_ms", + "kv_page_indices_ms", + "kv_sender_send_ms", +) + + +def loop_detail_enabled() -> bool: + """True when detail is on and the *current* loop is sampled.""" + if not _DETAIL_ENABLED: + return False + lt = getattr(_LOOP_TLS, "current", None) + return lt is not None and loop_sample_decision(lt.predicted_seq) + + +def mooncake_detail_enabled() -> bool: + return _MOONCAKE_DETAIL_ENABLED + + +def _detail_state(): + """Detail state of the current loop, only when that loop is sampled.""" + if not _DETAIL_ENABLED: + return None + lt = getattr(_LOOP_TLS, "current", None) + if lt is None or not loop_sample_decision(lt.predicted_seq): + return None + if lt.detail is None: + lt.detail = {"phases": {}, "counts": {}, "sums": {}} + return lt.detail + + +def detail_begin(name: str) -> Optional[int]: + """Start a single-occurrence phase. Returns a monotonic-ns token.""" + if not _DETAIL_ENABLED: + return None + state = _detail_state() + if state is None: + return None + now_ns = time.monotonic_ns() + opens = getattr(_DETAIL_TLS, "open", None) + if opens is None: + opens = {} + _DETAIL_TLS.open = opens + opens[name] = (now_ns, time.time_ns(), _thread_time_ns()) + return now_ns + + +def detail_end(name: str, token: Optional[int] = None) -> None: + """Close a phase opened by ``detail_begin`` (or started at ``token``).""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + now_ns = time.monotonic_ns() + opens = getattr(_DETAIL_TLS, "open", None) or {} + entry = opens.pop(name, None) + start_wall_ns = entry[1] if entry is not None else None + thread_cpu_ns = None + if entry is not None and len(entry) > 2 and entry[2] is not None: + cpu_now = _thread_time_ns() + if cpu_now is not None: + thread_cpu_ns = max(0, cpu_now - int(entry[2])) + if token is None: + if entry is None: + return + token = entry[0] + if start_wall_ns is None: + start_wall_ns = time.time_ns() - max(0, now_ns - int(token)) + _detail_record( + state, + name, + int(token), + now_ns, + start_wall_ns, + time.time_ns(), + thread_cpu_ns=thread_cpu_ns, + ) + + +def detail_add( + name: str, + start_ns: int, + end_ns: int, + thread_cpu_ns: Optional[int] = None, +) -> None: + """Accumulate one occurrence of a phase (no span per occurrence). + + ``thread_cpu_ns`` is optional: callers that did not measure CPU time must + leave it as None so reports can tell "not measured" apart from "zero". + """ + if not _DETAIL_ENABLED or start_ns is None or end_ns is None: + return + state = _detail_state() + if state is None: + return + start_ns = int(start_ns) + end_ns = int(end_ns) + if end_ns < start_ns: + end_ns = start_ns + now_wall_ns = time.time_ns() + _detail_record( + state, + name, + start_ns, + end_ns, + now_wall_ns - (end_ns - start_ns), + now_wall_ns, + thread_cpu_ns=thread_cpu_ns, + ) + + +def _detail_record( + state, + name, + start_ns, + end_ns, + start_wall_ns, + end_wall_ns, + thread_cpu_ns: Optional[int] = None, +) -> None: + rec = state["phases"].get(name) + if rec is None: + rec = [end_ns - start_ns, 1, int(start_wall_ns), int(end_wall_ns), 0, 0] + state["phases"][name] = rec + else: + rec[0] += end_ns - start_ns + rec[1] += 1 + rec[3] = int(end_wall_ns) + if thread_cpu_ns is not None: + rec[4] += int(thread_cpu_ns) + rec[5] += 1 + + +def thread_cpu_ns() -> Optional[int]: + """Thread CPU clock, or None when the platform has no such clock. + + Callers outside this module (the scheduler's parent span) must use this + wrapper so an unsupported or failing clock degrades to "not measured" + instead of raising on the scheduling path. + """ + try: + return time.thread_time_ns() + except (AttributeError, OSError, ValueError): + return None + + +# Backwards-compatible private alias for module-local call sites. +_thread_time_ns = thread_cpu_ns + + +def detail_attr_setdefault(name: str, value) -> None: + """Set a loop attribute only if this loop has not recorded it yet.""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + attrs = state.setdefault("attrs", {}) + if name not in attrs: + attrs[name] = value + + +def detail_attr_sum(name: str, value: int) -> None: + """Accumulate a numeric loop attribute (e.g. evicted tokens).""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + sums = state.setdefault("sums", {}) + sums[name] = int(sums.get(name, 0)) + int(value) + + +def detail_count(name: str, n: int = 1) -> None: + """Accumulate a low-cardinality counter for this loop.""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + state["counts"][name] = int(state["counts"].get(name, 0)) + int(n) + + +def detail_attr(name: str, value) -> None: + """Set a low-cardinality loop attribute (last write wins).""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + state.setdefault("attrs", {})[name] = value + + +def detail_phase_total_ns(names) -> int: + """Sum of accumulated ns for the given phases in the current loop. + + ``names`` are *base* names: a phase recorded as ``kv_sender_send.last`` + (per send kind) is included when ``kv_sender_send`` is requested. Phase + names are only generated by this module, so prefix matching cannot pick up + an unrelated phase that merely shares a leading string. + """ + state = _detail_state() + if state is None: + return 0 + phases = state.get("phases") or {} + total = 0 + for name, rec in phases.items(): + if rec is None: + continue + for want in names: + if name == want or name.startswith(want + "."): + total += int(rec[0]) + break + return total + + +class detail_phase: + """Context manager for one single-occurrence phase (cheap when disabled).""" + + __slots__ = ("name", "token") + + def __init__(self, name: str): + self.name = name + self.token = None + + def __enter__(self): + self.token = detail_begin(self.name) + return self + + def __exit__(self, exc_type, exc, tb): + detail_end(self.name, self.token) + return False + + +# --------------------------------------------------------------------------- +# HiCache flush micro timing (SGLANG_TRACE_BACKUP_DETAIL=1). +# +# One tracer per ``cache_controller.start_writing()`` call (i.e. per +# ``flush_pending_backups``). Sub-phases are host-wall timings on the caller +# thread; a synchronous ``.cpu()`` therefore shows up as a long host phase and +# must not be read as device-transfer time. The tracer keeps per-flush +# resolution by emitting ONE child span of the current loop +# (``scheduler.cache_flush_backup``), because the loop-level detail only keeps +# per-loop sums (design: handoff/prefill-flush-backups-validation-20261107.md). +# +# Disabled by default: every entry point is a single module-level bool check, +# and when off no object, dict or span is created. +# --------------------------------------------------------------------------- + +_BACKUP_DETAIL_ENABLED = _DETAIL_ENABLED and os.getenv( + "SGLANG_TRACE_BACKUP_DETAIL", "0" +).strip().lower() not in ("", "0", "false", "no", "off") + +_FLUSH_TLS = threading.local() + + +def backup_detail_enabled() -> bool: + """True when per-flush micro timing is compiled in (``*_DETAIL`` is on).""" + return _BACKUP_DETAIL_ENABLED + + +def set_flush_origin(origin: Optional[str]) -> Optional[str]: + """Tag the next flushes with the caller's scheduler context. + + Only used to tell ``process_batch_result`` flushes (the current suspect) + from the ones issued by ``check_hicache_events`` / ``write``. Returns the + previous value so the caller can restore it. + """ + if not _BACKUP_DETAIL_ENABLED: + return None + previous = getattr(_FLUSH_TLS, "origin", None) + _FLUSH_TLS.origin = origin + return previous + + +def current_flush_origin() -> str: + return getattr(_FLUSH_TLS, "origin", None) or "other" + + +class BackupFlushTracer: + """Sub-phase timings for one flush; emits a single span on ``end``.""" + + __slots__ = ( + "origin", + "start_ns", + "loop_seq", + "loop_ctx", + "phases", + "attrs", + "counts", + "_open", + "_prev", + ) + + def __init__(self, origin: str): + self.origin = origin + # Captured on the scheduler thread: the flushed span may be emitted by + # the async submit worker, which has no scheduler loop TLS. + self.loop_seq = current_loop_seq() + self.loop_ctx = current_loop_context_for_spans() + self.start_ns = time.monotonic_ns() + self.phases: dict[str, int] = {} + self.attrs: dict[str, Any] = {} + self.counts: dict[str, int] = {} + self._open: dict[str, int] = {} + self._prev = None + + # -- phase helpers ----------------------------------------------------- + def begin(self, name: str) -> int: + start = time.monotonic_ns() + self._open[name] = start + return start + + def end(self, name: str, start_ns: Optional[int] = None) -> None: + start = self._open.pop(name, None) + if start is None: + start = start_ns + if start is None: + return + self.add(name, start) + + def add(self, name: str, start_ns: int) -> None: + self.record(name, int(start_ns), time.monotonic_ns()) + + def record(self, name: str, start_ns: int, end_ns: int) -> None: + self.phases[name] = self.phases.get(name, 0) + max( + 0, int(end_ns) - int(start_ns) + ) + + def set(self, key: str, value: Any) -> None: + self.attrs[key] = value + + def inc(self, key: str, n: int = 1) -> None: + self.counts[key] = self.counts.get(key, 0) + int(n) + + def total_ns(self) -> int: + return max(0, time.monotonic_ns() - self.start_ns) + + # -- emission ---------------------------------------------------------- + def end_and_emit(self) -> None: + now_ns = time.monotonic_ns() + total_ns = max(0, now_ns - self.start_ns) + attrs = { + "sglang.flush_origin": self.origin, + "sglang.flush_total_ms": total_ns / 1e6, + "sglang.loop_seq": int(self.loop_seq or 0), + } + for name, ns in self.phases.items(): + attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6 + for name, value in self.counts.items(): + attrs["sglang.flush_" + name] = int(value) + attrs.update(self.attrs) + # OTel rejects None; keep a single malformed dimension from dropping the + # whole span. + attrs = {k: v for k, v in attrs.items() if v is not None} + try: + from opentelemetry import trace as otel_trace + + ctx = self.loop_ctx + start_wall_ns = time.time_ns() - total_ns + span = otel_trace.get_tracer("sglang.cache.flush").start_span( + "scheduler.cache_flush_backup", + context=ctx, + start_time=int(start_wall_ns), + attributes=attrs, + ) + span.end(end_time=int(time.time_ns())) + except Exception: + # Telemetry must never break the cache controller. + pass + + +class backup_phase: + """Context manager for one sub-phase of the active flush (no-op when off).""" + + __slots__ = ("name", "tracer", "start_ns") + + def __init__(self, name: str): + self.name = name + self.tracer = None + self.start_ns = None + + def __enter__(self): + if not _BACKUP_DETAIL_ENABLED: + return self + self.tracer = getattr(_FLUSH_TLS, "tracer", None) + if self.tracer is not None: + self.start_ns = self.tracer.begin(self.name) + return self + + def __exit__(self, exc_type, exc, tb): + if self.tracer is not None: + self.tracer.end(self.name, self.start_ns) + return False + + +def backup_flush_begin(origin: Optional[str] = None) -> Optional[BackupFlushTracer]: + """Start tracing one flush; returns None when disabled / not sampled.""" + if not _BACKUP_DETAIL_ENABLED: + return None + if not loop_detail_enabled(): + return None + tracer = BackupFlushTracer(origin or current_flush_origin()) + tracer._prev = getattr(_FLUSH_TLS, "tracer", None) + _FLUSH_TLS.tracer = tracer + return tracer + + +def backup_flush_end(tracer: Optional[BackupFlushTracer]) -> None: + if tracer is None: + return + if getattr(_FLUSH_TLS, "tracer", None) is tracer: + _FLUSH_TLS.tracer = tracer._prev + try: + tracer.end_and_emit() + except Exception: + pass + + +def current_flush_tracer() -> Optional[BackupFlushTracer]: + if not _BACKUP_DETAIL_ENABLED: + return None + return getattr(_FLUSH_TLS, "tracer", None) + + +def backup_flush_attach(tracer: Optional[BackupFlushTracer]): + """Bind ``tracer`` to this thread so sub-phases land on the same span. + + Used by the async submit worker, which executes phases on a different + thread than the one that created the tracer. Returns the previous binding. + """ + if not _BACKUP_DETAIL_ENABLED: + return None + previous = getattr(_FLUSH_TLS, "tracer", None) + _FLUSH_TLS.tracer = tracer + return previous + + +def backup_flush_detach(previous) -> None: + if not _BACKUP_DETAIL_ENABLED: + return + _FLUSH_TLS.tracer = previous + + +def flush_note(key: str, value: Any) -> None: + """Attach a dimension to the active flush (no-op when disabled).""" + if not _BACKUP_DETAIL_ENABLED: + return + tracer = getattr(_FLUSH_TLS, "tracer", None) + if tracer is not None: + tracer.set(key, value) + + +def flush_count(key: str, n: int = 1) -> None: + """Accumulate a counter on the active flush (no-op when disabled).""" + if not _BACKUP_DETAIL_ENABLED: + return + tracer = getattr(_FLUSH_TLS, "tracer", None) + if tracer is not None: + tracer.inc(key, n) + + +def flush_mark() -> Optional[int]: + """Timestamp for a manual phase (None when no flush is being traced).""" + if not _BACKUP_DETAIL_ENABLED: + return None + if getattr(_FLUSH_TLS, "tracer", None) is None: + return None + return time.monotonic_ns() + + +def flush_phase_since(name: str, start_ns: Optional[int]) -> None: + """Record ``name`` from ``start_ns`` to now (no-op when untraced).""" + if start_ns is None: + return + tracer = getattr(_FLUSH_TLS, "tracer", None) + if tracer is not None: + tracer.add(name, start_ns) + + +def current_loop_seq() -> int: + """Loop sequence for cross-record correlation (0 when unavailable).""" + lt = getattr(_LOOP_TLS, "current", None) + if lt is None: + return 0 + return int(lt.seq or lt.predicted_seq or 0) + + +def current_loop_context_for_spans(): + """OTel context of the current loop span (None if unsampled).""" + lt = getattr(_LOOP_TLS, "current", None) + return lt.span_ctx if lt is not None else None + + +def _chunk_parent_context(chunk): + trace_ctx = getattr(chunk, "trace_ctx", None) + return getattr(trace_ctx, "root_span_context", None) + + +def split_queue_wait( + queue_wait_accum_ns: int, last_enqueue_mono_ns: int, dequeue_mono_ns: int +) -> tuple: + """Return ``(attempt_queue_wait_ns, total_queue_wait_ns)``. + + The attempt value matches the last_enqueue->dequeue wall window used by the + ``mooncake.queue_wait`` child span; the total value additionally includes + the queue waits of earlier deferred attempts (design §5.3 metric). + Keeping them separate prevents a span whose wall length disagrees with its + ``wait_ms`` attribute (review finding). + """ + attempt = max(0, int(dequeue_mono_ns) - int(last_enqueue_mono_ns)) + return attempt, max(0, int(queue_wait_accum_ns)) + attempt + + +class RoomDetailState: + """Thread-safe per-room detail bookkeeping (scheduler vs worker threads). + + The scheduler aborts/retracts a room while the transfer worker may still be + finishing an attempt. A reset therefore records a cancellation mark, and + the worker's join update re-checks under the same lock before writing, so a + late worker can never resurrect stats after abort (review finding). + """ + + # Bounded: a marker only needs to survive until the in-flight worker + # attempt finishes (milliseconds). The cap prevents unbounded growth if an + # aborted room never gets a worker teardown. + _MAX_CANCELLED = 8192 + + def __init__(self): + self._lock = threading.Lock() + self._join: dict = {} + self._seq: dict = {} + self._first: dict = {} + self._cancelled: dict = {} + + def _mark_cancelled(self, room: int) -> None: + self._cancelled[int(room)] = time.monotonic_ns() + while len(self._cancelled) > self._MAX_CANCELLED: + self._cancelled.pop(next(iter(self._cancelled))) + + def begin_chunk(self, room: int, now_mono_ns: int, now_wall_ns: int): + """Stamp a real enqueue. Returns ``(chunk_seq, first_enqueue_wall_ns)``. + + A new chunk clears a previous cancellation mark: it is the start of a + fresh flow (a retry may land on the same room id). + """ + with self._lock: + self._cancelled.pop(int(room), None) + seq = int(self._seq.get(int(room), 0)) + self._seq[int(room)] = seq + 1 + first = self._first.get(int(room)) + if first is None: + first = (int(now_mono_ns), int(now_wall_ns)) + self._first[int(room)] = first + return seq, int(first[1]) + + def record_join( + self, + room: int, + *, + status: str, + room_failed: bool, + now_wall_ns: int, + queue_wait_total_ns: int, + worker_busy_ns: int, + enqueue_wall_ns: int, + is_last_chunk: bool, + chunk_seq: int, + ) -> bool: + """Update the per-room join summary; False when the write was refused.""" + if status != "ok": + return False + with self._lock: + if room_failed or int(room) in self._cancelled: + return False + stats = self._join.get(int(room)) + if stats is None: + stats = {"chunk_count": 0} + self._join[int(room)] = stats + stats["chunk_count"] = int(stats.get("chunk_count", 0)) + 1 + first = self._first.get(int(room)) + if first is not None: + stats["first_enqueue_wall_ns"] = int(first[1]) + if is_last_chunk: + stats["last_chunk_enqueue_wall_ns"] = int(enqueue_wall_ns) + stats["last_chunk_complete_wall_ns"] = int(now_wall_ns) + stats["last_chunk_queue_wait_ms"] = queue_wait_total_ns / 1e6 + stats["last_chunk_worker_ms"] = worker_busy_ns / 1e6 + stats["last_chunk_chunk_seq"] = int(chunk_seq) + return True + + def reset(self, room: int) -> None: + with self._lock: + self._join.pop(int(room), None) + self._seq.pop(int(room), None) + self._first.pop(int(room), None) + self._mark_cancelled(room) + + def teardown(self, room: int, keep_join: bool) -> None: + with self._lock: + if not keep_join: + self._join.pop(int(room), None) + self._seq.pop(int(room), None) + self._first.pop(int(room), None) + self._cancelled.pop(int(room), None) + + def pop_join(self, room: int): + with self._lock: + return self._join.pop(int(room), None) + + +def note_chunk_requeue(chunk, now_mono_ns: int, now_wall_ns: int) -> None: + """Accumulate only the *queue* part of a finished attempt before re-enqueue. + + The previous implementation added ``requeue - last_enqueue``, which also + contains the worker's own execution and the staging watermark wait, thus + inflating queue wait on every defer (review finding). Only the interval + from the last enqueue to this attempt's dequeue is queue wait. + """ + if not _MOONCAKE_DETAIL_ENABLED or not getattr(chunk, "observe", False): + return + last_enqueue = int(getattr(chunk, "last_enqueue_mono_ns", 0) or 0) + dequeue = int(getattr(chunk, "dequeue_mono_ns", 0) or 0) + if last_enqueue and dequeue: + chunk.queue_wait_accum_ns = int( + getattr(chunk, "queue_wait_accum_ns", 0) or 0 + ) + max(0, dequeue - last_enqueue) + chunk.last_enqueue_mono_ns = int(now_mono_ns) + chunk.last_enqueue_wall_ns = int(now_wall_ns) + chunk.dequeue_mono_ns = 0 + chunk.requeue_count = int(getattr(chunk, "requeue_count", 0) or 0) + 1 + + +def emit_mooncake_chunk( + *, + chunk, + status: str, + queue_wait_ns: int, + queue_wait_total_ns: int, + source_wait_ns: int, + indices_wait_ns: int, + kv_transfer_ns: int, + state_transfer_ns: int, + aux_transfer_ns: int, + attempt_wall_ns: int, + dequeue_wall_ns: int, + end_wall_ns: int, + n_destinations: int, + source_wait_wall=None, + indices_wait_wall=None, + dp_rank: int = -1, + tp_rank: int = -1, +) -> None: + """One span per transfer-worker attempt, attributed by room/chunk_seq. + + ``worker_busy_ms`` follows the design definition: attempt wall time minus + the source/indices event waits (those are blocking waits, not worker work). + ``attempt_wall_ms`` is the raw dequeue->end time. + + Wait child spans use the real event timestamps captured by the worker. + A wait without a recorded window is attributes-only; no synthetic + ``end - duration`` placement is drawn (review finding). + """ + if not _MOONCAKE_DETAIL_ENABLED: + return + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.mooncake.chunk") + # Attempt-scoped start: the last enqueue of this attempt, not the + # first enqueue of the chunk, so deferred retries do not overlap. + start_wall_ns = int( + getattr(chunk, "last_enqueue_wall_ns", 0) + or chunk.enqueue_wall_ns + or dequeue_wall_ns + ) or end_wall_ns + worker_busy_ns = max( + 0, int(attempt_wall_ns) - int(source_wait_ns) - int(indices_wait_ns) + ) + residual_ns = max( + 0, + worker_busy_ns + - int(kv_transfer_ns) + - int(state_transfer_ns) + - int(aux_transfer_ns), + ) + span = tracer.start_span( + "mooncake.kv_chunk", + context=_chunk_parent_context(chunk), + start_time=start_wall_ns, + attributes={ + "sglang.bootstrap_room": int(chunk.room), + "sglang.chunk_seq": int(chunk.chunk_seq), + "sglang.loop_seq": int(chunk.loop_seq), + "sglang.send_kind": str(chunk.send_kind or "unknown"), + "sglang.is_last_chunk": bool(chunk.is_last_chunk), + "sglang.queue_id": int(chunk.queue_id), + "sglang.requeue_count": int(chunk.requeue_count), + "sglang.num_kv_tokens": int(chunk.num_kv_tokens or 0), + "sglang.num_pages": int(chunk.num_pages), + "sglang.approx_bytes": int(chunk.approx_bytes), + "sglang.n_destinations": int(n_destinations), + "sglang.chunk_status": str(status), + "sglang.dp_rank": int(dp_rank), + "sglang.tp_rank": int(tp_rank), + # Attempt window (matches the child span) vs cumulative wait + # across deferred attempts (design §5.3 metric). + "sglang.queue_wait_ms": queue_wait_ns / 1e6, + "sglang.queue_wait_total_ms": queue_wait_total_ns / 1e6, + "sglang.source_wait_ms": source_wait_ns / 1e6, + "sglang.indices_wait_ms": indices_wait_ns / 1e6, + "sglang.kv_transfer_ms": kv_transfer_ns / 1e6, + "sglang.state_transfer_ms": state_transfer_ns / 1e6, + "sglang.aux_transfer_ms": aux_transfer_ns / 1e6, + "sglang.attempt_wall_ms": attempt_wall_ns / 1e6, + "sglang.worker_busy_ms": worker_busy_ns / 1e6, + "sglang.worker_residual_ms": residual_ns / 1e6, + }, + ) + span_ctx = otel_trace.set_span_in_context(span) + + def _child(name, start_ns, stop_ns, wait_ns): + start_ns = int(start_ns or 0) + stop_ns = int(stop_ns or 0) + if start_ns <= 0 or stop_ns <= start_ns: + return + try: + tracer.start_span( + name, + context=span_ctx, + start_time=start_ns, + attributes={"sglang.wait_ms": max(0, int(wait_ns)) / 1e6}, + ).end(end_time=stop_ns) + except Exception: + pass + + # Real event windows only; queue wait ends at dequeue. Zero-wait + # attempts (immediate dequeue) get no span. + if queue_wait_ns > 0: + _child( + "mooncake.queue_wait", + getattr(chunk, "last_enqueue_wall_ns", 0), + dequeue_wall_ns, + queue_wait_ns, + ) + if source_wait_wall is not None: + _child( + "mooncake.wait_source", + source_wait_wall[0], + source_wait_wall[1], + source_wait_ns, + ) + if indices_wait_wall is not None: + _child( + "mooncake.wait_indices", + indices_wait_wall[0], + indices_wait_wall[1], + indices_wait_ns, + ) + span.end(end_time=int(end_wall_ns)) + except Exception: + # Telemetry must never break the transfer worker. + pass + + +def emit_request_kv_join( + *, + req, + stats: dict, + dp_rank: int = -1, + completion_wall_ns: Optional[int] = None, +) -> None: + """Request-level join summary: last-chunk exposure + completion poll.""" + if not _MOONCAKE_DETAIL_ENABLED or not stats: + return + try: + from opentelemetry import trace as otel_trace + from opentelemetry.trace import Link + + now_wall_ns = int(completion_wall_ns or time.time_ns()) + enqueue_wall_ns = int(stats.get("last_chunk_enqueue_wall_ns", 0) or 0) + complete_wall_ns = int(stats.get("last_chunk_complete_wall_ns", 0) or 0) + final_exposed_ns = ( + max(0, complete_wall_ns - enqueue_wall_ns) + if enqueue_wall_ns and complete_wall_ns + else 0 + ) + completion_poll_ns = ( + max(0, now_wall_ns - complete_wall_ns) if complete_wall_ns else 0 + ) + # First enqueue comes from the manager's real queue.put() event + # (wall clock), not from pre-send preparation (review finding). + first_enqueue_wall_ns = int(stats.get("first_enqueue_wall_ns", 0) or 0) + first_enqueue_to_completion_ms = None + if first_enqueue_wall_ns: + first_enqueue_to_completion_ms = max( + 0.0, (now_wall_ns - first_enqueue_wall_ns) / 1e6 + ) + req_ctx = _request_span_context(req) + attrs = { + "sglang.bootstrap_room": int(getattr(req, "bootstrap_room", 0) or 0), + "sglang.rid": str(getattr(req, "rid", "")), + "sglang.dp_rank": int(dp_rank), + "sglang.chunk_count": int(stats.get("chunk_count", 0)), + "sglang.last_chunk_queue_wait_ms": float( + stats.get("last_chunk_queue_wait_ms", 0.0) + ), + "sglang.last_chunk_worker_ms": float( + stats.get("last_chunk_worker_ms", 0.0) + ), + "sglang.final_exposed_ms": final_exposed_ns / 1e6, + "sglang.completion_poll_ms": completion_poll_ns / 1e6, + # Per-rank request join. rank_join_skew_ms (max-min over TP/DP + # ranks) is computed offline by joining this span by room. + "sglang.request_join_ms": (final_exposed_ns + completion_poll_ns) / 1e6, + } + if first_enqueue_to_completion_ms is not None: + attrs["sglang.first_enqueue_to_completion_ms"] = ( + first_enqueue_to_completion_ms + ) + tracer = otel_trace.get_tracer("sglang.kv.request") + tracer.start_span( + "kv.request_join", + # Cover the whole request transfer when first enqueue is known. + start_time=first_enqueue_wall_ns or enqueue_wall_ns or now_wall_ns, + links=[Link(req_ctx)] if req_ctx is not None else [], + attributes=attrs, + ).end(end_time=now_wall_ns) + except Exception: + pass + + +def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None: + """Record the recv_requests phase inside the current loop (if any).""" + lt = getattr(_LOOP_TLS, "current", None) + if lt is not None: + lt.recv = (int(start_wall_ns), int(end_wall_ns), dict(parts)) + + +def note_pass_snapshot(t: "PassTimings", total_s: float) -> None: + """Record get_next timings for the current loop (called from end_pass).""" + lt = getattr(_LOOP_TLS, "current", None) + if lt is None: + return + end_wall_ns = time.time_ns() + start_wall_ns = end_wall_ns - int(total_s * 1e9) + measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync + other = max(0.0, total_s - measured) + lt.get_next = ( + start_wall_ns, + end_wall_ns, + { + "sglang.get_next_ms": total_s * 1e3, + "sglang.match_ms": t.match * 1e3, + "sglang.adder_ms": t.adder * 1e3, + "sglang.hicache_ms": t.hicache * 1e3, + "sglang.bootstrap_ms": t.bootstrap * 1e3, + "sglang.mlp_sync_ms": t.mlp_sync * 1e3, + "sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3, + "sglang.other_ms": other * 1e3, + "sglang.candidates": int(t.candidates), + "sglang.skipped_prefetch": int(t.skipped_prefetch), + "sglang.admitted": int(t.admitted), + "sglang.bootstrap_candidates": int(t.bootstrap_candidates), + "sglang.pending_bootstrap": int(t.pending_bootstrap), + "sglang.resumed_chunked": int(bool(t.had_chunked)), + "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), + "sglang.mlp_sync_seq": int(t.seq), + }, + ) + lt.seq = int(t.seq) + lt.pass_end_wall_ns = end_wall_ns + # Estimated DP-sync arrival: the pass ended, then mlp_sync waited. + # See design §1.2: arrival_est = pass_end - mlp_sync. + lt.arrival_est_wall_ns = end_wall_ns - int(t.mlp_sync * 1e9) + + +def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): + lt = getattr(_LOOP_TLS, "current", None) + if lt is not None: + attrs = dict(attrs or {}) + attrs.setdefault( + "sglang.run_batch_ms", max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6 + ) + lt.run_batch = (int(start_wall_ns), int(end_wall_ns), attrs) + + +def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): + lt = getattr(_LOOP_TLS, "current", None) + if lt is not None: + attrs = dict(attrs or {}) + attrs.setdefault( + "sglang.process_result_ms", + max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6, + ) + lt.process = (int(start_wall_ns), int(end_wall_ns), attrs) + + +def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None: + lt = getattr(_LOOP_TLS, "current", None) + if lt is None: + return + lt.kv_chunks += 1 + if lt.kv_send_start is None or start_wall_ns < lt.kv_send_start: + lt.kv_send_start = int(start_wall_ns) + if lt.kv_send_end is None or end_wall_ns > lt.kv_send_end: + lt.kv_send_end = int(end_wall_ns) + + +def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None: + """Set by dp_attn.prepare_mlp_sync_batch_raw once per pass.""" + if not _LOOP_ENABLED: + return + global _LAST_MLP_OUTCOME + _LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome)) + if local_kind != "none" or global_outcome != "none": + # DP idle work is only known after the sync. Activate the span before + # end_pass so the get_next child can share the loop trace. + ensure_loop_span(getattr(_LOOP_TLS, "current", None)) + + +def note_forward_device(t: float, category: str = "", **kwargs) -> None: + """DeviceTimer reporter: accumulate completed forward device time.""" + global _FORWARD_DEVICE_NS_TOTAL + try: + ns = int(float(t) * 1e9) + except Exception: + return + _FORWARD_DEVICE_NS_TOTAL += ns + if _FORWARD_SPAN_ENABLED: + _FORWARD_EVENTS.append((time.time_ns(), ns, str(category))) + + +def install_forward_reporter(scheduler) -> None: + """Attach the forward device-time reporter to the existing DeviceTimer. + + No CUDA event is added here: SGLANG_ENABLE_METRICS_DEVICE_TIMER already + runs the timer; this only reads its completed intervals. + """ + if not _LOOP_ENABLED: + return + try: + reporter = getattr(scheduler, "metrics_reporter", None) + timer = getattr(reporter, "forward_pass_device_timer", None) + if timer is not None: + timer.add_reporter(note_forward_device) + except Exception: + # Telemetry must never break scheduler init. + pass + + +def _ensure_loop_metrics(): + global _LOOP_METRICS + if _LOOP_METRICS is None: + from prometheus_client import Counter + + _LOOP_METRICS = { + "no_batch_seconds": Counter( + "sglang:scheduler_no_batch_seconds_total", + "Wall time of scheduler loops that ended without a runnable batch, " + "by reason and forward_overlap", + ["dp_rank", "reason", "forward_overlap"], + ), + "no_batch_total": Counter( + "sglang:scheduler_no_batch_total", + "Count of scheduler loops that ended without a runnable batch, " + "by reason and forward_overlap", + ["dp_rank", "reason", "forward_overlap"], + ), + "lockstep_seconds": Counter( + "sglang:scheduler_dp_lockstep_idle_seconds_total", + "Wall time of DP-lockstep idle-forward loops (this rank had no " + "local batch but the DP group required an idle forward)", + ["dp_rank", "forward_overlap"], + ), + } + return _LOOP_METRICS + + +def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str: + """Low-cardinality no-batch reason, in the design's priority order. + + ``pass_snapshot`` is the get_next attribute dict (candidates / skipped / + admitted). It is None when the loop skipped batch preparation entirely. + """ + if state.get("result_queue", 0) >= state.get("result_limit", 2): + return "result_backpressure" + if state.get("deferred", 0) > 0 or state.get("paused"): + return "deferred_input" + waiting = int(state.get("waiting", 0) or 0) + if waiting > 0 and pass_snapshot is not None: + bootstrap_candidates = int( + pass_snapshot.get("sglang.bootstrap_candidates", 0) or 0 + ) + pending_bootstrap = int( + pass_snapshot.get("sglang.pending_bootstrap", 0) or 0 + ) + if bootstrap_candidates > 0 and pending_bootstrap == bootstrap_candidates: + return "waiting_bootstrap" + candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0) + skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0) + admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0) + if candidates > 0 and skipped == candidates: + return "waiting_prefetch" + if admitted == 0: + return "admission_blocked" + inflight = int(state.get("inflight", 0) or 0) + bootstrap = int(state.get("bootstrap", 0) or 0) + if waiting == 0 and bootstrap: + return "waiting_bootstrap" + if waiting == 0 and inflight: + return "waiting_pd" + if ( + waiting == 0 + and inflight == 0 + and bootstrap == 0 + and int(state.get("result_queue", 0) or 0) == 0 + and int(state.get("deferred", 0) or 0) == 0 + and not state.get("waiting_for_result") + ): + return "fully_idle" + if state.get("skip_batch_creation") and pass_snapshot is None: + # A failed preparation defers retries; keep the last classified reason. + return _LAST_NO_BATCH_REASON + return "no_ready_other" + + +def _batch_attrs(batch) -> dict: + if batch is None: + return { + "sglang.n_reqs": 0, + "sglang.n_new": 0, + "sglang.n_cached": 0, + "sglang.chunked": False, + } + try: + n_reqs = int(batch.batch_size()) + except Exception: + n_reqs = len(getattr(batch, "reqs", ()) or ()) + try: + n_new = int(getattr(batch, "extend_num_tokens", 0) or 0) + except Exception: + n_new = 0 + try: + n_cached = int(sum(getattr(batch, "prefix_lens", ()) or ())) + except Exception: + n_cached = 0 + try: + chunked = getattr(batch, "chunked_req", None) is not None + except Exception: + chunked = False + return { + "sglang.n_reqs": n_reqs, + "sglang.n_new": n_new, + "sglang.n_cached": n_cached, + "sglang.chunked": bool(chunked), + } + + +_LAST_LOOP_SEQ = 0 +_LAST_LOOP_HAS_DETAIL = False + + +def _total_ms(ms: dict, base: str) -> float: + return sum( + value for name, value in ms.items() if name == base or name.startswith(base + ".") + ) + + +def _update_prev_state(lt: LoopTimings, loop_seq: int): + """Capture the previous loop summary and store this loop for the next one. + + Returns ``(prev_seq, prev_has_detail, prev_summary, consecutive)``. Only a + loop that actually recorded detail becomes a valid predecessor, so a + sampled loop after an unsampled one is marked non-consecutive instead of + silently publishing a stale window (design §4.5). + """ + global _LAST_LOOP_SEQ, _LAST_LOOP_HAS_DETAIL + prev_seq = int(_LAST_LOOP_SEQ) + prev_has = bool(_LAST_LOOP_HAS_DETAIL) + prev_summary = dict(_LAST_LOOP_SUMMARY) + consecutive = prev_has and prev_seq == loop_seq - 1 + + ms = { + name: rec[0] / 1e6 + for name, rec in ((lt.detail or {}).get("phases") or {}).items() + if rec + } + _LAST_LOOP_SUMMARY.clear() + if lt.detail is not None: + _LAST_LOOP_SUMMARY.update( + { + "run_batch_ms": float( + lt.run_batch[2].get("sglang.run_batch_ms", 0.0) + ) + if lt.run_batch is not None + else 0.0, + "process_result_ms": float( + lt.process[2].get("sglang.process_result_ms", 0.0) + ) + if lt.process is not None + else 0.0, + "copy_done_wait_ms": _total_ms(ms, "copy_done_wait"), + "kv_state_payload_ms": _total_ms(ms, "kv_state_payload"), + "kv_page_indices_ms": _total_ms(ms, "kv_page_indices"), + "kv_sender_send_ms": _total_ms(ms, "kv_sender_send"), + } + ) + _LAST_LOOP_SEQ = int(loop_seq) + _LAST_LOOP_HAS_DETAIL = lt.detail is not None + return prev_seq, prev_has, prev_summary, consecutive + + +def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None: + """Expose detail phases on the loop span and its child spans. + + Single-occurrence phases become children (``scheduler.loop.``); + repeated phases only contribute summed attributes, so span count never + grows with n_reqs / n_segments. + """ + detail = lt.detail or {} + phases = detail.get("phases") or {} + ms = {} + accounting_anomalies = 0 + for name, rec in phases.items(): + try: + total_ns, count, start_wall, end_wall = rec[:4] + except Exception: + continue + key = "sglang." + name.replace(".", "_") + "_ms" + ms[name] = total_ns / 1e6 + attrs[key] = ms[name] + cpu_ns, cpu_count = (rec[4], rec[5]) if len(rec) > 5 else (0, 0) + prefix = "sglang." + name.replace(".", "_") + non_cpu_ms = None + if cpu_count: + attrs[prefix + "_thread_cpu_ms"] = cpu_ns / 1e6 + attrs[prefix + "_thread_cpu_measured_count"] = int(cpu_count) + # Only a fully measured phase may report the wall - CPU remainder; + # a partial measurement would silently attribute unmeasured time to + # "non-CPU". + if cpu_count == count: + non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6) + attrs[prefix + "_non_cpu_ms"] = non_cpu_ms + # Only a deficit beyond clock granularity is an accounting + # anomaly; a few hundred ns of skew on a 2 ms phase is noise. + deficit_ns = cpu_ns - total_ns + if deficit_ns > max(100_000, total_ns // 100): + attrs[prefix + "_non_cpu_raw_ms"] = -deficit_ns / 1e6 + attrs[prefix + "_accounting_anomaly"] = True + accounting_anomalies += 1 + if count > 1: + attrs[prefix + "_count"] = int(count) + continue + if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES: + continue + child_attrs = { + "sglang.dp_rank": int(dp_rank), + "sglang.phase_ms": total_ns / 1e6, + } + if cpu_count: + child_attrs["sglang.phase_thread_cpu_ms"] = cpu_ns / 1e6 + child_attrs["sglang.phase_cpu_measured_count"] = int(cpu_count) + if non_cpu_ms is not None: + child_attrs["sglang.phase_non_cpu_ms"] = non_cpu_ms + child( + "scheduler.loop." + name.replace(".", "_"), + start_wall, + end_wall, + child_attrs, + ) + + if accounting_anomalies: + attrs["sglang.accounting_anomaly_count"] = int(accounting_anomalies) + for name, value in (detail.get("counts") or {}).items(): + attrs["sglang." + name.replace(".", "_") + "_count"] = int(value) + for name, value in (detail.get("sums") or {}).items(): + attrs["sglang." + name.replace(".", "_")] = int(value) + for name, value in (detail.get("attrs") or {}).items(): + attrs["sglang." + name.replace(".", "_")] = value + + # Accounting residuals (design §3.2): never negative, never folded in. + def _total(*bases): + total = 0.0 + for base in bases: + for name, value in ms.items(): + if name == base or name.startswith(base + "."): + total += value + return total + + # Roll per-kind variants (kv_page_indices.last, ...) up to one base + # attribute so analysis can use the total without knowing the kinds. + bases = {name.split(".", 1)[0] for name in ms} + for base in bases: + if base not in ms: + attrs["sglang." + base + "_ms"] = _total(base) + + if lt.run_batch is not None: + run_ms = float(lt.run_batch[2].get("sglang.run_batch_ms", 0.0)) + parts = _total( + "early_send_host", + "resolve_inputs", + "model_forward_host", + "copy_result_submit", + ) + attrs["sglang.run_residual_ms"] = max(0.0, run_ms - parts) + if lt.process is not None: + proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0)) + parts = _total( + # Outer phases of process_batch_result (scheduler.py). + "flush_trace", + "load_snapshot", + "flush_backups", + "step_counters", + "metrics_report", + # Inner disagg-prefill phases (prefill.py). + "copy_done_wait", + "auxiliary_and_logprob", + "per_request_result", + "report_stats", + ) + if "sglang.process_kv_ms" in lt.process[2]: + # KV phases charged to this call only (sends issued later in the + # loop, e.g. pending-bootstrap, are not part of process_result). + parts += float(lt.process[2]["sglang.process_kv_ms"]) + else: + parts += _total( + "kv_state_payload", "kv_page_indices", "kv_sender_send" + ) + attrs["sglang.process_residual_ms"] = max(0.0, proc_ms - parts) + + + +def end_loop( + lt: Optional[LoopTimings], + *, + dp_rank: int, + has_batch: bool, + batch=None, + state: Optional[dict] = None, +) -> None: + """Close one loop: counters (always), loop span + phases (if sampled).""" + if lt is None: + return + _LOOP_TLS.current = None + state = state or {} + try: + now_ns = time.monotonic_ns() + now_wall_ns = time.time_ns() + loop_s = max(0.0, (now_ns - lt.t0_ns) / 1e9) + gpu_ns = max(0, _FORWARD_DEVICE_NS_TOTAL - lt.forward_total0) + active1 = bool(state.get("forward_active", False)) + overlap = ( + "full" + if lt.active0 and active1 + else "partial" + if (lt.active0 or active1) + else "none" + ) + local_kind, global_outcome = _LAST_MLP_OUTCOME + pass_snapshot = lt.get_next[2] if lt.get_next is not None else None + pending = bool( + state.get("waiting", 0) + or state.get("result_queue", 0) + or state.get("inflight", 0) + or state.get("bootstrap", 0) + or state.get("deferred", 0) + or state.get("paused") + or state.get("waiting_for_result") + ) + reason = "" + + try: + if local_kind == "none" and global_outcome == "idle": + # The DP sync replaces local None with a real idle batch, so this + # accounting must not depend on has_batch being False. + _ensure_loop_metrics()["lockstep_seconds"].labels( + dp_rank=str(dp_rank), forward_overlap=overlap + ).inc(loop_s) + elif not has_batch: + reason = lt.launch_blocked_reason or _classify_no_batch( + state, pass_snapshot + ) + global _LAST_NO_BATCH_REASON + _LAST_NO_BATCH_REASON = reason + metrics = _ensure_loop_metrics() + metrics["no_batch_seconds"].labels( + dp_rank=str(dp_rank), + reason=reason, + forward_overlap=overlap, + ).inc(loop_s) + metrics["no_batch_total"].labels( + dp_rank=str(dp_rank), + reason=reason, + forward_overlap=overlap, + ).inc() + except Exception: + # Telemetry must never break the scheduler loop. + pass + lt.no_batch_reason = reason + + # Update the prev-loop chain on every loop (sampled or not) so a + # sampled loop after an unsampled one is reported as non-consecutive + # instead of silently carrying a stale window (review finding). + loop_seq_cur = int(lt.seq or lt.predicted_seq) + prev_state = None + if _DETAIL_ENABLED: + prev_state = _update_prev_state(lt, loop_seq_cur) + + if lt.span is None and (has_batch or pending): + # Covers work discovered after begin_loop, including first input and + # deferred-only loops. DP outcomes activate earlier for pass parenting. + ensure_loop_span(lt) + if lt.span is None: + return + + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.prefill.loop") + attrs = { + "sglang.phase": lt.phase, + "sglang.dp_rank": int(dp_rank), + "sglang.loop_seq": int(lt.seq or lt.predicted_seq), + "sglang.loop_ms": loop_s * 1e3, + "sglang.gpu_ms": gpu_ns / 1e6, + "sglang.gpu_idle_ms": max(0.0, loop_s * 1e3 - gpu_ns / 1e6), + "sglang.has_batch": bool(has_batch), + "sglang.engine_has_pending_work": pending, + "sglang.local_batch_kind": local_kind, + "sglang.global_outcome": global_outcome, + "sglang.no_batch_reason": reason, + "sglang.forward_overlap": overlap, + } + attrs.update(_batch_attrs(batch)) + + def _child(name, start_ns, end_ns, child_attrs): + try: + tracer.start_span( + name, + context=lt.span_ctx, + start_time=int(start_ns), + attributes=child_attrs, + ).end(end_time=int(end_ns)) + except Exception: + pass + + if lt.detail is not None: + _emit_loop_detail(lt, attrs, _child, int(dp_rank)) + if _DETAIL_ENABLED: + if lt.pass_end_wall_ns: + # DP-skew join fields (design §4.5); independent of phases. + attrs["sglang.pass_end_wall_ns"] = int(lt.pass_end_wall_ns) + attrs["sglang.arrival_est_wall_ns"] = int(lt.arrival_est_wall_ns) + if prev_state is not None: + prev_seq, prev_has, prev_summary, consecutive = prev_state + attrs["sglang.prev_loop_seq"] = int(prev_seq) + if consecutive: + for key in _DETAIL_PREV_KEYS: + if key in prev_summary: + attrs["sglang.prev_" + key] = prev_summary[key] + elif prev_seq: + # The previous loop was unsampled or had no detail; + # its phases must not be treated as seq-1 values. + attrs["sglang.prev_detail_missing"] = True + lt.span.set_attributes(attrs) + + if lt.recv is not None and not lt.recv_span_child: + start, end, parts = lt.recv + child = {"sglang.dp_rank": int(dp_rank)} + child.update(parts) + _child("scheduler.loop.recv", start, end, child) + if lt.get_next is not None and not lt.pass_span_child: + start, end, ng_attrs = lt.get_next + _child("scheduler.loop.get_next", start, end, ng_attrs) + if lt.run_batch is not None: + start, end, rb_attrs = lt.run_batch + _child("scheduler.loop.run_batch", start, end, rb_attrs) + if lt.process is not None: + start, end, pr_attrs = lt.process + _child("scheduler.loop.process_result", start, end, pr_attrs) + if _KV_SEND_SPAN_ENABLED and lt.kv_send_start is not None: + _child( + "scheduler.loop.kv_send", + lt.kv_send_start, + lt.kv_send_end, + {"sglang.n_chunks": int(lt.kv_chunks)}, + ) + if _FORWARD_SPAN_ENABLED: + for completion_wall, elapsed_ns, category in list(_FORWARD_EVENTS): + if completion_wall < lt.t0_wall_ns or completion_wall > now_wall_ns: + continue + _child( + "scheduler.loop.forward.gpu", + completion_wall - elapsed_ns, + completion_wall, + { + "sglang.gpu_ms": elapsed_ns / 1e6, + "sglang.category": category, + }, + ) + lt.span.end(end_time=now_wall_ns) + except Exception: + # Telemetry must never break the scheduler loop. + pass + finally: + _LOOP_TLS.current = None