A 2 ms phase whose CPU clock reads 0.1 us over its wall clock is clock granularity, not an accounting bug; the anomaly flag fired on nearly every short phase in the live verification run. Only flag (and record non_cpu_raw_ms) when the deficit exceeds both 100 us and 1% of the phase wall. Larger real anomalies keep the previous behaviour.
2743 lines
94 KiB
Python
2743 lines
94 KiB
Python
"""Per-batch scheduler telemetry (opt-in).
|
|
|
|
Adds, at the *batch* granularity (one record per scheduler batch per DP rank):
|
|
|
|
* Prometheus histograms (safe for the multiprocess collector):
|
|
sglang:batch_new_tokens{phase,dp_rank}
|
|
sglang:batch_cached_tokens{phase,dp_rank}
|
|
sglang:batch_duration_seconds{phase,dp_rank}
|
|
sglang:batch_queue_depth{phase,dp_rank}
|
|
sglang:batch_pending_tokens{phase,dp_rank}
|
|
sglang:batch_n_reqs{phase,dp_rank}
|
|
* An OTLP span ``scheduler.batch.{phase}`` with ``Link``s to the request
|
|
traces that took part in the batch (batch -> requests navigation), and a
|
|
``sglang.batch_id`` attribute for reverse lookups / log correlation.
|
|
|
|
Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
|
|
|
|
SGLANG_BATCH_SPAN=0 histograms only (no spans)
|
|
SGLANG_BATCH_SPAN_MIN_MS=50 only emit spans for batches slower than this
|
|
SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0)
|
|
SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span
|
|
|
|
SGLANG_TRACE_LOOP_DETAIL=1 split run_batch/process_result into host
|
|
phases (requires SGLANG_BATCH_STAGE_TIMING=1
|
|
and SGLANG_TRACE_LOOP_SPAN=1; only sampled
|
|
loops are recorded)
|
|
SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown
|
|
(queue/source/indices/transfer/join)
|
|
SGLANG_TRACE_BACKUP_DETAIL=1 per-flush HiCache backup sub-phases
|
|
(one ``scheduler.cache_flush_backup`` span;
|
|
requires SGLANG_TRACE_LOOP_DETAIL=1)
|
|
|
|
Cost model (measured on our 4xDP4 / 4-GPU instances):
|
|
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
|
|
* span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s.
|
|
Everything is behind a single module-level bool, so when disabled the hot
|
|
path pays one ``if not _ENABLED: return``.
|
|
|
|
NOTE: sglang runs prometheus_client in multiprocess mode, which does not
|
|
carry exemplars, so the span trace id is returned to the caller and appended
|
|
to the batch log line instead (see ``metrics_reporter``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import random
|
|
import threading
|
|
import time
|
|
from typing import Any, Iterable, Optional
|
|
|
|
_ENABLED = os.getenv("SGLANG_BATCH_TELEMETRY", "0").strip().lower() not in (
|
|
"",
|
|
"0",
|
|
"false",
|
|
"no",
|
|
"off",
|
|
)
|
|
_SPAN_ENABLED = _ENABLED and os.getenv("SGLANG_BATCH_SPAN", "1").strip().lower() not in (
|
|
"0",
|
|
"false",
|
|
"no",
|
|
"off",
|
|
)
|
|
_SPAN_MIN_S = float(os.getenv("SGLANG_BATCH_SPAN_MIN_MS", "0")) / 1000.0
|
|
_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_SPAN_SAMPLE", "1.0"))
|
|
_MAX_LINKS = int(os.getenv("SGLANG_BATCH_SPAN_MAX_LINKS", "32"))
|
|
|
|
_HISTOGRAMS = None
|
|
_CHILDREN: dict[tuple[str, str, int], Any] = {}
|
|
_TRACER = None
|
|
|
|
|
|
def enabled() -> bool:
|
|
return _ENABLED
|
|
|
|
|
|
_REQUEST_COUNTER = None
|
|
_DP_RANK = None
|
|
_COMPLETION_RANK = None
|
|
|
|
|
|
def _should_report_completion() -> bool:
|
|
"""Whether this scheduler is the canonical reporter of request completions.
|
|
|
|
Every TP/CP rank runs a scheduler and observes the same requests, and a PP
|
|
stage only sees its own microbatches, so an ungated counter would be
|
|
multiplied across replicas once the multiprocess registry aggregates them.
|
|
Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same
|
|
shape the existing scheduler metrics use for rank-unique series.
|
|
"""
|
|
global _COMPLETION_RANK
|
|
if _COMPLETION_RANK is None:
|
|
try:
|
|
from sglang.srt.runtime_context import get_parallel
|
|
|
|
parallel = get_parallel()
|
|
_COMPLETION_RANK = bool(
|
|
parallel.attn_tp_rank == 0
|
|
and parallel.attn_cp_rank == 0
|
|
and parallel.pp_group.is_last_rank
|
|
)
|
|
except Exception:
|
|
# Unknown topology (e.g. a bare unit-test process): keep counting.
|
|
_COMPLETION_RANK = True
|
|
return _COMPLETION_RANK
|
|
|
|
|
|
def _current_dp_rank() -> int:
|
|
"""This process's attention-DP rank (0 for non-DP engines)."""
|
|
global _DP_RANK
|
|
if _DP_RANK is None:
|
|
try:
|
|
from sglang.srt.runtime_context import get_parallel
|
|
|
|
_DP_RANK = int(get_parallel().attn_dp_rank or 0)
|
|
except Exception:
|
|
_DP_RANK = 0
|
|
return _DP_RANK
|
|
|
|
|
|
def _ensure_request_counter():
|
|
global _REQUEST_COUNTER
|
|
if _REQUEST_COUNTER is None:
|
|
from prometheus_client import Counter
|
|
|
|
_REQUEST_COUNTER = Counter(
|
|
"sglang:batch_requests_total",
|
|
"Requests completed by the canonical scheduler rank, by phase. "
|
|
"phase=prefill: prefill compute finished and the request entered the "
|
|
"PD transfer queue (not a guarantee that KV transfer succeeded). "
|
|
"phase=decode: request finished decoding.",
|
|
["phase", "dp_rank"],
|
|
)
|
|
return _REQUEST_COUNTER
|
|
|
|
|
|
def note_request_completed(phase: str, n: int = 1) -> None:
|
|
"""Count finished requests per DP rank.
|
|
|
|
This is the per-rank RPS source: the frontend ``num_requests_total`` has no
|
|
rank label, while the scheduler knows exactly which DP shard completed the
|
|
request. Cheap (one counter increment per finished request) and gated by
|
|
SGLANG_BATCH_TELEMETRY like the other batch metrics.
|
|
"""
|
|
if not _ENABLED or n <= 0 or not _should_report_completion():
|
|
return
|
|
try:
|
|
_ensure_request_counter().labels(
|
|
phase=phase, dp_rank=str(_current_dp_rank())
|
|
).inc(int(n))
|
|
except Exception:
|
|
# Telemetry must never fail a request.
|
|
pass
|
|
|
|
|
|
def _ensure_histograms():
|
|
global _HISTOGRAMS
|
|
if _HISTOGRAMS is None:
|
|
from prometheus_client import Histogram
|
|
|
|
labels = ["phase", "dp_rank"]
|
|
_HISTOGRAMS = {
|
|
"batch_new_tokens": Histogram(
|
|
"sglang:batch_new_tokens",
|
|
"New (uncached) prompt tokens processed by one scheduler batch",
|
|
labels,
|
|
buckets=(128, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536),
|
|
),
|
|
"batch_cached_tokens": Histogram(
|
|
"sglang:batch_cached_tokens",
|
|
"Cached prompt tokens carried by one scheduler batch",
|
|
labels,
|
|
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
|
|
),
|
|
"batch_duration_seconds": Histogram(
|
|
"sglang:batch_duration_seconds",
|
|
"Wall time spent in one scheduler batch",
|
|
labels,
|
|
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0),
|
|
),
|
|
"batch_queue_depth": Histogram(
|
|
"sglang:batch_queue_depth",
|
|
"Waiting-queue depth observed by one scheduler batch",
|
|
labels,
|
|
buckets=(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
|
|
),
|
|
"batch_pending_tokens": Histogram(
|
|
"sglang:batch_pending_tokens",
|
|
"Prefill backlog (pending tokens) observed by one scheduler batch",
|
|
labels,
|
|
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
|
|
),
|
|
"batch_n_reqs": Histogram(
|
|
"sglang:batch_n_reqs",
|
|
"Requests carried by one scheduler batch",
|
|
labels,
|
|
buckets=(1, 2, 4, 8, 16, 32, 64, 128, 256),
|
|
),
|
|
}
|
|
return _HISTOGRAMS
|
|
|
|
|
|
def _child(name: str, phase: str, dp_rank: int):
|
|
key = (name, phase, dp_rank)
|
|
child = _CHILDREN.get(key)
|
|
if child is None:
|
|
child = _ensure_histograms()[name].labels(phase=phase, dp_rank=str(dp_rank))
|
|
_CHILDREN[key] = child
|
|
return child
|
|
|
|
|
|
def _request_span_context(req: Any):
|
|
"""SpanContext of a request, from either a live span or a serialized copy.
|
|
|
|
Scheduler processes receive a *copy* of the request trace context where
|
|
``root_span`` is None (see ``trace_async.copy_for_thread``) and only
|
|
``root_span_context`` (an OTel ``Context``) survives. Handle both.
|
|
"""
|
|
trace_ctx = getattr(getattr(req, "time_stats", None), "trace_ctx", None)
|
|
if trace_ctx is None:
|
|
return None
|
|
try:
|
|
root_span = getattr(trace_ctx, "root_span", None)
|
|
if root_span is not None:
|
|
span_ctx = root_span.get_span_context()
|
|
if span_ctx.is_valid:
|
|
return span_ctx
|
|
root_ctx = getattr(trace_ctx, "root_span_context", None)
|
|
if root_ctx is not None:
|
|
from opentelemetry import trace as otel_trace
|
|
|
|
span_ctx = otel_trace.get_current_span(root_ctx).get_span_context()
|
|
if span_ctx.is_valid:
|
|
return span_ctx
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _emit_span(
|
|
phase: str,
|
|
dp_rank: int,
|
|
iter_no: int,
|
|
n_reqs: int,
|
|
n_new: int,
|
|
n_cached: int,
|
|
queue_depth: int,
|
|
pending_tokens: int,
|
|
duration_s: float,
|
|
reqs: Optional[Iterable[Any]],
|
|
) -> Optional[str]:
|
|
"""Create one batch span with links to the request traces. Returns trace id."""
|
|
if not _SPAN_ENABLED:
|
|
return None
|
|
if _SPAN_MIN_S > 0 and duration_s < _SPAN_MIN_S:
|
|
return None
|
|
if _SPAN_SAMPLE < 1.0 and random.random() > _SPAN_SAMPLE:
|
|
return None
|
|
global _TRACER
|
|
try:
|
|
from opentelemetry import trace as otel_trace
|
|
from opentelemetry.trace import Link
|
|
|
|
if _TRACER is None:
|
|
_TRACER = otel_trace.get_tracer("sglang.batch")
|
|
links = []
|
|
if reqs:
|
|
for req in reqs:
|
|
if len(links) >= _MAX_LINKS:
|
|
break
|
|
ctx = _request_span_context(req)
|
|
if ctx is not None:
|
|
links.append(Link(ctx))
|
|
end_ns = time.time_ns()
|
|
start_ns = end_ns - int(duration_s * 1e9)
|
|
span = _TRACER.start_span(
|
|
f"scheduler.batch.{phase}",
|
|
start_time=start_ns,
|
|
attributes={
|
|
"sglang.batch_id": f"{dp_rank}:{iter_no}",
|
|
"sglang.dp_rank": int(dp_rank),
|
|
"sglang.phase": phase,
|
|
"sglang.n_reqs": int(n_reqs),
|
|
"sglang.n_new_tokens": int(n_new),
|
|
"sglang.n_cached_tokens": int(n_cached),
|
|
"sglang.queue_depth": int(queue_depth),
|
|
"sglang.pending_tokens": int(pending_tokens),
|
|
"sglang.batch_duration_s": float(duration_s),
|
|
},
|
|
links=links,
|
|
)
|
|
span.end(end_time=end_ns)
|
|
span_ctx = span.get_span_context()
|
|
if not span_ctx.is_valid:
|
|
# No TracerProvider configured (tracing disabled) -> NoOp span.
|
|
return None
|
|
return format(span_ctx.trace_id, "032x")
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _observe(
|
|
phase: str,
|
|
dp_rank: int,
|
|
iter_no: int,
|
|
n_reqs: int,
|
|
n_new: int,
|
|
n_cached: int,
|
|
queue_depth: int,
|
|
pending_tokens: int,
|
|
duration_s: float,
|
|
reqs: Optional[Iterable[Any]],
|
|
) -> Optional[str]:
|
|
if not _ENABLED:
|
|
return None
|
|
try:
|
|
trace_id = _emit_span(
|
|
phase,
|
|
dp_rank,
|
|
iter_no,
|
|
n_reqs,
|
|
n_new,
|
|
n_cached,
|
|
queue_depth,
|
|
pending_tokens,
|
|
duration_s,
|
|
reqs,
|
|
)
|
|
_child("batch_new_tokens", phase, dp_rank).observe(n_new)
|
|
_child("batch_cached_tokens", phase, dp_rank).observe(n_cached)
|
|
_child("batch_duration_seconds", phase, dp_rank).observe(duration_s)
|
|
_child("batch_queue_depth", phase, dp_rank).observe(queue_depth)
|
|
_child("batch_pending_tokens", phase, dp_rank).observe(pending_tokens)
|
|
_child("batch_n_reqs", phase, dp_rank).observe(n_reqs)
|
|
return trace_id
|
|
except Exception:
|
|
# Telemetry must never break the scheduler loop.
|
|
return None
|
|
|
|
|
|
def observe_prefill(
|
|
*,
|
|
dp_rank: int,
|
|
iter_no: int,
|
|
n_reqs: int,
|
|
n_new: int,
|
|
n_cached: int,
|
|
queue_depth: int,
|
|
pending_tokens: int,
|
|
duration_s: float,
|
|
reqs: Optional[Iterable[Any]] = None,
|
|
) -> Optional[str]:
|
|
return _observe(
|
|
"prefill",
|
|
dp_rank,
|
|
iter_no,
|
|
n_reqs,
|
|
n_new,
|
|
n_cached,
|
|
queue_depth,
|
|
pending_tokens,
|
|
duration_s,
|
|
reqs,
|
|
)
|
|
|
|
|
|
def observe_decode(
|
|
*,
|
|
dp_rank: int,
|
|
iter_no: int,
|
|
n_reqs: int,
|
|
n_new: int,
|
|
n_cached: int,
|
|
queue_depth: int,
|
|
pending_tokens: int,
|
|
duration_s: float,
|
|
reqs: Optional[Iterable[Any]] = None,
|
|
) -> Optional[str]:
|
|
return _observe(
|
|
"decode",
|
|
dp_rank,
|
|
iter_no,
|
|
n_reqs,
|
|
n_new,
|
|
n_cached,
|
|
queue_depth,
|
|
pending_tokens,
|
|
duration_s,
|
|
reqs,
|
|
)
|
|
|
|
_DP_HISTS = None
|
|
|
|
|
|
def _ensure_dp_histograms():
|
|
global _DP_HISTS
|
|
if _DP_HISTS is None:
|
|
from prometheus_client import Histogram
|
|
|
|
_DP_HISTS = {
|
|
"dp_step_tokens": Histogram(
|
|
"sglang:dp_step_tokens",
|
|
"Per-rank token count in one DP-attention step",
|
|
["dp_rank", "phase"],
|
|
buckets=(0, 256, 1024, 2048, 4096, 8192, 16384, 32768),
|
|
),
|
|
"dp_step_padding_ratio": Histogram(
|
|
"sglang:dp_step_padding_ratio",
|
|
"1 - mean/max tokens across active DP ranks in one step (padding waste)",
|
|
["phase", "n_active"],
|
|
buckets=(0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8),
|
|
),
|
|
"dp_step_spread": Histogram(
|
|
"sglang:dp_step_spread",
|
|
"(max-min)/mean tokens across active DP ranks in one step",
|
|
["phase", "n_active"],
|
|
buckets=(0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0),
|
|
),
|
|
}
|
|
return _DP_HISTS
|
|
|
|
|
|
_DP_IDLE_MODE = 4 # ForwardMode.IDLE
|
|
|
|
|
|
def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = "unknown") -> None:
|
|
"""Record per-step shape alignment across *active* DP ranks.
|
|
|
|
``tokens``/``modes``/``health`` are the per-rank vectors from the DP
|
|
all-gather (info tensor columns 0, 5, 9). IDLE ranks and health-check
|
|
steps are excluded so the padding ratio reflects real batch-shape
|
|
misalignment rather than idle-slot padding.
|
|
"""
|
|
if not _ENABLED:
|
|
return
|
|
try:
|
|
if health and any(bool(h) for h in health):
|
|
return
|
|
sel = []
|
|
for i, t in enumerate(tokens):
|
|
if t is None:
|
|
continue
|
|
t = int(t)
|
|
if t <= 1:
|
|
continue
|
|
if modes is not None and i < len(modes) and int(modes[i]) == _DP_IDLE_MODE:
|
|
continue
|
|
sel.append(t)
|
|
if len(sel) < 2:
|
|
return
|
|
hist = _ensure_dp_histograms()
|
|
mx = max(sel)
|
|
mean = sum(sel) / len(sel)
|
|
n_active = str(len(sel))
|
|
key = ("dp_step_tokens", phase, dp_rank)
|
|
child = _CHILDREN.get(key)
|
|
if child is None:
|
|
child = hist["dp_step_tokens"].labels(dp_rank=str(dp_rank), phase=phase)
|
|
_CHILDREN[key] = child
|
|
child.observe(int(tokens[dp_rank]) if 0 <= dp_rank < len(tokens) else int(mean))
|
|
if mx > 0:
|
|
hist["dp_step_padding_ratio"].labels(phase=phase, n_active=n_active).observe(
|
|
1.0 - mean / mx
|
|
)
|
|
if mean > 0:
|
|
hist["dp_step_spread"].labels(phase=phase, n_active=n_active).observe(
|
|
(mx - min(sel)) / mean
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-pass stage timing (SGLANG_BATCH_STAGE_TIMING=1) -- P (disagg prefill).
|
|
#
|
|
# One sample per get_next_disagg_prefill_batch_to_run() pass, including passes
|
|
# that produce no batch. Disabled by default; when off every hook below is a
|
|
# single module-level bool check. No CUDA calls, no per-pass logging.
|
|
#
|
|
# Cross-rank analysis uses a dedicated, independently sampled span
|
|
# ``scheduler.prefill.pass`` emitted from end_pass(): the sampling decision is a
|
|
# pure function of ``mlp_sync_seq`` (identical on every DP rank that took part
|
|
# in the same gather), so a sampled gather round is sampled on *all* ranks --
|
|
# unlike the per-rank random sampling of the batch span. Idle ranks and ranks
|
|
# with no local batch therefore still emit, which is exactly the case the
|
|
# skew analysis needs.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_STAGE_TIMING = os.getenv("SGLANG_BATCH_STAGE_TIMING", "0").strip().lower() not in (
|
|
"",
|
|
"0",
|
|
"false",
|
|
"no",
|
|
"off",
|
|
)
|
|
|
|
_STAGE_LABELS = ["phase", "dp_rank"]
|
|
_STAGE_BUCKETS = (
|
|
1e-5,
|
|
3e-5,
|
|
1e-4,
|
|
3e-4,
|
|
1e-3,
|
|
3e-3,
|
|
1e-2,
|
|
3e-2,
|
|
1e-1,
|
|
3e-1,
|
|
1.0,
|
|
3.0,
|
|
)
|
|
|
|
_STAGE_HISTS = None
|
|
_STAGE_COUNTERS = None
|
|
_STAGE_CHILDREN: dict = {}
|
|
_STAGE_TLS = threading.local()
|
|
|
|
# Set by the DP all-gather path (dp_attn.prepare_mlp_sync_batch_raw).
|
|
_LAST_DP_GATHER = None
|
|
_DP_GATHER_SEQ = 0
|
|
|
|
_PASS_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
|
|
"SGLANG_BATCH_PASS_SPAN", "1"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
_PASS_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_PASS_SPAN_SAMPLE", "0.05"))
|
|
|
|
_STAGE_HIST_DOCS = {
|
|
"get_next": "Wall time of one get_next_disagg_prefill_batch_to_run pass",
|
|
"match": "Per-pass sum of init_next_round_input (prefix match) over candidates",
|
|
"adder": "Per-pass sum of PrefillAdder.add_one_req (budget/admission)",
|
|
"hicache_events": "Per-pass time in _process_hicache_events (includes its collectives)",
|
|
"bootstrap": "Per-pass time in resolve_waiting_queue_bootstrap (poll + all-reduce)",
|
|
"mlp_sync": "Per-pass outer maybe_prepare_mlp_sync_batch time (not pure gather wait)",
|
|
"other": "get_next total minus the instrumented segments",
|
|
}
|
|
|
|
|
|
def stage_timing_enabled() -> bool:
|
|
return _STAGE_TIMING
|
|
|
|
|
|
def pass_span_enabled() -> bool:
|
|
return _PASS_SPAN_ENABLED
|
|
|
|
|
|
def pass_span_decision(seq: int) -> bool:
|
|
"""Deterministic, rank-identical sampling decision for one gather round.
|
|
|
|
``seq`` is the DP all-gather sequence number, which is the same on every
|
|
rank that participated in that round. Therefore one decision covers the
|
|
whole round, instead of four independent ``random.random()`` draws.
|
|
"""
|
|
if not _PASS_SPAN_ENABLED or seq <= 0:
|
|
return False
|
|
if _PASS_SPAN_SAMPLE >= 1.0:
|
|
return True
|
|
if _PASS_SPAN_SAMPLE <= 0.0:
|
|
return False
|
|
return (seq * 2654435761) % 10000 < int(_PASS_SPAN_SAMPLE * 10000)
|
|
|
|
|
|
def _ensure_stage_metrics():
|
|
global _STAGE_HISTS, _STAGE_COUNTERS
|
|
if _STAGE_HISTS is None:
|
|
from prometheus_client import Counter, Histogram
|
|
|
|
_STAGE_HISTS = {
|
|
name: Histogram(
|
|
f"sglang:batch_prefill_{name}_seconds",
|
|
doc,
|
|
_STAGE_LABELS,
|
|
buckets=_STAGE_BUCKETS,
|
|
)
|
|
for name, doc in _STAGE_HIST_DOCS.items()
|
|
}
|
|
_STAGE_COUNTERS = {
|
|
"candidates": Counter(
|
|
"sglang:batch_prefill_candidates_total",
|
|
"Requests examined by the per-pass admission candidate loop",
|
|
_STAGE_LABELS,
|
|
),
|
|
"skipped_prefetch": Counter(
|
|
"sglang:batch_prefill_skipped_prefetch_total",
|
|
"Candidates skipped because their L2/L3 prefetch was not ready",
|
|
_STAGE_LABELS,
|
|
),
|
|
"admitted": Counter(
|
|
"sglang:batch_prefill_admitted_total",
|
|
"Requests admitted into can_run_list by one pass",
|
|
_STAGE_LABELS,
|
|
),
|
|
"resumed_chunk": Counter(
|
|
"sglang:batch_prefill_resumed_chunk_total",
|
|
"Passes whose batch continues a chunked-prefill request "
|
|
"(including its last chunk)",
|
|
_STAGE_LABELS,
|
|
),
|
|
"new_chunk": Counter(
|
|
"sglang:batch_prefill_new_chunk_total",
|
|
"Passes that start a new chunked-prefill request",
|
|
_STAGE_LABELS,
|
|
),
|
|
}
|
|
return _STAGE_HISTS, _STAGE_COUNTERS
|
|
|
|
|
|
class PassTimings:
|
|
__slots__ = (
|
|
"phase",
|
|
"t0",
|
|
"match",
|
|
"adder",
|
|
"hicache",
|
|
"bootstrap",
|
|
"mlp_sync",
|
|
"pending_abort",
|
|
"chunk",
|
|
"new_batch",
|
|
"set_sched",
|
|
"chunk_count",
|
|
"dp_gather",
|
|
"candidates",
|
|
"skipped_prefetch",
|
|
"admitted",
|
|
"bootstrap_candidates",
|
|
"pending_bootstrap",
|
|
"had_chunked",
|
|
"new_chunk",
|
|
"seq",
|
|
)
|
|
|
|
def __init__(self, phase: str):
|
|
self.phase = phase
|
|
self.t0 = time.monotonic()
|
|
self.match = 0.0
|
|
self.adder = 0.0
|
|
self.hicache = 0.0
|
|
self.bootstrap = 0.0
|
|
self.mlp_sync = 0.0
|
|
self.pending_abort = 0.0
|
|
self.chunk = 0.0
|
|
self.new_batch = 0.0
|
|
self.set_sched = 0.0
|
|
self.chunk_count = 0
|
|
self.dp_gather = None
|
|
self.candidates = 0
|
|
self.skipped_prefetch = 0
|
|
self.admitted = 0
|
|
self.bootstrap_candidates = 0
|
|
self.pending_bootstrap = 0
|
|
self.had_chunked = False
|
|
self.new_chunk = False
|
|
self.seq = 0
|
|
|
|
|
|
def begin_pass(phase: str = "prefill") -> Optional[PassTimings]:
|
|
"""Start a new pass. Returns None when stage timing is disabled."""
|
|
if not _STAGE_TIMING:
|
|
return None
|
|
timings = PassTimings(phase)
|
|
_STAGE_TLS.current = timings
|
|
return timings
|
|
|
|
|
|
def _current() -> Optional[PassTimings]:
|
|
if not _STAGE_TIMING:
|
|
return None
|
|
return getattr(_STAGE_TLS, "current", None)
|
|
|
|
|
|
def add_match(seconds: float) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.match += seconds
|
|
|
|
|
|
def add_adder(seconds: float) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.adder += seconds
|
|
|
|
|
|
def add_hicache(seconds: float) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.hicache += seconds
|
|
|
|
|
|
def add_bootstrap(seconds: float) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.bootstrap += seconds
|
|
|
|
|
|
def add_pending_abort(seconds: float) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.pending_abort += seconds
|
|
|
|
|
|
def add_chunk(seconds: float, count: int = 0) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.chunk += seconds
|
|
t.chunk_count += int(count)
|
|
|
|
|
|
def add_new_batch(seconds: float) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.new_batch += seconds
|
|
|
|
|
|
def add_set_sched(seconds: float) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.set_sched += seconds
|
|
|
|
|
|
def mark_candidate(n: int = 1) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.candidates += n
|
|
|
|
|
|
def mark_skipped_prefetch(n: int = 1) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.skipped_prefetch += n
|
|
|
|
|
|
def set_admitted(n: int) -> None:
|
|
t = _current()
|
|
if t is not None:
|
|
t.admitted = int(n)
|
|
|
|
|
|
def set_bootstrap_pending(pending: int, candidates: int) -> None:
|
|
"""Reuse the bootstrap resolver's existing pass; do not rescan the queue."""
|
|
t = _current()
|
|
if t is not None:
|
|
t.pending_bootstrap = int(pending)
|
|
t.bootstrap_candidates = int(candidates)
|
|
|
|
|
|
def note_had_chunked(flag: bool) -> None:
|
|
"""Record whether a chunked request was already in flight when the pass began.
|
|
|
|
Must be called at pass start, before process_prefill_chunk()/admission can
|
|
consume the chunked request (the last chunk clears self.chunked_req).
|
|
"""
|
|
t = _current()
|
|
if t is not None:
|
|
t.had_chunked = bool(flag)
|
|
|
|
|
|
def note_new_chunk(flag: bool) -> None:
|
|
"""Record whether this pass started a new chunked request (first chunk)."""
|
|
t = _current()
|
|
if t is not None:
|
|
t.new_chunk = bool(flag)
|
|
|
|
|
|
def note_dp_gather(seconds: Optional[float]) -> None:
|
|
"""Called by the DP all-gather path; None means no gather ran this pass."""
|
|
global _LAST_DP_GATHER, _DP_GATHER_SEQ
|
|
if not _STAGE_TIMING:
|
|
return
|
|
if seconds is None:
|
|
_LAST_DP_GATHER = None
|
|
return
|
|
_LAST_DP_GATHER = float(seconds)
|
|
_DP_GATHER_SEQ += 1
|
|
|
|
|
|
def _take_dp_gather():
|
|
global _LAST_DP_GATHER
|
|
value = _LAST_DP_GATHER
|
|
_LAST_DP_GATHER = None
|
|
return value
|
|
|
|
|
|
def add_mlp_sync(seconds: float) -> None:
|
|
t = _current()
|
|
if t is None:
|
|
return
|
|
t.mlp_sync += seconds
|
|
gather = _take_dp_gather()
|
|
if gather is not None:
|
|
t.dp_gather = gather
|
|
t.seq = _DP_GATHER_SEQ
|
|
|
|
|
|
def _emit_pass_span(
|
|
dp_rank: int,
|
|
t: PassTimings,
|
|
total: float,
|
|
other: float,
|
|
residual: float = 0.0,
|
|
) -> None:
|
|
if not pass_span_decision(t.seq):
|
|
return
|
|
loop = getattr(_LOOP_TLS, "current", None)
|
|
loop_ctx = loop.span_ctx if loop is not None else None
|
|
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
|
|
# An unsampled loop owns this pass; do not leak an independent pass trace.
|
|
# If work is discovered later, end_loop emits the snapshot as its child.
|
|
return
|
|
try:
|
|
from opentelemetry import trace as otel_trace
|
|
|
|
tracer = otel_trace.get_tracer("sglang.prefill.pass")
|
|
end_ns = time.time_ns()
|
|
start_ns = end_ns - int(total * 1e9)
|
|
tracer.start_span(
|
|
"scheduler.prefill.pass",
|
|
context=loop_ctx,
|
|
start_time=start_ns,
|
|
attributes={
|
|
"sglang.phase": t.phase,
|
|
"sglang.dp_rank": int(dp_rank),
|
|
"sglang.mlp_sync_seq": int(t.seq),
|
|
"sglang.get_next_ms": total * 1e3,
|
|
"sglang.match_ms": t.match * 1e3,
|
|
"sglang.adder_ms": t.adder * 1e3,
|
|
"sglang.hicache_ms": t.hicache * 1e3,
|
|
"sglang.bootstrap_ms": t.bootstrap * 1e3,
|
|
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
|
|
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
|
|
"sglang.other_ms": other * 1e3,
|
|
"sglang.pending_abort_ms": t.pending_abort * 1e3,
|
|
"sglang.chunk_ms": t.chunk * 1e3,
|
|
"sglang.new_batch_ms": t.new_batch * 1e3,
|
|
"sglang.set_sched_ms": t.set_sched * 1e3,
|
|
"sglang.get_next_residual_ms": residual * 1e3,
|
|
"sglang.chunk_count": int(t.chunk_count),
|
|
"sglang.candidates": int(t.candidates),
|
|
"sglang.skipped_prefetch": int(t.skipped_prefetch),
|
|
"sglang.admitted": int(t.admitted),
|
|
"sglang.resumed_chunked": int(bool(t.had_chunked)),
|
|
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
|
|
},
|
|
).end(end_time=end_ns)
|
|
if loop is not None and loop_ctx is not None:
|
|
loop.pass_span_child = True
|
|
except Exception:
|
|
# Tracing must never break the scheduler loop.
|
|
pass
|
|
|
|
|
|
def end_pass(dp_rank: int) -> None:
|
|
"""Close the current pass: observe histograms/counters and emit the pass span."""
|
|
t = _current()
|
|
if t is None:
|
|
return
|
|
_STAGE_TLS.current = None
|
|
try:
|
|
total = time.monotonic() - t.t0
|
|
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
|
|
other = total - measured
|
|
if other < 0.0:
|
|
other = 0.0
|
|
# Residual after also charging the get_next sub-phases.
|
|
measured2 = measured + (
|
|
t.pending_abort + t.chunk + t.new_batch + t.set_sched
|
|
)
|
|
residual = total - measured2
|
|
if residual < 0.0:
|
|
residual = 0.0
|
|
hist, counters = _ensure_stage_metrics()
|
|
child_key = (t.phase, dp_rank)
|
|
keyed = _STAGE_CHILDREN.get(child_key)
|
|
if keyed is None:
|
|
keyed = {
|
|
name: h.labels(phase=t.phase, dp_rank=str(dp_rank))
|
|
for name, h in hist.items()
|
|
}
|
|
keyed.update(
|
|
{
|
|
name: c.labels(phase=t.phase, dp_rank=str(dp_rank))
|
|
for name, c in counters.items()
|
|
}
|
|
)
|
|
_STAGE_CHILDREN[child_key] = keyed
|
|
keyed["get_next"].observe(total)
|
|
keyed["match"].observe(t.match)
|
|
keyed["adder"].observe(t.adder)
|
|
keyed["hicache_events"].observe(t.hicache)
|
|
keyed["bootstrap"].observe(t.bootstrap)
|
|
keyed["mlp_sync"].observe(t.mlp_sync)
|
|
keyed["other"].observe(other)
|
|
if t.candidates:
|
|
keyed["candidates"].inc(t.candidates)
|
|
if t.skipped_prefetch:
|
|
keyed["skipped_prefetch"].inc(t.skipped_prefetch)
|
|
if t.admitted:
|
|
keyed["admitted"].inc(t.admitted)
|
|
if t.had_chunked:
|
|
keyed["resumed_chunk"].inc(1)
|
|
elif t.new_chunk:
|
|
keyed["new_chunk"].inc(1)
|
|
note_pass_snapshot(t, total)
|
|
_emit_pass_span(dp_rank, t, total, other, residual)
|
|
except Exception:
|
|
# Telemetry must never break the scheduler loop
|
|
_STAGE_TLS.current = None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scheduler receive path (same SGLANG_BATCH_STAGE_TIMING gate): split the
|
|
# recv_requests stage into pull / broadcast / unwrap / mm.
|
|
#
|
|
# NOTE: tokenization itself runs in the tokenizer worker processes (HTTP side).
|
|
# What is measured here is the scheduler-side IPC: ZMQ pull + decode of the
|
|
# tokenized request objects, the cross-rank broadcast of those objects, and
|
|
# the unpickling of their token-id buffers. Large prompts make all three
|
|
# visible (e.g. 14.6k tokens per request).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_RECV_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
|
|
"SGLANG_RECV_SPAN", "1"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
_RECV_SPAN_SAMPLE = float(os.getenv("SGLANG_RECV_SPAN_SAMPLE", "0.05"))
|
|
|
|
_RECV_PARTS = ("total", "pull", "broadcast", "unwrap", "mm")
|
|
_RECV_HISTS = None
|
|
_RECV_COUNTERS = None
|
|
_RECV_SEQ = 0
|
|
|
|
|
|
def recv_timing_enabled() -> bool:
|
|
return _STAGE_TIMING
|
|
|
|
|
|
def _ensure_recv_metrics():
|
|
global _RECV_HISTS, _RECV_COUNTERS
|
|
if _RECV_HISTS is None:
|
|
from prometheus_client import Counter, Histogram
|
|
|
|
_RECV_HISTS = Histogram(
|
|
"sglang:recv_requests_seconds",
|
|
"Scheduler recv_requests wall time by part "
|
|
"(total/pull/broadcast/unwrap/mm)",
|
|
["dp_rank", "part"],
|
|
buckets=_STAGE_BUCKETS,
|
|
)
|
|
_RECV_COUNTERS = {
|
|
"msgs": Counter(
|
|
"sglang:recv_requests_msgs_total",
|
|
"Messages received by the scheduler recv path",
|
|
["dp_rank"],
|
|
),
|
|
"prompt_tokens": Counter(
|
|
"sglang:recv_requests_prompt_tokens_total",
|
|
"Prompt tokens carried by messages received in the recv path",
|
|
["dp_rank"],
|
|
),
|
|
}
|
|
return _RECV_HISTS, _RECV_COUNTERS
|
|
|
|
|
|
def observe_recv(
|
|
*,
|
|
dp_rank: int,
|
|
total: float,
|
|
pull: float,
|
|
broadcast: float,
|
|
unwrap: float,
|
|
mm: float,
|
|
n_msgs: int,
|
|
n_tokens: int,
|
|
) -> None:
|
|
if not _STAGE_TIMING:
|
|
return
|
|
global _RECV_SEQ
|
|
_RECV_SEQ += 1
|
|
try:
|
|
hist, counters = _ensure_recv_metrics()
|
|
for part, value in zip(_RECV_PARTS, (total, pull, broadcast, unwrap, mm)):
|
|
hist.labels(dp_rank=str(dp_rank), part=part).observe(value)
|
|
if n_msgs:
|
|
counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs)
|
|
if n_tokens:
|
|
counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens)
|
|
end_ns = time.time_ns()
|
|
start_ns = end_ns - int(total * 1e9)
|
|
note_recv_phase(
|
|
start_wall_ns=start_ns,
|
|
end_wall_ns=end_ns,
|
|
parts={
|
|
"sglang.recv_total_ms": total * 1e3,
|
|
"sglang.recv_pull_ms": pull * 1e3,
|
|
"sglang.recv_broadcast_ms": broadcast * 1e3,
|
|
"sglang.recv_unwrap_ms": unwrap * 1e3,
|
|
"sglang.recv_mm_ms": mm * 1e3,
|
|
"sglang.recv_msgs": int(n_msgs),
|
|
"sglang.recv_prompt_tokens": int(n_tokens),
|
|
},
|
|
)
|
|
|
|
if not _RECV_SPAN_ENABLED:
|
|
return
|
|
if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int(
|
|
_RECV_SPAN_SAMPLE * 10000
|
|
):
|
|
return
|
|
loop = current_loop()
|
|
loop_ctx = current_loop_context()
|
|
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
|
|
# Keep the snapshot only; end_loop emits it if this loop has work.
|
|
return
|
|
from opentelemetry import trace as otel_trace
|
|
|
|
tracer = otel_trace.get_tracer("sglang.recv")
|
|
tracer.start_span(
|
|
"scheduler.recv_requests",
|
|
context=loop_ctx,
|
|
start_time=start_ns,
|
|
attributes={
|
|
"sglang.phase": "recv",
|
|
"sglang.dp_rank": int(dp_rank),
|
|
"sglang.recv_msgs": int(n_msgs),
|
|
"sglang.recv_prompt_tokens": int(n_tokens),
|
|
"sglang.recv_total_ms": total * 1e3,
|
|
"sglang.recv_pull_ms": pull * 1e3,
|
|
"sglang.recv_broadcast_ms": broadcast * 1e3,
|
|
"sglang.recv_unwrap_ms": unwrap * 1e3,
|
|
"sglang.recv_mm_ms": mm * 1e3,
|
|
},
|
|
).end(end_time=end_ns)
|
|
if loop is not None and loop_ctx is not None:
|
|
loop.recv_span_child = True
|
|
except Exception:
|
|
# Telemetry must never break the scheduler loop.
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Storage prefetch ready latency (same SGLANG_BATCH_STAGE_TIMING gate).
|
|
#
|
|
# Request-level wait from "prefetch in flight" to "usable" (the scheduler
|
|
# check returns True), split by L3 hit/miss. Under wait_complete this is the
|
|
# admission delay a request pays for its L3 lookup.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_PREFETCH_READY_HIST = None
|
|
|
|
|
|
def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None:
|
|
if not _STAGE_TIMING:
|
|
return
|
|
global _PREFETCH_READY_HIST
|
|
try:
|
|
if _PREFETCH_READY_HIST is None:
|
|
from prometheus_client import Histogram
|
|
|
|
_PREFETCH_READY_HIST = Histogram(
|
|
"sglang:storage_prefetch_ready_seconds",
|
|
"Prefetch in-flight -> usable latency, split by L3 hit/miss",
|
|
["dp_rank", "result"],
|
|
buckets=(1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0, 3.0, 10.0, 30.0),
|
|
)
|
|
_PREFETCH_READY_HIST.labels(dp_rank=str(dp_rank), result=result).observe(
|
|
seconds
|
|
)
|
|
except Exception:
|
|
# Telemetry must never break the scheduler loop.
|
|
pass
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scheduler loop span + no-batch accounting (opt-in, same stage-timing gate).
|
|
#
|
|
# One ``scheduler.loop`` span per scheduler iteration (sampled), with phase
|
|
# children (recv / get_next / run_batch / process_result / kv_send). The
|
|
# no-batch counters are incremented on *every* loop (sampling only controls
|
|
# spans), split by a low-cardinality reason and by whether a forward was in
|
|
# flight (forward_overlap). Fully idle loops never emit a span.
|
|
#
|
|
# Enable with ``SGLANG_TRACE_LOOP_SPAN=1`` (requires SGLANG_BATCH_STAGE_TIMING).
|
|
# Knobs:
|
|
# SGLANG_TRACE_LOOP_SAMPLE=0.05 head-sample ratio (deterministic per
|
|
# mlp_sync_seq, identical on all ranks)
|
|
# SGLANG_TRACE_FORWARD_SPAN=0 emit approximate forward.gpu lanes
|
|
# SGLANG_TRACE_KV_SEND_SPAN=1 emit the kv_send child (default on
|
|
# when the loop span is on)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
from collections import deque as _deque
|
|
|
|
_LOOP_ENABLED = _STAGE_TIMING
|
|
_LOOP_SPAN_ENABLED = _LOOP_ENABLED and os.getenv(
|
|
"SGLANG_TRACE_LOOP_SPAN", "0"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
_LOOP_SPAN_SAMPLE = float(os.getenv("SGLANG_TRACE_LOOP_SAMPLE", "0.05"))
|
|
_FORWARD_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
|
|
"SGLANG_TRACE_FORWARD_SPAN", "0"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
_KV_SEND_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
|
|
"SGLANG_TRACE_KV_SEND_SPAN", "1"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
|
|
_NO_BATCH_REASONS = (
|
|
"fully_idle",
|
|
"waiting_prefetch",
|
|
"waiting_bootstrap",
|
|
"result_backpressure",
|
|
"deferred_input",
|
|
"admission_blocked",
|
|
"waiting_pd",
|
|
"no_ready_other",
|
|
)
|
|
|
|
_LOOP_METRICS = None
|
|
_LOOP_TLS = threading.local()
|
|
|
|
# Monotonic total of completed forward device time (from the DeviceTimer),
|
|
# used to derive one loop's gpu_ms without adding a second timer.
|
|
_FORWARD_DEVICE_NS_TOTAL = 0
|
|
_FORWARD_EVENTS = _deque(maxlen=128) # (completion_wall_ns, elapsed_ns, category)
|
|
|
|
# Set by dp_attn.prepare_mlp_sync_batch_raw() once per pass.
|
|
_LAST_MLP_OUTCOME = ("none", "none")
|
|
_LAST_NO_BATCH_REASON = "no_ready_other"
|
|
|
|
|
|
def loop_accounting_enabled() -> bool:
|
|
return _LOOP_ENABLED
|
|
|
|
|
|
def loop_span_enabled() -> bool:
|
|
return _LOOP_SPAN_ENABLED
|
|
|
|
|
|
def forward_span_enabled() -> bool:
|
|
return _FORWARD_SPAN_ENABLED
|
|
|
|
|
|
def kv_send_span_enabled() -> bool:
|
|
if not _KV_SEND_SPAN_ENABLED:
|
|
return False
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
return lt is not None and lt.span is not None
|
|
|
|
|
|
def loop_sample_decision(seq: int) -> bool:
|
|
"""Deterministic, rank-identical loop-span sampling decision."""
|
|
if not _LOOP_SPAN_ENABLED or seq <= 0:
|
|
return False
|
|
if _LOOP_SPAN_SAMPLE >= 1.0:
|
|
return True
|
|
if _LOOP_SPAN_SAMPLE <= 0.0:
|
|
return False
|
|
return (seq * 2654435761) % 10000 < int(_LOOP_SPAN_SAMPLE * 10000)
|
|
|
|
|
|
class LoopTimings:
|
|
__slots__ = (
|
|
"phase",
|
|
"t0_ns",
|
|
"t0_wall_ns",
|
|
"predicted_seq",
|
|
"active0",
|
|
"has_work0",
|
|
"forward_total0",
|
|
"recv",
|
|
"get_next",
|
|
"run_batch",
|
|
"process",
|
|
"kv_send_start",
|
|
"kv_send_end",
|
|
"kv_chunks",
|
|
"span",
|
|
"span_ctx",
|
|
"pass_span_child",
|
|
"recv_span_child",
|
|
"launch_blocked_reason",
|
|
"seq",
|
|
"no_batch_reason",
|
|
"pass_end_wall_ns",
|
|
"arrival_est_wall_ns",
|
|
"detail",
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
phase: str,
|
|
t0_ns: int,
|
|
t0_wall_ns: int,
|
|
predicted_seq: int,
|
|
active0: bool,
|
|
has_work0: bool,
|
|
forward_total0: int,
|
|
):
|
|
self.phase = phase
|
|
self.t0_ns = t0_ns
|
|
self.t0_wall_ns = t0_wall_ns
|
|
self.predicted_seq = predicted_seq
|
|
self.active0 = bool(active0)
|
|
self.has_work0 = bool(has_work0)
|
|
self.forward_total0 = forward_total0
|
|
self.recv = None
|
|
self.get_next = None
|
|
self.run_batch = None
|
|
self.process = None
|
|
self.kv_send_start = None
|
|
self.kv_send_end = None
|
|
self.kv_chunks = 0
|
|
self.span = None
|
|
self.span_ctx = None
|
|
self.pass_span_child = False
|
|
self.recv_span_child = False
|
|
self.launch_blocked_reason = ""
|
|
self.seq = 0
|
|
self.no_batch_reason = ""
|
|
self.pass_end_wall_ns = 0
|
|
self.arrival_est_wall_ns = 0
|
|
# Only populated when SGLANG_TRACE_LOOP_DETAIL=1 and this loop is
|
|
# sampled; see the "Loop detail phases" section below.
|
|
self.detail = None
|
|
|
|
|
|
def ensure_loop_span(lt: Optional[LoopTimings]) -> None:
|
|
"""Materialize a sampled span once the loop is known to have work."""
|
|
if (
|
|
lt is None
|
|
or lt.span is not None
|
|
or not loop_sample_decision(lt.predicted_seq)
|
|
):
|
|
return
|
|
try:
|
|
from opentelemetry import trace as otel_trace
|
|
|
|
tracer = otel_trace.get_tracer("sglang.prefill.loop")
|
|
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
|
|
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
|
|
except Exception:
|
|
lt.span = None
|
|
lt.span_ctx = None
|
|
|
|
|
|
def note_launch_blocked(lt: Optional[LoopTimings], reason: str) -> None:
|
|
"""Preserve why this loop could not launch before queues are consumed."""
|
|
if lt is not None and not lt.launch_blocked_reason:
|
|
lt.launch_blocked_reason = reason
|
|
|
|
|
|
def begin_loop(
|
|
phase: str = "prefill",
|
|
*,
|
|
pre_has_work: bool = True,
|
|
forward_active=None,
|
|
):
|
|
"""Start one scheduler-loop accounting record.
|
|
|
|
``pre_has_work`` must be cheap to compute (queue lengths / last_batch):
|
|
fully idle loops are still counted, but never create a span.
|
|
``forward_active`` is a callable (usually DeviceTimer.is_active) or a bool.
|
|
"""
|
|
global _LAST_MLP_OUTCOME
|
|
if not _LOOP_ENABLED:
|
|
return None
|
|
try:
|
|
active0 = bool(forward_active() if callable(forward_active) else forward_active)
|
|
except Exception:
|
|
active0 = False
|
|
lt = LoopTimings(
|
|
phase=phase,
|
|
t0_ns=time.monotonic_ns(),
|
|
t0_wall_ns=time.time_ns(),
|
|
predicted_seq=_DP_GATHER_SEQ + 1,
|
|
active0=active0,
|
|
has_work0=bool(pre_has_work),
|
|
forward_total0=_FORWARD_DEVICE_NS_TOTAL,
|
|
)
|
|
_LAST_MLP_OUTCOME = ("none", "none")
|
|
_LOOP_TLS.current = lt
|
|
if lt.has_work0:
|
|
ensure_loop_span(lt)
|
|
return lt
|
|
|
|
|
|
def current_loop() -> Optional[LoopTimings]:
|
|
return getattr(_LOOP_TLS, "current", None)
|
|
|
|
|
|
def current_loop_context():
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
return lt.span_ctx if lt is not None else None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Loop detail phases (SGLANG_TRACE_LOOP_DETAIL=1).
|
|
#
|
|
# Splits run_batch / process_result into host phases. Gated twice: the module
|
|
# switch must be on AND the current loop must be sampled (same deterministic
|
|
# decision as the loop span), so detail data never covers an unsampled loop.
|
|
#
|
|
# Accounting rules (see handoff/prefill-cpu-kv-overlap-design-20261106.md §3.2):
|
|
# * single phases -> one child span (``scheduler.loop.<name>``)
|
|
# * repeated phases -> summed attributes only (``sglang.<name>_ms`` + count)
|
|
# * durations use time.monotonic_ns(); OTLP start/end use time.time_ns()
|
|
# * residuals are clamped >= 0 and reported, never folded into a phase
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_DETAIL_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
|
|
"SGLANG_TRACE_LOOP_DETAIL", "0"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
|
|
_DETAIL_TLS = threading.local()
|
|
|
|
# Chunk-level (Mooncake transfer worker) detail tracing. Independent switch:
|
|
# the worker runs on other threads and has no scheduler loop TLS.
|
|
_MOONCAKE_DETAIL_ENABLED = os.getenv(
|
|
"SGLANG_TRACE_MOONCAKE_DETAIL", "0"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
|
|
# Previous loop summary per scheduler thread, used for DP-skew correlation
|
|
# (previous loop's phases vs this loop's arrival skew).
|
|
_LAST_LOOP_SUMMARY: dict = {}
|
|
|
|
# Phases that are inherently per-request / per-segment: always attributes,
|
|
# even when a loop happens to have exactly one occurrence, so span count never
|
|
# grows with n_reqs / n_segments (design §4.2).
|
|
_DETAIL_ATTRIBUTE_ONLY_PREFIXES = (
|
|
"per_request_result",
|
|
"prefix_device_indices",
|
|
"kv_state_payload",
|
|
"kv_page_indices",
|
|
"kv_sender_send",
|
|
)
|
|
|
|
_DETAIL_PREV_KEYS = (
|
|
"run_batch_ms",
|
|
"process_result_ms",
|
|
"copy_done_wait_ms",
|
|
"kv_state_payload_ms",
|
|
"kv_page_indices_ms",
|
|
"kv_sender_send_ms",
|
|
)
|
|
|
|
|
|
def loop_detail_enabled() -> bool:
|
|
"""True when detail is on and the *current* loop is sampled."""
|
|
if not _DETAIL_ENABLED:
|
|
return False
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
return lt is not None and loop_sample_decision(lt.predicted_seq)
|
|
|
|
|
|
def mooncake_detail_enabled() -> bool:
|
|
return _MOONCAKE_DETAIL_ENABLED
|
|
|
|
|
|
def _detail_state():
|
|
"""Detail state of the current loop, only when that loop is sampled."""
|
|
if not _DETAIL_ENABLED:
|
|
return None
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
if lt is None or not loop_sample_decision(lt.predicted_seq):
|
|
return None
|
|
if lt.detail is None:
|
|
lt.detail = {"phases": {}, "counts": {}, "sums": {}}
|
|
return lt.detail
|
|
|
|
|
|
def detail_begin(name: str) -> Optional[int]:
|
|
"""Start a single-occurrence phase. Returns a monotonic-ns token."""
|
|
if not _DETAIL_ENABLED:
|
|
return None
|
|
state = _detail_state()
|
|
if state is None:
|
|
return None
|
|
now_ns = time.monotonic_ns()
|
|
opens = getattr(_DETAIL_TLS, "open", None)
|
|
if opens is None:
|
|
opens = {}
|
|
_DETAIL_TLS.open = opens
|
|
opens[name] = (now_ns, time.time_ns(), _thread_time_ns())
|
|
return now_ns
|
|
|
|
|
|
def detail_end(name: str, token: Optional[int] = None) -> None:
|
|
"""Close a phase opened by ``detail_begin`` (or started at ``token``)."""
|
|
if not _DETAIL_ENABLED:
|
|
return
|
|
state = _detail_state()
|
|
if state is None:
|
|
return
|
|
now_ns = time.monotonic_ns()
|
|
opens = getattr(_DETAIL_TLS, "open", None) or {}
|
|
entry = opens.pop(name, None)
|
|
start_wall_ns = entry[1] if entry is not None else None
|
|
thread_cpu_ns = None
|
|
if entry is not None and len(entry) > 2 and entry[2] is not None:
|
|
cpu_now = _thread_time_ns()
|
|
if cpu_now is not None:
|
|
thread_cpu_ns = max(0, cpu_now - int(entry[2]))
|
|
if token is None:
|
|
if entry is None:
|
|
return
|
|
token = entry[0]
|
|
if start_wall_ns is None:
|
|
start_wall_ns = time.time_ns() - max(0, now_ns - int(token))
|
|
_detail_record(
|
|
state,
|
|
name,
|
|
int(token),
|
|
now_ns,
|
|
start_wall_ns,
|
|
time.time_ns(),
|
|
thread_cpu_ns=thread_cpu_ns,
|
|
)
|
|
|
|
|
|
def detail_add(
|
|
name: str,
|
|
start_ns: int,
|
|
end_ns: int,
|
|
thread_cpu_ns: Optional[int] = None,
|
|
) -> None:
|
|
"""Accumulate one occurrence of a phase (no span per occurrence).
|
|
|
|
``thread_cpu_ns`` is optional: callers that did not measure CPU time must
|
|
leave it as None so reports can tell "not measured" apart from "zero".
|
|
"""
|
|
if not _DETAIL_ENABLED or start_ns is None or end_ns is None:
|
|
return
|
|
state = _detail_state()
|
|
if state is None:
|
|
return
|
|
start_ns = int(start_ns)
|
|
end_ns = int(end_ns)
|
|
if end_ns < start_ns:
|
|
end_ns = start_ns
|
|
now_wall_ns = time.time_ns()
|
|
_detail_record(
|
|
state,
|
|
name,
|
|
start_ns,
|
|
end_ns,
|
|
now_wall_ns - (end_ns - start_ns),
|
|
now_wall_ns,
|
|
thread_cpu_ns=thread_cpu_ns,
|
|
)
|
|
|
|
|
|
def _detail_record(
|
|
state,
|
|
name,
|
|
start_ns,
|
|
end_ns,
|
|
start_wall_ns,
|
|
end_wall_ns,
|
|
thread_cpu_ns: Optional[int] = None,
|
|
) -> None:
|
|
rec = state["phases"].get(name)
|
|
if rec is None:
|
|
rec = [end_ns - start_ns, 1, int(start_wall_ns), int(end_wall_ns), 0, 0]
|
|
state["phases"][name] = rec
|
|
else:
|
|
rec[0] += end_ns - start_ns
|
|
rec[1] += 1
|
|
rec[3] = int(end_wall_ns)
|
|
if thread_cpu_ns is not None:
|
|
rec[4] += int(thread_cpu_ns)
|
|
rec[5] += 1
|
|
|
|
|
|
def thread_cpu_ns() -> Optional[int]:
|
|
"""Thread CPU clock, or None when the platform has no such clock.
|
|
|
|
Callers outside this module (the scheduler's parent span) must use this
|
|
wrapper so an unsupported or failing clock degrades to "not measured"
|
|
instead of raising on the scheduling path.
|
|
"""
|
|
try:
|
|
return time.thread_time_ns()
|
|
except (AttributeError, OSError, ValueError):
|
|
return None
|
|
|
|
|
|
# Backwards-compatible private alias for module-local call sites.
|
|
_thread_time_ns = thread_cpu_ns
|
|
|
|
|
|
def detail_attr_setdefault(name: str, value) -> None:
|
|
"""Set a loop attribute only if this loop has not recorded it yet."""
|
|
if not _DETAIL_ENABLED:
|
|
return
|
|
state = _detail_state()
|
|
if state is None:
|
|
return
|
|
attrs = state.setdefault("attrs", {})
|
|
if name not in attrs:
|
|
attrs[name] = value
|
|
|
|
|
|
def detail_attr_sum(name: str, value: int) -> None:
|
|
"""Accumulate a numeric loop attribute (e.g. evicted tokens)."""
|
|
if not _DETAIL_ENABLED:
|
|
return
|
|
state = _detail_state()
|
|
if state is None:
|
|
return
|
|
sums = state.setdefault("sums", {})
|
|
sums[name] = int(sums.get(name, 0)) + int(value)
|
|
|
|
|
|
def detail_count(name: str, n: int = 1) -> None:
|
|
"""Accumulate a low-cardinality counter for this loop."""
|
|
if not _DETAIL_ENABLED:
|
|
return
|
|
state = _detail_state()
|
|
if state is None:
|
|
return
|
|
state["counts"][name] = int(state["counts"].get(name, 0)) + int(n)
|
|
|
|
|
|
def detail_attr(name: str, value) -> None:
|
|
"""Set a low-cardinality loop attribute (last write wins)."""
|
|
if not _DETAIL_ENABLED:
|
|
return
|
|
state = _detail_state()
|
|
if state is None:
|
|
return
|
|
state.setdefault("attrs", {})[name] = value
|
|
|
|
|
|
def detail_phase_total_ns(names) -> int:
|
|
"""Sum of accumulated ns for the given phases in the current loop.
|
|
|
|
``names`` are *base* names: a phase recorded as ``kv_sender_send.last``
|
|
(per send kind) is included when ``kv_sender_send`` is requested. Phase
|
|
names are only generated by this module, so prefix matching cannot pick up
|
|
an unrelated phase that merely shares a leading string.
|
|
"""
|
|
state = _detail_state()
|
|
if state is None:
|
|
return 0
|
|
phases = state.get("phases") or {}
|
|
total = 0
|
|
for name, rec in phases.items():
|
|
if rec is None:
|
|
continue
|
|
for want in names:
|
|
if name == want or name.startswith(want + "."):
|
|
total += int(rec[0])
|
|
break
|
|
return total
|
|
|
|
|
|
class detail_phase:
|
|
"""Context manager for one single-occurrence phase (cheap when disabled)."""
|
|
|
|
__slots__ = ("name", "token")
|
|
|
|
def __init__(self, name: str):
|
|
self.name = name
|
|
self.token = None
|
|
|
|
def __enter__(self):
|
|
self.token = detail_begin(self.name)
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
detail_end(self.name, self.token)
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HiCache flush micro timing (SGLANG_TRACE_BACKUP_DETAIL=1).
|
|
#
|
|
# One tracer per ``cache_controller.start_writing()`` call (i.e. per
|
|
# ``flush_pending_backups``). Sub-phases are host-wall timings on the caller
|
|
# thread; a synchronous ``.cpu()`` therefore shows up as a long host phase and
|
|
# must not be read as device-transfer time. The tracer keeps per-flush
|
|
# resolution by emitting ONE child span of the current loop
|
|
# (``scheduler.cache_flush_backup``), because the loop-level detail only keeps
|
|
# per-loop sums (design: handoff/prefill-flush-backups-validation-20261107.md).
|
|
#
|
|
# Disabled by default: every entry point is a single module-level bool check,
|
|
# and when off no object, dict or span is created.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_BACKUP_DETAIL_ENABLED = _DETAIL_ENABLED and os.getenv(
|
|
"SGLANG_TRACE_BACKUP_DETAIL", "0"
|
|
).strip().lower() not in ("", "0", "false", "no", "off")
|
|
|
|
_FLUSH_TLS = threading.local()
|
|
|
|
|
|
def backup_detail_enabled() -> bool:
|
|
"""True when per-flush micro timing is compiled in (``*_DETAIL`` is on)."""
|
|
return _BACKUP_DETAIL_ENABLED
|
|
|
|
|
|
def set_flush_origin(origin: Optional[str]) -> Optional[str]:
|
|
"""Tag the next flushes with the caller's scheduler context.
|
|
|
|
Only used to tell ``process_batch_result`` flushes (the current suspect)
|
|
from the ones issued by ``check_hicache_events`` / ``write``. Returns the
|
|
previous value so the caller can restore it.
|
|
"""
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return None
|
|
previous = getattr(_FLUSH_TLS, "origin", None)
|
|
_FLUSH_TLS.origin = origin
|
|
return previous
|
|
|
|
|
|
def current_flush_origin() -> str:
|
|
return getattr(_FLUSH_TLS, "origin", None) or "other"
|
|
|
|
|
|
class BackupFlushTracer:
|
|
"""Sub-phase timings for one flush; emits a single span on ``end``."""
|
|
|
|
__slots__ = (
|
|
"origin",
|
|
"start_ns",
|
|
"loop_seq",
|
|
"loop_ctx",
|
|
"phases",
|
|
"attrs",
|
|
"counts",
|
|
"_open",
|
|
"_prev",
|
|
)
|
|
|
|
def __init__(self, origin: str):
|
|
self.origin = origin
|
|
# Captured on the scheduler thread: the flushed span may be emitted by
|
|
# the async submit worker, which has no scheduler loop TLS.
|
|
self.loop_seq = current_loop_seq()
|
|
self.loop_ctx = current_loop_context_for_spans()
|
|
self.start_ns = time.monotonic_ns()
|
|
self.phases: dict[str, int] = {}
|
|
self.attrs: dict[str, Any] = {}
|
|
self.counts: dict[str, int] = {}
|
|
self._open: dict[str, int] = {}
|
|
self._prev = None
|
|
|
|
# -- phase helpers -----------------------------------------------------
|
|
def begin(self, name: str) -> int:
|
|
start = time.monotonic_ns()
|
|
self._open[name] = start
|
|
return start
|
|
|
|
def end(self, name: str, start_ns: Optional[int] = None) -> None:
|
|
start = self._open.pop(name, None)
|
|
if start is None:
|
|
start = start_ns
|
|
if start is None:
|
|
return
|
|
self.add(name, start)
|
|
|
|
def add(self, name: str, start_ns: int) -> None:
|
|
self.record(name, int(start_ns), time.monotonic_ns())
|
|
|
|
def record(self, name: str, start_ns: int, end_ns: int) -> None:
|
|
self.phases[name] = self.phases.get(name, 0) + max(
|
|
0, int(end_ns) - int(start_ns)
|
|
)
|
|
|
|
def set(self, key: str, value: Any) -> None:
|
|
self.attrs[key] = value
|
|
|
|
def inc(self, key: str, n: int = 1) -> None:
|
|
self.counts[key] = self.counts.get(key, 0) + int(n)
|
|
|
|
def total_ns(self) -> int:
|
|
return max(0, time.monotonic_ns() - self.start_ns)
|
|
|
|
# -- emission ----------------------------------------------------------
|
|
def end_and_emit(self) -> None:
|
|
now_ns = time.monotonic_ns()
|
|
total_ns = max(0, now_ns - self.start_ns)
|
|
attrs = {
|
|
"sglang.flush_origin": self.origin,
|
|
"sglang.flush_total_ms": total_ns / 1e6,
|
|
"sglang.loop_seq": int(self.loop_seq or 0),
|
|
}
|
|
for name, ns in self.phases.items():
|
|
attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6
|
|
for name, value in self.counts.items():
|
|
attrs["sglang.flush_" + name] = int(value)
|
|
attrs.update(self.attrs)
|
|
# OTel rejects None; keep a single malformed dimension from dropping the
|
|
# whole span.
|
|
attrs = {k: v for k, v in attrs.items() if v is not None}
|
|
try:
|
|
from opentelemetry import trace as otel_trace
|
|
|
|
ctx = self.loop_ctx
|
|
start_wall_ns = time.time_ns() - total_ns
|
|
span = otel_trace.get_tracer("sglang.cache.flush").start_span(
|
|
"scheduler.cache_flush_backup",
|
|
context=ctx,
|
|
start_time=int(start_wall_ns),
|
|
attributes=attrs,
|
|
)
|
|
span.end(end_time=int(time.time_ns()))
|
|
except Exception:
|
|
# Telemetry must never break the cache controller.
|
|
pass
|
|
|
|
|
|
class backup_phase:
|
|
"""Context manager for one sub-phase of the active flush (no-op when off)."""
|
|
|
|
__slots__ = ("name", "tracer", "start_ns")
|
|
|
|
def __init__(self, name: str):
|
|
self.name = name
|
|
self.tracer = None
|
|
self.start_ns = None
|
|
|
|
def __enter__(self):
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return self
|
|
self.tracer = getattr(_FLUSH_TLS, "tracer", None)
|
|
if self.tracer is not None:
|
|
self.start_ns = self.tracer.begin(self.name)
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
if self.tracer is not None:
|
|
self.tracer.end(self.name, self.start_ns)
|
|
return False
|
|
|
|
|
|
def backup_flush_begin(origin: Optional[str] = None) -> Optional[BackupFlushTracer]:
|
|
"""Start tracing one flush; returns None when disabled / not sampled."""
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return None
|
|
if not loop_detail_enabled():
|
|
return None
|
|
tracer = BackupFlushTracer(origin or current_flush_origin())
|
|
tracer._prev = getattr(_FLUSH_TLS, "tracer", None)
|
|
_FLUSH_TLS.tracer = tracer
|
|
return tracer
|
|
|
|
|
|
def backup_flush_end(tracer: Optional[BackupFlushTracer]) -> None:
|
|
if tracer is None:
|
|
return
|
|
if getattr(_FLUSH_TLS, "tracer", None) is tracer:
|
|
_FLUSH_TLS.tracer = tracer._prev
|
|
try:
|
|
tracer.end_and_emit()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def current_flush_tracer() -> Optional[BackupFlushTracer]:
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return None
|
|
return getattr(_FLUSH_TLS, "tracer", None)
|
|
|
|
|
|
def backup_flush_attach(tracer: Optional[BackupFlushTracer]):
|
|
"""Bind ``tracer`` to this thread so sub-phases land on the same span.
|
|
|
|
Used by the async submit worker, which executes phases on a different
|
|
thread than the one that created the tracer. Returns the previous binding.
|
|
"""
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return None
|
|
previous = getattr(_FLUSH_TLS, "tracer", None)
|
|
_FLUSH_TLS.tracer = tracer
|
|
return previous
|
|
|
|
|
|
def backup_flush_detach(previous) -> None:
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return
|
|
_FLUSH_TLS.tracer = previous
|
|
|
|
|
|
def flush_note(key: str, value: Any) -> None:
|
|
"""Attach a dimension to the active flush (no-op when disabled)."""
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return
|
|
tracer = getattr(_FLUSH_TLS, "tracer", None)
|
|
if tracer is not None:
|
|
tracer.set(key, value)
|
|
|
|
|
|
def flush_count(key: str, n: int = 1) -> None:
|
|
"""Accumulate a counter on the active flush (no-op when disabled)."""
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return
|
|
tracer = getattr(_FLUSH_TLS, "tracer", None)
|
|
if tracer is not None:
|
|
tracer.inc(key, n)
|
|
|
|
|
|
def flush_mark() -> Optional[int]:
|
|
"""Timestamp for a manual phase (None when no flush is being traced)."""
|
|
if not _BACKUP_DETAIL_ENABLED:
|
|
return None
|
|
if getattr(_FLUSH_TLS, "tracer", None) is None:
|
|
return None
|
|
return time.monotonic_ns()
|
|
|
|
|
|
def flush_phase_since(name: str, start_ns: Optional[int]) -> None:
|
|
"""Record ``name`` from ``start_ns`` to now (no-op when untraced)."""
|
|
if start_ns is None:
|
|
return
|
|
tracer = getattr(_FLUSH_TLS, "tracer", None)
|
|
if tracer is not None:
|
|
tracer.add(name, start_ns)
|
|
|
|
|
|
def current_loop_seq() -> int:
|
|
"""Loop sequence for cross-record correlation (0 when unavailable)."""
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
if lt is None:
|
|
return 0
|
|
return int(lt.seq or lt.predicted_seq or 0)
|
|
|
|
|
|
def current_loop_context_for_spans():
|
|
"""OTel context of the current loop span (None if unsampled)."""
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
return lt.span_ctx if lt is not None else None
|
|
|
|
|
|
def _chunk_parent_context(chunk):
|
|
trace_ctx = getattr(chunk, "trace_ctx", None)
|
|
return getattr(trace_ctx, "root_span_context", None)
|
|
|
|
|
|
def split_queue_wait(
|
|
queue_wait_accum_ns: int, last_enqueue_mono_ns: int, dequeue_mono_ns: int
|
|
) -> tuple:
|
|
"""Return ``(attempt_queue_wait_ns, total_queue_wait_ns)``.
|
|
|
|
The attempt value matches the last_enqueue->dequeue wall window used by the
|
|
``mooncake.queue_wait`` child span; the total value additionally includes
|
|
the queue waits of earlier deferred attempts (design §5.3 metric).
|
|
Keeping them separate prevents a span whose wall length disagrees with its
|
|
``wait_ms`` attribute (review finding).
|
|
"""
|
|
attempt = max(0, int(dequeue_mono_ns) - int(last_enqueue_mono_ns))
|
|
return attempt, max(0, int(queue_wait_accum_ns)) + attempt
|
|
|
|
|
|
class RoomDetailState:
|
|
"""Thread-safe per-room detail bookkeeping (scheduler vs worker threads).
|
|
|
|
The scheduler aborts/retracts a room while the transfer worker may still be
|
|
finishing an attempt. A reset therefore records a cancellation mark, and
|
|
the worker's join update re-checks under the same lock before writing, so a
|
|
late worker can never resurrect stats after abort (review finding).
|
|
"""
|
|
|
|
# Bounded: a marker only needs to survive until the in-flight worker
|
|
# attempt finishes (milliseconds). The cap prevents unbounded growth if an
|
|
# aborted room never gets a worker teardown.
|
|
_MAX_CANCELLED = 8192
|
|
|
|
def __init__(self):
|
|
self._lock = threading.Lock()
|
|
self._join: dict = {}
|
|
self._seq: dict = {}
|
|
self._first: dict = {}
|
|
self._cancelled: dict = {}
|
|
|
|
def _mark_cancelled(self, room: int) -> None:
|
|
self._cancelled[int(room)] = time.monotonic_ns()
|
|
while len(self._cancelled) > self._MAX_CANCELLED:
|
|
self._cancelled.pop(next(iter(self._cancelled)))
|
|
|
|
def begin_chunk(self, room: int, now_mono_ns: int, now_wall_ns: int):
|
|
"""Stamp a real enqueue. Returns ``(chunk_seq, first_enqueue_wall_ns)``.
|
|
|
|
A new chunk clears a previous cancellation mark: it is the start of a
|
|
fresh flow (a retry may land on the same room id).
|
|
"""
|
|
with self._lock:
|
|
self._cancelled.pop(int(room), None)
|
|
seq = int(self._seq.get(int(room), 0))
|
|
self._seq[int(room)] = seq + 1
|
|
first = self._first.get(int(room))
|
|
if first is None:
|
|
first = (int(now_mono_ns), int(now_wall_ns))
|
|
self._first[int(room)] = first
|
|
return seq, int(first[1])
|
|
|
|
def record_join(
|
|
self,
|
|
room: int,
|
|
*,
|
|
status: str,
|
|
room_failed: bool,
|
|
now_wall_ns: int,
|
|
queue_wait_total_ns: int,
|
|
worker_busy_ns: int,
|
|
enqueue_wall_ns: int,
|
|
is_last_chunk: bool,
|
|
chunk_seq: int,
|
|
) -> bool:
|
|
"""Update the per-room join summary; False when the write was refused."""
|
|
if status != "ok":
|
|
return False
|
|
with self._lock:
|
|
if room_failed or int(room) in self._cancelled:
|
|
return False
|
|
stats = self._join.get(int(room))
|
|
if stats is None:
|
|
stats = {"chunk_count": 0}
|
|
self._join[int(room)] = stats
|
|
stats["chunk_count"] = int(stats.get("chunk_count", 0)) + 1
|
|
first = self._first.get(int(room))
|
|
if first is not None:
|
|
stats["first_enqueue_wall_ns"] = int(first[1])
|
|
if is_last_chunk:
|
|
stats["last_chunk_enqueue_wall_ns"] = int(enqueue_wall_ns)
|
|
stats["last_chunk_complete_wall_ns"] = int(now_wall_ns)
|
|
stats["last_chunk_queue_wait_ms"] = queue_wait_total_ns / 1e6
|
|
stats["last_chunk_worker_ms"] = worker_busy_ns / 1e6
|
|
stats["last_chunk_chunk_seq"] = int(chunk_seq)
|
|
return True
|
|
|
|
def reset(self, room: int) -> None:
|
|
with self._lock:
|
|
self._join.pop(int(room), None)
|
|
self._seq.pop(int(room), None)
|
|
self._first.pop(int(room), None)
|
|
self._mark_cancelled(room)
|
|
|
|
def teardown(self, room: int, keep_join: bool) -> None:
|
|
with self._lock:
|
|
if not keep_join:
|
|
self._join.pop(int(room), None)
|
|
self._seq.pop(int(room), None)
|
|
self._first.pop(int(room), None)
|
|
self._cancelled.pop(int(room), None)
|
|
|
|
def pop_join(self, room: int):
|
|
with self._lock:
|
|
return self._join.pop(int(room), None)
|
|
|
|
|
|
def note_chunk_requeue(chunk, now_mono_ns: int, now_wall_ns: int) -> None:
|
|
"""Accumulate only the *queue* part of a finished attempt before re-enqueue.
|
|
|
|
The previous implementation added ``requeue - last_enqueue``, which also
|
|
contains the worker's own execution and the staging watermark wait, thus
|
|
inflating queue wait on every defer (review finding). Only the interval
|
|
from the last enqueue to this attempt's dequeue is queue wait.
|
|
"""
|
|
if not _MOONCAKE_DETAIL_ENABLED or not getattr(chunk, "observe", False):
|
|
return
|
|
last_enqueue = int(getattr(chunk, "last_enqueue_mono_ns", 0) or 0)
|
|
dequeue = int(getattr(chunk, "dequeue_mono_ns", 0) or 0)
|
|
if last_enqueue and dequeue:
|
|
chunk.queue_wait_accum_ns = int(
|
|
getattr(chunk, "queue_wait_accum_ns", 0) or 0
|
|
) + max(0, dequeue - last_enqueue)
|
|
chunk.last_enqueue_mono_ns = int(now_mono_ns)
|
|
chunk.last_enqueue_wall_ns = int(now_wall_ns)
|
|
chunk.dequeue_mono_ns = 0
|
|
chunk.requeue_count = int(getattr(chunk, "requeue_count", 0) or 0) + 1
|
|
|
|
|
|
def emit_mooncake_chunk(
|
|
*,
|
|
chunk,
|
|
status: str,
|
|
queue_wait_ns: int,
|
|
queue_wait_total_ns: int,
|
|
source_wait_ns: int,
|
|
indices_wait_ns: int,
|
|
kv_transfer_ns: int,
|
|
state_transfer_ns: int,
|
|
aux_transfer_ns: int,
|
|
attempt_wall_ns: int,
|
|
dequeue_wall_ns: int,
|
|
end_wall_ns: int,
|
|
n_destinations: int,
|
|
source_wait_wall=None,
|
|
indices_wait_wall=None,
|
|
dp_rank: int = -1,
|
|
tp_rank: int = -1,
|
|
) -> None:
|
|
"""One span per transfer-worker attempt, attributed by room/chunk_seq.
|
|
|
|
``worker_busy_ms`` follows the design definition: attempt wall time minus
|
|
the source/indices event waits (those are blocking waits, not worker work).
|
|
``attempt_wall_ms`` is the raw dequeue->end time.
|
|
|
|
Wait child spans use the real event timestamps captured by the worker.
|
|
A wait without a recorded window is attributes-only; no synthetic
|
|
``end - duration`` placement is drawn (review finding).
|
|
"""
|
|
if not _MOONCAKE_DETAIL_ENABLED:
|
|
return
|
|
try:
|
|
from opentelemetry import trace as otel_trace
|
|
|
|
tracer = otel_trace.get_tracer("sglang.mooncake.chunk")
|
|
# Attempt-scoped start: the last enqueue of this attempt, not the
|
|
# first enqueue of the chunk, so deferred retries do not overlap.
|
|
start_wall_ns = int(
|
|
getattr(chunk, "last_enqueue_wall_ns", 0)
|
|
or chunk.enqueue_wall_ns
|
|
or dequeue_wall_ns
|
|
) or end_wall_ns
|
|
worker_busy_ns = max(
|
|
0, int(attempt_wall_ns) - int(source_wait_ns) - int(indices_wait_ns)
|
|
)
|
|
residual_ns = max(
|
|
0,
|
|
worker_busy_ns
|
|
- int(kv_transfer_ns)
|
|
- int(state_transfer_ns)
|
|
- int(aux_transfer_ns),
|
|
)
|
|
span = tracer.start_span(
|
|
"mooncake.kv_chunk",
|
|
context=_chunk_parent_context(chunk),
|
|
start_time=start_wall_ns,
|
|
attributes={
|
|
"sglang.bootstrap_room": int(chunk.room),
|
|
"sglang.chunk_seq": int(chunk.chunk_seq),
|
|
"sglang.loop_seq": int(chunk.loop_seq),
|
|
"sglang.send_kind": str(chunk.send_kind or "unknown"),
|
|
"sglang.is_last_chunk": bool(chunk.is_last_chunk),
|
|
"sglang.queue_id": int(chunk.queue_id),
|
|
"sglang.requeue_count": int(chunk.requeue_count),
|
|
"sglang.num_kv_tokens": int(chunk.num_kv_tokens or 0),
|
|
"sglang.num_pages": int(chunk.num_pages),
|
|
"sglang.approx_bytes": int(chunk.approx_bytes),
|
|
"sglang.n_destinations": int(n_destinations),
|
|
"sglang.chunk_status": str(status),
|
|
"sglang.dp_rank": int(dp_rank),
|
|
"sglang.tp_rank": int(tp_rank),
|
|
# Attempt window (matches the child span) vs cumulative wait
|
|
# across deferred attempts (design §5.3 metric).
|
|
"sglang.queue_wait_ms": queue_wait_ns / 1e6,
|
|
"sglang.queue_wait_total_ms": queue_wait_total_ns / 1e6,
|
|
"sglang.source_wait_ms": source_wait_ns / 1e6,
|
|
"sglang.indices_wait_ms": indices_wait_ns / 1e6,
|
|
"sglang.kv_transfer_ms": kv_transfer_ns / 1e6,
|
|
"sglang.state_transfer_ms": state_transfer_ns / 1e6,
|
|
"sglang.aux_transfer_ms": aux_transfer_ns / 1e6,
|
|
"sglang.attempt_wall_ms": attempt_wall_ns / 1e6,
|
|
"sglang.worker_busy_ms": worker_busy_ns / 1e6,
|
|
"sglang.worker_residual_ms": residual_ns / 1e6,
|
|
},
|
|
)
|
|
span_ctx = otel_trace.set_span_in_context(span)
|
|
|
|
def _child(name, start_ns, stop_ns, wait_ns):
|
|
start_ns = int(start_ns or 0)
|
|
stop_ns = int(stop_ns or 0)
|
|
if start_ns <= 0 or stop_ns <= start_ns:
|
|
return
|
|
try:
|
|
tracer.start_span(
|
|
name,
|
|
context=span_ctx,
|
|
start_time=start_ns,
|
|
attributes={"sglang.wait_ms": max(0, int(wait_ns)) / 1e6},
|
|
).end(end_time=stop_ns)
|
|
except Exception:
|
|
pass
|
|
|
|
# Real event windows only; queue wait ends at dequeue. Zero-wait
|
|
# attempts (immediate dequeue) get no span.
|
|
if queue_wait_ns > 0:
|
|
_child(
|
|
"mooncake.queue_wait",
|
|
getattr(chunk, "last_enqueue_wall_ns", 0),
|
|
dequeue_wall_ns,
|
|
queue_wait_ns,
|
|
)
|
|
if source_wait_wall is not None:
|
|
_child(
|
|
"mooncake.wait_source",
|
|
source_wait_wall[0],
|
|
source_wait_wall[1],
|
|
source_wait_ns,
|
|
)
|
|
if indices_wait_wall is not None:
|
|
_child(
|
|
"mooncake.wait_indices",
|
|
indices_wait_wall[0],
|
|
indices_wait_wall[1],
|
|
indices_wait_ns,
|
|
)
|
|
span.end(end_time=int(end_wall_ns))
|
|
except Exception:
|
|
# Telemetry must never break the transfer worker.
|
|
pass
|
|
|
|
|
|
def emit_request_kv_join(
|
|
*,
|
|
req,
|
|
stats: dict,
|
|
dp_rank: int = -1,
|
|
completion_wall_ns: Optional[int] = None,
|
|
) -> None:
|
|
"""Request-level join summary: last-chunk exposure + completion poll."""
|
|
if not _MOONCAKE_DETAIL_ENABLED or not stats:
|
|
return
|
|
try:
|
|
from opentelemetry import trace as otel_trace
|
|
from opentelemetry.trace import Link
|
|
|
|
now_wall_ns = int(completion_wall_ns or time.time_ns())
|
|
enqueue_wall_ns = int(stats.get("last_chunk_enqueue_wall_ns", 0) or 0)
|
|
complete_wall_ns = int(stats.get("last_chunk_complete_wall_ns", 0) or 0)
|
|
final_exposed_ns = (
|
|
max(0, complete_wall_ns - enqueue_wall_ns)
|
|
if enqueue_wall_ns and complete_wall_ns
|
|
else 0
|
|
)
|
|
completion_poll_ns = (
|
|
max(0, now_wall_ns - complete_wall_ns) if complete_wall_ns else 0
|
|
)
|
|
# First enqueue comes from the manager's real queue.put() event
|
|
# (wall clock), not from pre-send preparation (review finding).
|
|
first_enqueue_wall_ns = int(stats.get("first_enqueue_wall_ns", 0) or 0)
|
|
first_enqueue_to_completion_ms = None
|
|
if first_enqueue_wall_ns:
|
|
first_enqueue_to_completion_ms = max(
|
|
0.0, (now_wall_ns - first_enqueue_wall_ns) / 1e6
|
|
)
|
|
req_ctx = _request_span_context(req)
|
|
attrs = {
|
|
"sglang.bootstrap_room": int(getattr(req, "bootstrap_room", 0) or 0),
|
|
"sglang.rid": str(getattr(req, "rid", "")),
|
|
"sglang.dp_rank": int(dp_rank),
|
|
"sglang.chunk_count": int(stats.get("chunk_count", 0)),
|
|
"sglang.last_chunk_queue_wait_ms": float(
|
|
stats.get("last_chunk_queue_wait_ms", 0.0)
|
|
),
|
|
"sglang.last_chunk_worker_ms": float(
|
|
stats.get("last_chunk_worker_ms", 0.0)
|
|
),
|
|
"sglang.final_exposed_ms": final_exposed_ns / 1e6,
|
|
"sglang.completion_poll_ms": completion_poll_ns / 1e6,
|
|
# Per-rank request join. rank_join_skew_ms (max-min over TP/DP
|
|
# ranks) is computed offline by joining this span by room.
|
|
"sglang.request_join_ms": (final_exposed_ns + completion_poll_ns) / 1e6,
|
|
}
|
|
if first_enqueue_to_completion_ms is not None:
|
|
attrs["sglang.first_enqueue_to_completion_ms"] = (
|
|
first_enqueue_to_completion_ms
|
|
)
|
|
tracer = otel_trace.get_tracer("sglang.kv.request")
|
|
tracer.start_span(
|
|
"kv.request_join",
|
|
# Cover the whole request transfer when first enqueue is known.
|
|
start_time=first_enqueue_wall_ns or enqueue_wall_ns or now_wall_ns,
|
|
links=[Link(req_ctx)] if req_ctx is not None else [],
|
|
attributes=attrs,
|
|
).end(end_time=now_wall_ns)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None:
|
|
"""Record the recv_requests phase inside the current loop (if any)."""
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
if lt is not None:
|
|
lt.recv = (int(start_wall_ns), int(end_wall_ns), dict(parts))
|
|
|
|
|
|
def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
|
|
"""Record get_next timings for the current loop (called from end_pass)."""
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
if lt is None:
|
|
return
|
|
end_wall_ns = time.time_ns()
|
|
start_wall_ns = end_wall_ns - int(total_s * 1e9)
|
|
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
|
|
other = max(0.0, total_s - measured)
|
|
lt.get_next = (
|
|
start_wall_ns,
|
|
end_wall_ns,
|
|
{
|
|
"sglang.get_next_ms": total_s * 1e3,
|
|
"sglang.match_ms": t.match * 1e3,
|
|
"sglang.adder_ms": t.adder * 1e3,
|
|
"sglang.hicache_ms": t.hicache * 1e3,
|
|
"sglang.bootstrap_ms": t.bootstrap * 1e3,
|
|
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
|
|
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
|
|
"sglang.other_ms": other * 1e3,
|
|
"sglang.candidates": int(t.candidates),
|
|
"sglang.skipped_prefetch": int(t.skipped_prefetch),
|
|
"sglang.admitted": int(t.admitted),
|
|
"sglang.bootstrap_candidates": int(t.bootstrap_candidates),
|
|
"sglang.pending_bootstrap": int(t.pending_bootstrap),
|
|
"sglang.resumed_chunked": int(bool(t.had_chunked)),
|
|
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
|
|
"sglang.mlp_sync_seq": int(t.seq),
|
|
},
|
|
)
|
|
lt.seq = int(t.seq)
|
|
lt.pass_end_wall_ns = end_wall_ns
|
|
# Estimated DP-sync arrival: the pass ended, then mlp_sync waited.
|
|
# See design §1.2: arrival_est = pass_end - mlp_sync.
|
|
lt.arrival_est_wall_ns = end_wall_ns - int(t.mlp_sync * 1e9)
|
|
|
|
|
|
def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
if lt is not None:
|
|
attrs = dict(attrs or {})
|
|
attrs.setdefault(
|
|
"sglang.run_batch_ms", max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6
|
|
)
|
|
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), attrs)
|
|
|
|
|
|
def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
if lt is not None:
|
|
attrs = dict(attrs or {})
|
|
attrs.setdefault(
|
|
"sglang.process_result_ms",
|
|
max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6,
|
|
)
|
|
lt.process = (int(start_wall_ns), int(end_wall_ns), attrs)
|
|
|
|
|
|
def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
|
|
lt = getattr(_LOOP_TLS, "current", None)
|
|
if lt is None:
|
|
return
|
|
lt.kv_chunks += 1
|
|
if lt.kv_send_start is None or start_wall_ns < lt.kv_send_start:
|
|
lt.kv_send_start = int(start_wall_ns)
|
|
if lt.kv_send_end is None or end_wall_ns > lt.kv_send_end:
|
|
lt.kv_send_end = int(end_wall_ns)
|
|
|
|
|
|
def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None:
|
|
"""Set by dp_attn.prepare_mlp_sync_batch_raw once per pass."""
|
|
if not _LOOP_ENABLED:
|
|
return
|
|
global _LAST_MLP_OUTCOME
|
|
_LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome))
|
|
if local_kind != "none" or global_outcome != "none":
|
|
# DP idle work is only known after the sync. Activate the span before
|
|
# end_pass so the get_next child can share the loop trace.
|
|
ensure_loop_span(getattr(_LOOP_TLS, "current", None))
|
|
|
|
|
|
def note_forward_device(t: float, category: str = "", **kwargs) -> None:
|
|
"""DeviceTimer reporter: accumulate completed forward device time."""
|
|
global _FORWARD_DEVICE_NS_TOTAL
|
|
try:
|
|
ns = int(float(t) * 1e9)
|
|
except Exception:
|
|
return
|
|
_FORWARD_DEVICE_NS_TOTAL += ns
|
|
if _FORWARD_SPAN_ENABLED:
|
|
_FORWARD_EVENTS.append((time.time_ns(), ns, str(category)))
|
|
|
|
|
|
def install_forward_reporter(scheduler) -> None:
|
|
"""Attach the forward device-time reporter to the existing DeviceTimer.
|
|
|
|
No CUDA event is added here: SGLANG_ENABLE_METRICS_DEVICE_TIMER already
|
|
runs the timer; this only reads its completed intervals.
|
|
"""
|
|
if not _LOOP_ENABLED:
|
|
return
|
|
try:
|
|
reporter = getattr(scheduler, "metrics_reporter", None)
|
|
timer = getattr(reporter, "forward_pass_device_timer", None)
|
|
if timer is not None:
|
|
timer.add_reporter(note_forward_device)
|
|
except Exception:
|
|
# Telemetry must never break scheduler init.
|
|
pass
|
|
|
|
|
|
def _ensure_loop_metrics():
|
|
global _LOOP_METRICS
|
|
if _LOOP_METRICS is None:
|
|
from prometheus_client import Counter
|
|
|
|
_LOOP_METRICS = {
|
|
"no_batch_seconds": Counter(
|
|
"sglang:scheduler_no_batch_seconds_total",
|
|
"Wall time of scheduler loops that ended without a runnable batch, "
|
|
"by reason and forward_overlap",
|
|
["dp_rank", "reason", "forward_overlap"],
|
|
),
|
|
"no_batch_total": Counter(
|
|
"sglang:scheduler_no_batch_total",
|
|
"Count of scheduler loops that ended without a runnable batch, "
|
|
"by reason and forward_overlap",
|
|
["dp_rank", "reason", "forward_overlap"],
|
|
),
|
|
"lockstep_seconds": Counter(
|
|
"sglang:scheduler_dp_lockstep_idle_seconds_total",
|
|
"Wall time of DP-lockstep idle-forward loops (this rank had no "
|
|
"local batch but the DP group required an idle forward)",
|
|
["dp_rank", "forward_overlap"],
|
|
),
|
|
}
|
|
return _LOOP_METRICS
|
|
|
|
|
|
def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str:
|
|
"""Low-cardinality no-batch reason, in the design's priority order.
|
|
|
|
``pass_snapshot`` is the get_next attribute dict (candidates / skipped /
|
|
admitted). It is None when the loop skipped batch preparation entirely.
|
|
"""
|
|
if state.get("result_queue", 0) >= state.get("result_limit", 2):
|
|
return "result_backpressure"
|
|
if state.get("deferred", 0) > 0 or state.get("paused"):
|
|
return "deferred_input"
|
|
waiting = int(state.get("waiting", 0) or 0)
|
|
if waiting > 0 and pass_snapshot is not None:
|
|
bootstrap_candidates = int(
|
|
pass_snapshot.get("sglang.bootstrap_candidates", 0) or 0
|
|
)
|
|
pending_bootstrap = int(
|
|
pass_snapshot.get("sglang.pending_bootstrap", 0) or 0
|
|
)
|
|
if bootstrap_candidates > 0 and pending_bootstrap == bootstrap_candidates:
|
|
return "waiting_bootstrap"
|
|
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
|
|
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
|
|
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
|
|
if candidates > 0 and skipped == candidates:
|
|
return "waiting_prefetch"
|
|
if admitted == 0:
|
|
return "admission_blocked"
|
|
inflight = int(state.get("inflight", 0) or 0)
|
|
bootstrap = int(state.get("bootstrap", 0) or 0)
|
|
if waiting == 0 and bootstrap:
|
|
return "waiting_bootstrap"
|
|
if waiting == 0 and inflight:
|
|
return "waiting_pd"
|
|
if (
|
|
waiting == 0
|
|
and inflight == 0
|
|
and bootstrap == 0
|
|
and int(state.get("result_queue", 0) or 0) == 0
|
|
and int(state.get("deferred", 0) or 0) == 0
|
|
and not state.get("waiting_for_result")
|
|
):
|
|
return "fully_idle"
|
|
if state.get("skip_batch_creation") and pass_snapshot is None:
|
|
# A failed preparation defers retries; keep the last classified reason.
|
|
return _LAST_NO_BATCH_REASON
|
|
return "no_ready_other"
|
|
|
|
|
|
def _batch_attrs(batch) -> dict:
|
|
if batch is None:
|
|
return {
|
|
"sglang.n_reqs": 0,
|
|
"sglang.n_new": 0,
|
|
"sglang.n_cached": 0,
|
|
"sglang.chunked": False,
|
|
}
|
|
try:
|
|
n_reqs = int(batch.batch_size())
|
|
except Exception:
|
|
n_reqs = len(getattr(batch, "reqs", ()) or ())
|
|
try:
|
|
n_new = int(getattr(batch, "extend_num_tokens", 0) or 0)
|
|
except Exception:
|
|
n_new = 0
|
|
try:
|
|
n_cached = int(sum(getattr(batch, "prefix_lens", ()) or ()))
|
|
except Exception:
|
|
n_cached = 0
|
|
try:
|
|
chunked = getattr(batch, "chunked_req", None) is not None
|
|
except Exception:
|
|
chunked = False
|
|
return {
|
|
"sglang.n_reqs": n_reqs,
|
|
"sglang.n_new": n_new,
|
|
"sglang.n_cached": n_cached,
|
|
"sglang.chunked": bool(chunked),
|
|
}
|
|
|
|
|
|
_LAST_LOOP_SEQ = 0
|
|
_LAST_LOOP_HAS_DETAIL = False
|
|
|
|
|
|
def _total_ms(ms: dict, base: str) -> float:
|
|
return sum(
|
|
value for name, value in ms.items() if name == base or name.startswith(base + ".")
|
|
)
|
|
|
|
|
|
def _update_prev_state(lt: LoopTimings, loop_seq: int):
|
|
"""Capture the previous loop summary and store this loop for the next one.
|
|
|
|
Returns ``(prev_seq, prev_has_detail, prev_summary, consecutive)``. Only a
|
|
loop that actually recorded detail becomes a valid predecessor, so a
|
|
sampled loop after an unsampled one is marked non-consecutive instead of
|
|
silently publishing a stale window (design §4.5).
|
|
"""
|
|
global _LAST_LOOP_SEQ, _LAST_LOOP_HAS_DETAIL
|
|
prev_seq = int(_LAST_LOOP_SEQ)
|
|
prev_has = bool(_LAST_LOOP_HAS_DETAIL)
|
|
prev_summary = dict(_LAST_LOOP_SUMMARY)
|
|
consecutive = prev_has and prev_seq == loop_seq - 1
|
|
|
|
ms = {
|
|
name: rec[0] / 1e6
|
|
for name, rec in ((lt.detail or {}).get("phases") or {}).items()
|
|
if rec
|
|
}
|
|
_LAST_LOOP_SUMMARY.clear()
|
|
if lt.detail is not None:
|
|
_LAST_LOOP_SUMMARY.update(
|
|
{
|
|
"run_batch_ms": float(
|
|
lt.run_batch[2].get("sglang.run_batch_ms", 0.0)
|
|
)
|
|
if lt.run_batch is not None
|
|
else 0.0,
|
|
"process_result_ms": float(
|
|
lt.process[2].get("sglang.process_result_ms", 0.0)
|
|
)
|
|
if lt.process is not None
|
|
else 0.0,
|
|
"copy_done_wait_ms": _total_ms(ms, "copy_done_wait"),
|
|
"kv_state_payload_ms": _total_ms(ms, "kv_state_payload"),
|
|
"kv_page_indices_ms": _total_ms(ms, "kv_page_indices"),
|
|
"kv_sender_send_ms": _total_ms(ms, "kv_sender_send"),
|
|
}
|
|
)
|
|
_LAST_LOOP_SEQ = int(loop_seq)
|
|
_LAST_LOOP_HAS_DETAIL = lt.detail is not None
|
|
return prev_seq, prev_has, prev_summary, consecutive
|
|
|
|
|
|
def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None:
|
|
"""Expose detail phases on the loop span and its child spans.
|
|
|
|
Single-occurrence phases become children (``scheduler.loop.<name>``);
|
|
repeated phases only contribute summed attributes, so span count never
|
|
grows with n_reqs / n_segments.
|
|
"""
|
|
detail = lt.detail or {}
|
|
phases = detail.get("phases") or {}
|
|
ms = {}
|
|
accounting_anomalies = 0
|
|
for name, rec in phases.items():
|
|
try:
|
|
total_ns, count, start_wall, end_wall = rec[:4]
|
|
except Exception:
|
|
continue
|
|
key = "sglang." + name.replace(".", "_") + "_ms"
|
|
ms[name] = total_ns / 1e6
|
|
attrs[key] = ms[name]
|
|
cpu_ns, cpu_count = (rec[4], rec[5]) if len(rec) > 5 else (0, 0)
|
|
prefix = "sglang." + name.replace(".", "_")
|
|
non_cpu_ms = None
|
|
if cpu_count:
|
|
attrs[prefix + "_thread_cpu_ms"] = cpu_ns / 1e6
|
|
attrs[prefix + "_thread_cpu_measured_count"] = int(cpu_count)
|
|
# Only a fully measured phase may report the wall - CPU remainder;
|
|
# a partial measurement would silently attribute unmeasured time to
|
|
# "non-CPU".
|
|
if cpu_count == count:
|
|
non_cpu_ms = max(0.0, (total_ns - cpu_ns) / 1e6)
|
|
attrs[prefix + "_non_cpu_ms"] = non_cpu_ms
|
|
# Only a deficit beyond clock granularity is an accounting
|
|
# anomaly; a few hundred ns of skew on a 2 ms phase is noise.
|
|
deficit_ns = cpu_ns - total_ns
|
|
if deficit_ns > max(100_000, total_ns // 100):
|
|
attrs[prefix + "_non_cpu_raw_ms"] = -deficit_ns / 1e6
|
|
attrs[prefix + "_accounting_anomaly"] = True
|
|
accounting_anomalies += 1
|
|
if count > 1:
|
|
attrs[prefix + "_count"] = int(count)
|
|
continue
|
|
if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES:
|
|
continue
|
|
child_attrs = {
|
|
"sglang.dp_rank": int(dp_rank),
|
|
"sglang.phase_ms": total_ns / 1e6,
|
|
}
|
|
if cpu_count:
|
|
child_attrs["sglang.phase_thread_cpu_ms"] = cpu_ns / 1e6
|
|
child_attrs["sglang.phase_cpu_measured_count"] = int(cpu_count)
|
|
if non_cpu_ms is not None:
|
|
child_attrs["sglang.phase_non_cpu_ms"] = non_cpu_ms
|
|
child(
|
|
"scheduler.loop." + name.replace(".", "_"),
|
|
start_wall,
|
|
end_wall,
|
|
child_attrs,
|
|
)
|
|
|
|
if accounting_anomalies:
|
|
attrs["sglang.accounting_anomaly_count"] = int(accounting_anomalies)
|
|
for name, value in (detail.get("counts") or {}).items():
|
|
attrs["sglang." + name.replace(".", "_") + "_count"] = int(value)
|
|
for name, value in (detail.get("sums") or {}).items():
|
|
attrs["sglang." + name.replace(".", "_")] = int(value)
|
|
for name, value in (detail.get("attrs") or {}).items():
|
|
attrs["sglang." + name.replace(".", "_")] = value
|
|
|
|
# Accounting residuals (design §3.2): never negative, never folded in.
|
|
def _total(*bases):
|
|
total = 0.0
|
|
for base in bases:
|
|
for name, value in ms.items():
|
|
if name == base or name.startswith(base + "."):
|
|
total += value
|
|
return total
|
|
|
|
# Roll per-kind variants (kv_page_indices.last, ...) up to one base
|
|
# attribute so analysis can use the total without knowing the kinds.
|
|
bases = {name.split(".", 1)[0] for name in ms}
|
|
for base in bases:
|
|
if base not in ms:
|
|
attrs["sglang." + base + "_ms"] = _total(base)
|
|
|
|
if lt.run_batch is not None:
|
|
run_ms = float(lt.run_batch[2].get("sglang.run_batch_ms", 0.0))
|
|
parts = _total(
|
|
"early_send_host",
|
|
"resolve_inputs",
|
|
"model_forward_host",
|
|
"copy_result_submit",
|
|
)
|
|
attrs["sglang.run_residual_ms"] = max(0.0, run_ms - parts)
|
|
if lt.process is not None:
|
|
proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0))
|
|
parts = _total(
|
|
# Outer phases of process_batch_result (scheduler.py).
|
|
"flush_trace",
|
|
"load_snapshot",
|
|
"flush_backups",
|
|
"step_counters",
|
|
"metrics_report",
|
|
# Inner disagg-prefill phases (prefill.py).
|
|
"copy_done_wait",
|
|
"auxiliary_and_logprob",
|
|
"per_request_result",
|
|
"report_stats",
|
|
)
|
|
if "sglang.process_kv_ms" in lt.process[2]:
|
|
# KV phases charged to this call only (sends issued later in the
|
|
# loop, e.g. pending-bootstrap, are not part of process_result).
|
|
parts += float(lt.process[2]["sglang.process_kv_ms"])
|
|
else:
|
|
parts += _total(
|
|
"kv_state_payload", "kv_page_indices", "kv_sender_send"
|
|
)
|
|
attrs["sglang.process_residual_ms"] = max(0.0, proc_ms - parts)
|
|
|
|
|
|
|
|
def end_loop(
|
|
lt: Optional[LoopTimings],
|
|
*,
|
|
dp_rank: int,
|
|
has_batch: bool,
|
|
batch=None,
|
|
state: Optional[dict] = None,
|
|
) -> None:
|
|
"""Close one loop: counters (always), loop span + phases (if sampled)."""
|
|
if lt is None:
|
|
return
|
|
_LOOP_TLS.current = None
|
|
state = state or {}
|
|
try:
|
|
now_ns = time.monotonic_ns()
|
|
now_wall_ns = time.time_ns()
|
|
loop_s = max(0.0, (now_ns - lt.t0_ns) / 1e9)
|
|
gpu_ns = max(0, _FORWARD_DEVICE_NS_TOTAL - lt.forward_total0)
|
|
active1 = bool(state.get("forward_active", False))
|
|
overlap = (
|
|
"full"
|
|
if lt.active0 and active1
|
|
else "partial"
|
|
if (lt.active0 or active1)
|
|
else "none"
|
|
)
|
|
local_kind, global_outcome = _LAST_MLP_OUTCOME
|
|
pass_snapshot = lt.get_next[2] if lt.get_next is not None else None
|
|
pending = bool(
|
|
state.get("waiting", 0)
|
|
or state.get("result_queue", 0)
|
|
or state.get("inflight", 0)
|
|
or state.get("bootstrap", 0)
|
|
or state.get("deferred", 0)
|
|
or state.get("paused")
|
|
or state.get("waiting_for_result")
|
|
)
|
|
reason = ""
|
|
|
|
try:
|
|
if local_kind == "none" and global_outcome == "idle":
|
|
# The DP sync replaces local None with a real idle batch, so this
|
|
# accounting must not depend on has_batch being False.
|
|
_ensure_loop_metrics()["lockstep_seconds"].labels(
|
|
dp_rank=str(dp_rank), forward_overlap=overlap
|
|
).inc(loop_s)
|
|
elif not has_batch:
|
|
reason = lt.launch_blocked_reason or _classify_no_batch(
|
|
state, pass_snapshot
|
|
)
|
|
global _LAST_NO_BATCH_REASON
|
|
_LAST_NO_BATCH_REASON = reason
|
|
metrics = _ensure_loop_metrics()
|
|
metrics["no_batch_seconds"].labels(
|
|
dp_rank=str(dp_rank),
|
|
reason=reason,
|
|
forward_overlap=overlap,
|
|
).inc(loop_s)
|
|
metrics["no_batch_total"].labels(
|
|
dp_rank=str(dp_rank),
|
|
reason=reason,
|
|
forward_overlap=overlap,
|
|
).inc()
|
|
except Exception:
|
|
# Telemetry must never break the scheduler loop.
|
|
pass
|
|
lt.no_batch_reason = reason
|
|
|
|
# Update the prev-loop chain on every loop (sampled or not) so a
|
|
# sampled loop after an unsampled one is reported as non-consecutive
|
|
# instead of silently carrying a stale window (review finding).
|
|
loop_seq_cur = int(lt.seq or lt.predicted_seq)
|
|
prev_state = None
|
|
if _DETAIL_ENABLED:
|
|
prev_state = _update_prev_state(lt, loop_seq_cur)
|
|
|
|
if lt.span is None and (has_batch or pending):
|
|
# Covers work discovered after begin_loop, including first input and
|
|
# deferred-only loops. DP outcomes activate earlier for pass parenting.
|
|
ensure_loop_span(lt)
|
|
if lt.span is None:
|
|
return
|
|
|
|
try:
|
|
from opentelemetry import trace as otel_trace
|
|
|
|
tracer = otel_trace.get_tracer("sglang.prefill.loop")
|
|
attrs = {
|
|
"sglang.phase": lt.phase,
|
|
"sglang.dp_rank": int(dp_rank),
|
|
"sglang.loop_seq": int(lt.seq or lt.predicted_seq),
|
|
"sglang.loop_ms": loop_s * 1e3,
|
|
"sglang.gpu_ms": gpu_ns / 1e6,
|
|
"sglang.gpu_idle_ms": max(0.0, loop_s * 1e3 - gpu_ns / 1e6),
|
|
"sglang.has_batch": bool(has_batch),
|
|
"sglang.engine_has_pending_work": pending,
|
|
"sglang.local_batch_kind": local_kind,
|
|
"sglang.global_outcome": global_outcome,
|
|
"sglang.no_batch_reason": reason,
|
|
"sglang.forward_overlap": overlap,
|
|
}
|
|
attrs.update(_batch_attrs(batch))
|
|
|
|
def _child(name, start_ns, end_ns, child_attrs):
|
|
try:
|
|
tracer.start_span(
|
|
name,
|
|
context=lt.span_ctx,
|
|
start_time=int(start_ns),
|
|
attributes=child_attrs,
|
|
).end(end_time=int(end_ns))
|
|
except Exception:
|
|
pass
|
|
|
|
if lt.detail is not None:
|
|
_emit_loop_detail(lt, attrs, _child, int(dp_rank))
|
|
if _DETAIL_ENABLED:
|
|
if lt.pass_end_wall_ns:
|
|
# DP-skew join fields (design §4.5); independent of phases.
|
|
attrs["sglang.pass_end_wall_ns"] = int(lt.pass_end_wall_ns)
|
|
attrs["sglang.arrival_est_wall_ns"] = int(lt.arrival_est_wall_ns)
|
|
if prev_state is not None:
|
|
prev_seq, prev_has, prev_summary, consecutive = prev_state
|
|
attrs["sglang.prev_loop_seq"] = int(prev_seq)
|
|
if consecutive:
|
|
for key in _DETAIL_PREV_KEYS:
|
|
if key in prev_summary:
|
|
attrs["sglang.prev_" + key] = prev_summary[key]
|
|
elif prev_seq:
|
|
# The previous loop was unsampled or had no detail;
|
|
# its phases must not be treated as seq-1 values.
|
|
attrs["sglang.prev_detail_missing"] = True
|
|
lt.span.set_attributes(attrs)
|
|
|
|
if lt.recv is not None and not lt.recv_span_child:
|
|
start, end, parts = lt.recv
|
|
child = {"sglang.dp_rank": int(dp_rank)}
|
|
child.update(parts)
|
|
_child("scheduler.loop.recv", start, end, child)
|
|
if lt.get_next is not None and not lt.pass_span_child:
|
|
start, end, ng_attrs = lt.get_next
|
|
_child("scheduler.loop.get_next", start, end, ng_attrs)
|
|
if lt.run_batch is not None:
|
|
start, end, rb_attrs = lt.run_batch
|
|
_child("scheduler.loop.run_batch", start, end, rb_attrs)
|
|
if lt.process is not None:
|
|
start, end, pr_attrs = lt.process
|
|
_child("scheduler.loop.process_result", start, end, pr_attrs)
|
|
if _KV_SEND_SPAN_ENABLED and lt.kv_send_start is not None:
|
|
_child(
|
|
"scheduler.loop.kv_send",
|
|
lt.kv_send_start,
|
|
lt.kv_send_end,
|
|
{"sglang.n_chunks": int(lt.kv_chunks)},
|
|
)
|
|
if _FORWARD_SPAN_ENABLED:
|
|
for completion_wall, elapsed_ns, category in list(_FORWARD_EVENTS):
|
|
if completion_wall < lt.t0_wall_ns or completion_wall > now_wall_ns:
|
|
continue
|
|
_child(
|
|
"scheduler.loop.forward.gpu",
|
|
completion_wall - elapsed_ns,
|
|
completion_wall,
|
|
{
|
|
"sglang.gpu_ms": elapsed_ns / 1e6,
|
|
"sglang.category": category,
|
|
},
|
|
)
|
|
lt.span.end(end_time=now_wall_ns)
|
|
except Exception:
|
|
# Telemetry must never break the scheduler loop.
|
|
pass
|
|
finally:
|
|
_LOOP_TLS.current = None
|