observability: commit the in-image per-batch telemetry baseline (btelem4-6)

These three files were already running in the gist production image
(gist-nvlink-ee9170e7bb-pzcfix) but were never committed: the image was
produced by docker-commit with the btelem4/5/6 files overlaid on top of
the built source tree.

Port them so the tree matches the image before layering new instrumentation:

  * observability/batch_telemetry.py: per-batch histograms, per-DP-step
    shape histograms and the sampled OTLP span scheduler.batch.{prefill,decode}.
  * managers/scheduler_components/metrics_reporter.py: observe_prefill/
    observe_decode call sites and the batch-trace log suffix.
  * managers/scheduler_components/dp_attn.py: observe_dp_step().

Content is identical to the image; only line endings were normalised to LF
(the image copy was mixed CRLF/LF). No behavior change when
SGLANG_BATCH_TELEMETRY is unset (default off).
This commit is contained in:
2026-11-04 14:48:38 +08:00
parent d47a99b2ad
commit 2434d09848
3 changed files with 423 additions and 0 deletions
@@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.metrics_reporter import (
from sglang.srt.managers.scheduler_components.recv_skipper import (
SchedulerRecvSkipper,
)
from sglang.srt.observability import batch_telemetry
from sglang.srt.managers.utils import is_health_check_generate_req
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
@@ -295,6 +296,14 @@ class MLPSyncBatchInfo:
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
self.sync_wait_seconds = time.perf_counter() - sync_start
if batch_telemetry.enabled():
batch_telemetry.observe_dp_step(
self.global_num_tokens,
get_parallel().attn_dp_rank,
modes=tp0_info_cpu[:, 5].tolist(),
health=tp0_info_cpu[:, 9].tolist(),
phase="extend" if self.is_extend_in_batch else "decode",
)
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
tp0_info_cpu[:, 5].tolist()
@@ -21,6 +21,7 @@ from sglang.srt.observability.forward_pass_metrics import (
WelfordAccumulator,
wrap_forward_with_fpm,
)
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.metrics_collector import (
DPCooperationInfo,
QueueCount,
@@ -798,6 +799,21 @@ class SchedulerMetricsReporter:
self.forward_pass_device_timer._report()
msg += ", " + self.forward_occupancy_log_string()
if batch_telemetry.enabled():
_batch_trace_id = batch_telemetry.observe_prefill(
dp_rank=get_parallel().attn_dp_rank,
iter_no=batch_iter,
n_reqs=prefill_stats.num_new_seqs,
n_new=prefill_stats.log_input_tokens,
n_cached=prefill_stats.log_hit_tokens,
queue_depth=len(self.scheduler.waiting_queue),
pending_tokens=prefill_stats.num_pending_tokens,
duration_s=gap_latency,
reqs=batch.reqs if batch is not None else None,
)
if _batch_trace_id:
msg += f", batch-trace: {_batch_trace_id}"
if self.is_stats_logging_rank:
logger.info(msg)
if self.current_scheduler_metrics_enabled:
@@ -940,6 +956,20 @@ class SchedulerMetricsReporter:
self.last_decode_stats_tic = time.perf_counter()
self.last_gen_throughput = self.num_generated_tokens / gap_latency
_decode_batch_trace = None
if batch_telemetry.enabled():
_decode_batch_trace = batch_telemetry.observe_decode(
dp_rank=get_parallel().attn_dp_rank,
iter_no=self.forward_ct_decode,
n_reqs=len(batch.reqs) if batch is not None else 0,
n_new=num_generated_tokens,
n_cached=0,
queue_depth=len(self.scheduler.waiting_queue),
pending_tokens=0,
duration_s=gap_latency,
reqs=batch.reqs if batch is not None else None,
)
self.num_generated_tokens = 0
num_running_reqs = len(batch.reqs)
@@ -1064,6 +1094,9 @@ class SchedulerMetricsReporter:
self.forward_pass_device_timer._report()
msg += ", " + self.forward_occupancy_log_string()
if _decode_batch_trace:
msg += f", batch-trace: {_decode_batch_trace}"
if self.is_stats_logging_rank:
logger.info(msg)
if self.current_scheduler_metrics_enabled:
@@ -0,0 +1,381 @@
"""Per-batch scheduler telemetry (opt-in).
Adds, at the *batch* granularity (one record per scheduler batch per DP rank):
* Prometheus histograms (safe for the multiprocess collector):
sglang:batch_new_tokens{phase,dp_rank}
sglang:batch_cached_tokens{phase,dp_rank}
sglang:batch_duration_seconds{phase,dp_rank}
sglang:batch_queue_depth{phase,dp_rank}
sglang:batch_pending_tokens{phase,dp_rank}
sglang:batch_n_reqs{phase,dp_rank}
* An OTLP span ``scheduler.batch.{phase}`` with ``Link``s to the request
traces that took part in the batch (batch -> requests navigation), and a
``sglang.batch_id`` attribute for reverse lookups / log correlation.
Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
SGLANG_BATCH_SPAN=0 histograms only (no spans)
SGLANG_BATCH_SPAN_MIN_MS=50 only emit spans for batches slower than this
SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0)
SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span
Cost model (measured on our 4xDP4 / 4-GPU instances):
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
* span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s.
Everything is behind a single module-level bool, so when disabled the hot
path pays one ``if not _ENABLED: return``.
NOTE: sglang runs prometheus_client in multiprocess mode, which does not
carry exemplars, so the span trace id is returned to the caller and appended
to the batch log line instead (see ``metrics_reporter``).
"""
from __future__ import annotations
import os
import random
import time
from typing import Any, Iterable, Optional
_ENABLED = os.getenv("SGLANG_BATCH_TELEMETRY", "0").strip().lower() not in (
"",
"0",
"false",
"no",
"off",
)
_SPAN_ENABLED = _ENABLED and os.getenv("SGLANG_BATCH_SPAN", "1").strip().lower() not in (
"0",
"false",
"no",
"off",
)
_SPAN_MIN_S = float(os.getenv("SGLANG_BATCH_SPAN_MIN_MS", "0")) / 1000.0
_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_SPAN_SAMPLE", "1.0"))
_MAX_LINKS = int(os.getenv("SGLANG_BATCH_SPAN_MAX_LINKS", "32"))
_HISTOGRAMS = None
_CHILDREN: dict[tuple[str, str, int], Any] = {}
_TRACER = None
def enabled() -> bool:
return _ENABLED
def _ensure_histograms():
global _HISTOGRAMS
if _HISTOGRAMS is None:
from prometheus_client import Histogram
labels = ["phase", "dp_rank"]
_HISTOGRAMS = {
"batch_new_tokens": Histogram(
"sglang:batch_new_tokens",
"New (uncached) prompt tokens processed by one scheduler batch",
labels,
buckets=(128, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536),
),
"batch_cached_tokens": Histogram(
"sglang:batch_cached_tokens",
"Cached prompt tokens carried by one scheduler batch",
labels,
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
),
"batch_duration_seconds": Histogram(
"sglang:batch_duration_seconds",
"Wall time spent in one scheduler batch",
labels,
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0),
),
"batch_queue_depth": Histogram(
"sglang:batch_queue_depth",
"Waiting-queue depth observed by one scheduler batch",
labels,
buckets=(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
),
"batch_pending_tokens": Histogram(
"sglang:batch_pending_tokens",
"Prefill backlog (pending tokens) observed by one scheduler batch",
labels,
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
),
"batch_n_reqs": Histogram(
"sglang:batch_n_reqs",
"Requests carried by one scheduler batch",
labels,
buckets=(1, 2, 4, 8, 16, 32, 64, 128, 256),
),
}
return _HISTOGRAMS
def _child(name: str, phase: str, dp_rank: int):
key = (name, phase, dp_rank)
child = _CHILDREN.get(key)
if child is None:
child = _ensure_histograms()[name].labels(phase=phase, dp_rank=str(dp_rank))
_CHILDREN[key] = child
return child
def _request_span_context(req: Any):
"""SpanContext of a request, from either a live span or a serialized copy.
Scheduler processes receive a *copy* of the request trace context where
``root_span`` is None (see ``trace_async.copy_for_thread``) and only
``root_span_context`` (an OTel ``Context``) survives. Handle both.
"""
trace_ctx = getattr(getattr(req, "time_stats", None), "trace_ctx", None)
if trace_ctx is None:
return None
try:
root_span = getattr(trace_ctx, "root_span", None)
if root_span is not None:
span_ctx = root_span.get_span_context()
if span_ctx.is_valid:
return span_ctx
root_ctx = getattr(trace_ctx, "root_span_context", None)
if root_ctx is not None:
from opentelemetry import trace as otel_trace
span_ctx = otel_trace.get_current_span(root_ctx).get_span_context()
if span_ctx.is_valid:
return span_ctx
except Exception:
return None
return None
def _emit_span(
phase: str,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]],
) -> Optional[str]:
"""Create one batch span with links to the request traces. Returns trace id."""
if not _SPAN_ENABLED:
return None
if _SPAN_MIN_S > 0 and duration_s < _SPAN_MIN_S:
return None
if _SPAN_SAMPLE < 1.0 and random.random() > _SPAN_SAMPLE:
return None
global _TRACER
try:
from opentelemetry import trace as otel_trace
from opentelemetry.trace import Link
if _TRACER is None:
_TRACER = otel_trace.get_tracer("sglang.batch")
links = []
if reqs:
for req in reqs:
if len(links) >= _MAX_LINKS:
break
ctx = _request_span_context(req)
if ctx is not None:
links.append(Link(ctx))
end_ns = time.time_ns()
start_ns = end_ns - int(duration_s * 1e9)
span = _TRACER.start_span(
f"scheduler.batch.{phase}",
start_time=start_ns,
attributes={
"sglang.batch_id": f"{dp_rank}:{iter_no}",
"sglang.dp_rank": int(dp_rank),
"sglang.phase": phase,
"sglang.n_reqs": int(n_reqs),
"sglang.n_new_tokens": int(n_new),
"sglang.n_cached_tokens": int(n_cached),
"sglang.queue_depth": int(queue_depth),
"sglang.pending_tokens": int(pending_tokens),
"sglang.batch_duration_s": float(duration_s),
},
links=links,
)
span.end(end_time=end_ns)
span_ctx = span.get_span_context()
if not span_ctx.is_valid:
# No TracerProvider configured (tracing disabled) -> NoOp span.
return None
return format(span_ctx.trace_id, "032x")
except Exception:
return None
def _observe(
phase: str,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]],
) -> Optional[str]:
if not _ENABLED:
return None
try:
trace_id = _emit_span(
phase,
dp_rank,
iter_no,
n_reqs,
n_new,
n_cached,
queue_depth,
pending_tokens,
duration_s,
reqs,
)
_child("batch_new_tokens", phase, dp_rank).observe(n_new)
_child("batch_cached_tokens", phase, dp_rank).observe(n_cached)
_child("batch_duration_seconds", phase, dp_rank).observe(duration_s)
_child("batch_queue_depth", phase, dp_rank).observe(queue_depth)
_child("batch_pending_tokens", phase, dp_rank).observe(pending_tokens)
_child("batch_n_reqs", phase, dp_rank).observe(n_reqs)
return trace_id
except Exception:
# Telemetry must never break the scheduler loop.
return None
def observe_prefill(
*,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]] = None,
) -> Optional[str]:
return _observe(
"prefill",
dp_rank,
iter_no,
n_reqs,
n_new,
n_cached,
queue_depth,
pending_tokens,
duration_s,
reqs,
)
def observe_decode(
*,
dp_rank: int,
iter_no: int,
n_reqs: int,
n_new: int,
n_cached: int,
queue_depth: int,
pending_tokens: int,
duration_s: float,
reqs: Optional[Iterable[Any]] = None,
) -> Optional[str]:
return _observe(
"decode",
dp_rank,
iter_no,
n_reqs,
n_new,
n_cached,
queue_depth,
pending_tokens,
duration_s,
reqs,
)
_DP_HISTS = None
def _ensure_dp_histograms():
global _DP_HISTS
if _DP_HISTS is None:
from prometheus_client import Histogram
_DP_HISTS = {
"dp_step_tokens": Histogram(
"sglang:dp_step_tokens",
"Per-rank token count in one DP-attention step",
["dp_rank", "phase"],
buckets=(0, 256, 1024, 2048, 4096, 8192, 16384, 32768),
),
"dp_step_padding_ratio": Histogram(
"sglang:dp_step_padding_ratio",
"1 - mean/max tokens across active DP ranks in one step (padding waste)",
["phase", "n_active"],
buckets=(0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8),
),
"dp_step_spread": Histogram(
"sglang:dp_step_spread",
"(max-min)/mean tokens across active DP ranks in one step",
["phase", "n_active"],
buckets=(0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0),
),
}
return _DP_HISTS
_DP_IDLE_MODE = 4 # ForwardMode.IDLE
def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = "unknown") -> None:
"""Record per-step shape alignment across *active* DP ranks.
``tokens``/``modes``/``health`` are the per-rank vectors from the DP
all-gather (info tensor columns 0, 5, 9). IDLE ranks and health-check
steps are excluded so the padding ratio reflects real batch-shape
misalignment rather than idle-slot padding.
"""
if not _ENABLED:
return
try:
if health and any(bool(h) for h in health):
return
sel = []
for i, t in enumerate(tokens):
if t is None:
continue
t = int(t)
if t <= 1:
continue
if modes is not None and i < len(modes) and int(modes[i]) == _DP_IDLE_MODE:
continue
sel.append(t)
if len(sel) < 2:
return
hist = _ensure_dp_histograms()
mx = max(sel)
mean = sum(sel) / len(sel)
n_active = str(len(sel))
key = ("dp_step_tokens", phase, dp_rank)
child = _CHILDREN.get(key)
if child is None:
child = hist["dp_step_tokens"].labels(dp_rank=str(dp_rank), phase=phase)
_CHILDREN[key] = child
child.observe(int(tokens[dp_rank]) if 0 <= dp_rank < len(tokens) else int(mean))
if mx > 0:
hist["dp_step_padding_ratio"].labels(phase=phase, n_active=n_active).observe(
1.0 - mean / mx
)
if mean > 0:
hist["dp_step_spread"].labels(phase=phase, n_active=n_active).observe(
(mx - min(sel)) / mean
)
except Exception:
pass