observability: commit the in-image per-batch telemetry baseline (btelem4-6)
These three files were already running in the gist production image
(gist-nvlink-ee9170e7bb-pzcfix) but were never committed: the image was
produced by docker-commit with the btelem4/5/6 files overlaid on top of
the built source tree.
Port them so the tree matches the image before layering new instrumentation:
* observability/batch_telemetry.py: per-batch histograms, per-DP-step
shape histograms and the sampled OTLP span scheduler.batch.{prefill,decode}.
* managers/scheduler_components/metrics_reporter.py: observe_prefill/
observe_decode call sites and the batch-trace log suffix.
* managers/scheduler_components/dp_attn.py: observe_dp_step().
Content is identical to the image; only line endings were normalised to LF
(the image copy was mixed CRLF/LF). No behavior change when
SGLANG_BATCH_TELEMETRY is unset (default off).
This commit is contained in:
@@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.metrics_reporter import (
|
||||
from sglang.srt.managers.scheduler_components.recv_skipper import (
|
||||
SchedulerRecvSkipper,
|
||||
)
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.managers.utils import is_health_check_generate_req
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
@@ -295,6 +296,14 @@ class MLPSyncBatchInfo:
|
||||
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
|
||||
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
|
||||
self.sync_wait_seconds = time.perf_counter() - sync_start
|
||||
if batch_telemetry.enabled():
|
||||
batch_telemetry.observe_dp_step(
|
||||
self.global_num_tokens,
|
||||
get_parallel().attn_dp_rank,
|
||||
modes=tp0_info_cpu[:, 5].tolist(),
|
||||
health=tp0_info_cpu[:, 9].tolist(),
|
||||
phase="extend" if self.is_extend_in_batch else "decode",
|
||||
)
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
tp0_info_cpu[:, 5].tolist()
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.observability.forward_pass_metrics import (
|
||||
WelfordAccumulator,
|
||||
wrap_forward_with_fpm,
|
||||
)
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
DPCooperationInfo,
|
||||
QueueCount,
|
||||
@@ -798,6 +799,21 @@ class SchedulerMetricsReporter:
|
||||
self.forward_pass_device_timer._report()
|
||||
msg += ", " + self.forward_occupancy_log_string()
|
||||
|
||||
if batch_telemetry.enabled():
|
||||
_batch_trace_id = batch_telemetry.observe_prefill(
|
||||
dp_rank=get_parallel().attn_dp_rank,
|
||||
iter_no=batch_iter,
|
||||
n_reqs=prefill_stats.num_new_seqs,
|
||||
n_new=prefill_stats.log_input_tokens,
|
||||
n_cached=prefill_stats.log_hit_tokens,
|
||||
queue_depth=len(self.scheduler.waiting_queue),
|
||||
pending_tokens=prefill_stats.num_pending_tokens,
|
||||
duration_s=gap_latency,
|
||||
reqs=batch.reqs if batch is not None else None,
|
||||
)
|
||||
if _batch_trace_id:
|
||||
msg += f", batch-trace: {_batch_trace_id}"
|
||||
|
||||
if self.is_stats_logging_rank:
|
||||
logger.info(msg)
|
||||
if self.current_scheduler_metrics_enabled:
|
||||
@@ -940,6 +956,20 @@ class SchedulerMetricsReporter:
|
||||
self.last_decode_stats_tic = time.perf_counter()
|
||||
self.last_gen_throughput = self.num_generated_tokens / gap_latency
|
||||
|
||||
_decode_batch_trace = None
|
||||
if batch_telemetry.enabled():
|
||||
_decode_batch_trace = batch_telemetry.observe_decode(
|
||||
dp_rank=get_parallel().attn_dp_rank,
|
||||
iter_no=self.forward_ct_decode,
|
||||
n_reqs=len(batch.reqs) if batch is not None else 0,
|
||||
n_new=num_generated_tokens,
|
||||
n_cached=0,
|
||||
queue_depth=len(self.scheduler.waiting_queue),
|
||||
pending_tokens=0,
|
||||
duration_s=gap_latency,
|
||||
reqs=batch.reqs if batch is not None else None,
|
||||
)
|
||||
|
||||
self.num_generated_tokens = 0
|
||||
num_running_reqs = len(batch.reqs)
|
||||
|
||||
@@ -1064,6 +1094,9 @@ class SchedulerMetricsReporter:
|
||||
self.forward_pass_device_timer._report()
|
||||
msg += ", " + self.forward_occupancy_log_string()
|
||||
|
||||
if _decode_batch_trace:
|
||||
msg += f", batch-trace: {_decode_batch_trace}"
|
||||
|
||||
if self.is_stats_logging_rank:
|
||||
logger.info(msg)
|
||||
if self.current_scheduler_metrics_enabled:
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Per-batch scheduler telemetry (opt-in).
|
||||
|
||||
Adds, at the *batch* granularity (one record per scheduler batch per DP rank):
|
||||
|
||||
* Prometheus histograms (safe for the multiprocess collector):
|
||||
sglang:batch_new_tokens{phase,dp_rank}
|
||||
sglang:batch_cached_tokens{phase,dp_rank}
|
||||
sglang:batch_duration_seconds{phase,dp_rank}
|
||||
sglang:batch_queue_depth{phase,dp_rank}
|
||||
sglang:batch_pending_tokens{phase,dp_rank}
|
||||
sglang:batch_n_reqs{phase,dp_rank}
|
||||
* An OTLP span ``scheduler.batch.{phase}`` with ``Link``s to the request
|
||||
traces that took part in the batch (batch -> requests navigation), and a
|
||||
``sglang.batch_id`` attribute for reverse lookups / log correlation.
|
||||
|
||||
Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
|
||||
|
||||
SGLANG_BATCH_SPAN=0 histograms only (no spans)
|
||||
SGLANG_BATCH_SPAN_MIN_MS=50 only emit spans for batches slower than this
|
||||
SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0)
|
||||
SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span
|
||||
|
||||
Cost model (measured on our 4xDP4 / 4-GPU instances):
|
||||
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
|
||||
* span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s.
|
||||
Everything is behind a single module-level bool, so when disabled the hot
|
||||
path pays one ``if not _ENABLED: return``.
|
||||
|
||||
NOTE: sglang runs prometheus_client in multiprocess mode, which does not
|
||||
carry exemplars, so the span trace id is returned to the caller and appended
|
||||
to the batch log line instead (see ``metrics_reporter``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
_ENABLED = os.getenv("SGLANG_BATCH_TELEMETRY", "0").strip().lower() not in (
|
||||
"",
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
)
|
||||
_SPAN_ENABLED = _ENABLED and os.getenv("SGLANG_BATCH_SPAN", "1").strip().lower() not in (
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
)
|
||||
_SPAN_MIN_S = float(os.getenv("SGLANG_BATCH_SPAN_MIN_MS", "0")) / 1000.0
|
||||
_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_SPAN_SAMPLE", "1.0"))
|
||||
_MAX_LINKS = int(os.getenv("SGLANG_BATCH_SPAN_MAX_LINKS", "32"))
|
||||
|
||||
_HISTOGRAMS = None
|
||||
_CHILDREN: dict[tuple[str, str, int], Any] = {}
|
||||
_TRACER = None
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return _ENABLED
|
||||
|
||||
|
||||
def _ensure_histograms():
|
||||
global _HISTOGRAMS
|
||||
if _HISTOGRAMS is None:
|
||||
from prometheus_client import Histogram
|
||||
|
||||
labels = ["phase", "dp_rank"]
|
||||
_HISTOGRAMS = {
|
||||
"batch_new_tokens": Histogram(
|
||||
"sglang:batch_new_tokens",
|
||||
"New (uncached) prompt tokens processed by one scheduler batch",
|
||||
labels,
|
||||
buckets=(128, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536),
|
||||
),
|
||||
"batch_cached_tokens": Histogram(
|
||||
"sglang:batch_cached_tokens",
|
||||
"Cached prompt tokens carried by one scheduler batch",
|
||||
labels,
|
||||
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
|
||||
),
|
||||
"batch_duration_seconds": Histogram(
|
||||
"sglang:batch_duration_seconds",
|
||||
"Wall time spent in one scheduler batch",
|
||||
labels,
|
||||
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0),
|
||||
),
|
||||
"batch_queue_depth": Histogram(
|
||||
"sglang:batch_queue_depth",
|
||||
"Waiting-queue depth observed by one scheduler batch",
|
||||
labels,
|
||||
buckets=(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
|
||||
),
|
||||
"batch_pending_tokens": Histogram(
|
||||
"sglang:batch_pending_tokens",
|
||||
"Prefill backlog (pending tokens) observed by one scheduler batch",
|
||||
labels,
|
||||
buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576),
|
||||
),
|
||||
"batch_n_reqs": Histogram(
|
||||
"sglang:batch_n_reqs",
|
||||
"Requests carried by one scheduler batch",
|
||||
labels,
|
||||
buckets=(1, 2, 4, 8, 16, 32, 64, 128, 256),
|
||||
),
|
||||
}
|
||||
return _HISTOGRAMS
|
||||
|
||||
|
||||
def _child(name: str, phase: str, dp_rank: int):
|
||||
key = (name, phase, dp_rank)
|
||||
child = _CHILDREN.get(key)
|
||||
if child is None:
|
||||
child = _ensure_histograms()[name].labels(phase=phase, dp_rank=str(dp_rank))
|
||||
_CHILDREN[key] = child
|
||||
return child
|
||||
|
||||
|
||||
def _request_span_context(req: Any):
|
||||
"""SpanContext of a request, from either a live span or a serialized copy.
|
||||
|
||||
Scheduler processes receive a *copy* of the request trace context where
|
||||
``root_span`` is None (see ``trace_async.copy_for_thread``) and only
|
||||
``root_span_context`` (an OTel ``Context``) survives. Handle both.
|
||||
"""
|
||||
trace_ctx = getattr(getattr(req, "time_stats", None), "trace_ctx", None)
|
||||
if trace_ctx is None:
|
||||
return None
|
||||
try:
|
||||
root_span = getattr(trace_ctx, "root_span", None)
|
||||
if root_span is not None:
|
||||
span_ctx = root_span.get_span_context()
|
||||
if span_ctx.is_valid:
|
||||
return span_ctx
|
||||
root_ctx = getattr(trace_ctx, "root_span_context", None)
|
||||
if root_ctx is not None:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
span_ctx = otel_trace.get_current_span(root_ctx).get_span_context()
|
||||
if span_ctx.is_valid:
|
||||
return span_ctx
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _emit_span(
|
||||
phase: str,
|
||||
dp_rank: int,
|
||||
iter_no: int,
|
||||
n_reqs: int,
|
||||
n_new: int,
|
||||
n_cached: int,
|
||||
queue_depth: int,
|
||||
pending_tokens: int,
|
||||
duration_s: float,
|
||||
reqs: Optional[Iterable[Any]],
|
||||
) -> Optional[str]:
|
||||
"""Create one batch span with links to the request traces. Returns trace id."""
|
||||
if not _SPAN_ENABLED:
|
||||
return None
|
||||
if _SPAN_MIN_S > 0 and duration_s < _SPAN_MIN_S:
|
||||
return None
|
||||
if _SPAN_SAMPLE < 1.0 and random.random() > _SPAN_SAMPLE:
|
||||
return None
|
||||
global _TRACER
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
from opentelemetry.trace import Link
|
||||
|
||||
if _TRACER is None:
|
||||
_TRACER = otel_trace.get_tracer("sglang.batch")
|
||||
links = []
|
||||
if reqs:
|
||||
for req in reqs:
|
||||
if len(links) >= _MAX_LINKS:
|
||||
break
|
||||
ctx = _request_span_context(req)
|
||||
if ctx is not None:
|
||||
links.append(Link(ctx))
|
||||
end_ns = time.time_ns()
|
||||
start_ns = end_ns - int(duration_s * 1e9)
|
||||
span = _TRACER.start_span(
|
||||
f"scheduler.batch.{phase}",
|
||||
start_time=start_ns,
|
||||
attributes={
|
||||
"sglang.batch_id": f"{dp_rank}:{iter_no}",
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.phase": phase,
|
||||
"sglang.n_reqs": int(n_reqs),
|
||||
"sglang.n_new_tokens": int(n_new),
|
||||
"sglang.n_cached_tokens": int(n_cached),
|
||||
"sglang.queue_depth": int(queue_depth),
|
||||
"sglang.pending_tokens": int(pending_tokens),
|
||||
"sglang.batch_duration_s": float(duration_s),
|
||||
},
|
||||
links=links,
|
||||
)
|
||||
span.end(end_time=end_ns)
|
||||
span_ctx = span.get_span_context()
|
||||
if not span_ctx.is_valid:
|
||||
# No TracerProvider configured (tracing disabled) -> NoOp span.
|
||||
return None
|
||||
return format(span_ctx.trace_id, "032x")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _observe(
|
||||
phase: str,
|
||||
dp_rank: int,
|
||||
iter_no: int,
|
||||
n_reqs: int,
|
||||
n_new: int,
|
||||
n_cached: int,
|
||||
queue_depth: int,
|
||||
pending_tokens: int,
|
||||
duration_s: float,
|
||||
reqs: Optional[Iterable[Any]],
|
||||
) -> Optional[str]:
|
||||
if not _ENABLED:
|
||||
return None
|
||||
try:
|
||||
trace_id = _emit_span(
|
||||
phase,
|
||||
dp_rank,
|
||||
iter_no,
|
||||
n_reqs,
|
||||
n_new,
|
||||
n_cached,
|
||||
queue_depth,
|
||||
pending_tokens,
|
||||
duration_s,
|
||||
reqs,
|
||||
)
|
||||
_child("batch_new_tokens", phase, dp_rank).observe(n_new)
|
||||
_child("batch_cached_tokens", phase, dp_rank).observe(n_cached)
|
||||
_child("batch_duration_seconds", phase, dp_rank).observe(duration_s)
|
||||
_child("batch_queue_depth", phase, dp_rank).observe(queue_depth)
|
||||
_child("batch_pending_tokens", phase, dp_rank).observe(pending_tokens)
|
||||
_child("batch_n_reqs", phase, dp_rank).observe(n_reqs)
|
||||
return trace_id
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
return None
|
||||
|
||||
|
||||
def observe_prefill(
|
||||
*,
|
||||
dp_rank: int,
|
||||
iter_no: int,
|
||||
n_reqs: int,
|
||||
n_new: int,
|
||||
n_cached: int,
|
||||
queue_depth: int,
|
||||
pending_tokens: int,
|
||||
duration_s: float,
|
||||
reqs: Optional[Iterable[Any]] = None,
|
||||
) -> Optional[str]:
|
||||
return _observe(
|
||||
"prefill",
|
||||
dp_rank,
|
||||
iter_no,
|
||||
n_reqs,
|
||||
n_new,
|
||||
n_cached,
|
||||
queue_depth,
|
||||
pending_tokens,
|
||||
duration_s,
|
||||
reqs,
|
||||
)
|
||||
|
||||
|
||||
def observe_decode(
|
||||
*,
|
||||
dp_rank: int,
|
||||
iter_no: int,
|
||||
n_reqs: int,
|
||||
n_new: int,
|
||||
n_cached: int,
|
||||
queue_depth: int,
|
||||
pending_tokens: int,
|
||||
duration_s: float,
|
||||
reqs: Optional[Iterable[Any]] = None,
|
||||
) -> Optional[str]:
|
||||
return _observe(
|
||||
"decode",
|
||||
dp_rank,
|
||||
iter_no,
|
||||
n_reqs,
|
||||
n_new,
|
||||
n_cached,
|
||||
queue_depth,
|
||||
pending_tokens,
|
||||
duration_s,
|
||||
reqs,
|
||||
)
|
||||
|
||||
_DP_HISTS = None
|
||||
|
||||
|
||||
def _ensure_dp_histograms():
|
||||
global _DP_HISTS
|
||||
if _DP_HISTS is None:
|
||||
from prometheus_client import Histogram
|
||||
|
||||
_DP_HISTS = {
|
||||
"dp_step_tokens": Histogram(
|
||||
"sglang:dp_step_tokens",
|
||||
"Per-rank token count in one DP-attention step",
|
||||
["dp_rank", "phase"],
|
||||
buckets=(0, 256, 1024, 2048, 4096, 8192, 16384, 32768),
|
||||
),
|
||||
"dp_step_padding_ratio": Histogram(
|
||||
"sglang:dp_step_padding_ratio",
|
||||
"1 - mean/max tokens across active DP ranks in one step (padding waste)",
|
||||
["phase", "n_active"],
|
||||
buckets=(0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8),
|
||||
),
|
||||
"dp_step_spread": Histogram(
|
||||
"sglang:dp_step_spread",
|
||||
"(max-min)/mean tokens across active DP ranks in one step",
|
||||
["phase", "n_active"],
|
||||
buckets=(0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0),
|
||||
),
|
||||
}
|
||||
return _DP_HISTS
|
||||
|
||||
|
||||
_DP_IDLE_MODE = 4 # ForwardMode.IDLE
|
||||
|
||||
|
||||
def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = "unknown") -> None:
|
||||
"""Record per-step shape alignment across *active* DP ranks.
|
||||
|
||||
``tokens``/``modes``/``health`` are the per-rank vectors from the DP
|
||||
all-gather (info tensor columns 0, 5, 9). IDLE ranks and health-check
|
||||
steps are excluded so the padding ratio reflects real batch-shape
|
||||
misalignment rather than idle-slot padding.
|
||||
"""
|
||||
if not _ENABLED:
|
||||
return
|
||||
try:
|
||||
if health and any(bool(h) for h in health):
|
||||
return
|
||||
sel = []
|
||||
for i, t in enumerate(tokens):
|
||||
if t is None:
|
||||
continue
|
||||
t = int(t)
|
||||
if t <= 1:
|
||||
continue
|
||||
if modes is not None and i < len(modes) and int(modes[i]) == _DP_IDLE_MODE:
|
||||
continue
|
||||
sel.append(t)
|
||||
if len(sel) < 2:
|
||||
return
|
||||
hist = _ensure_dp_histograms()
|
||||
mx = max(sel)
|
||||
mean = sum(sel) / len(sel)
|
||||
n_active = str(len(sel))
|
||||
key = ("dp_step_tokens", phase, dp_rank)
|
||||
child = _CHILDREN.get(key)
|
||||
if child is None:
|
||||
child = hist["dp_step_tokens"].labels(dp_rank=str(dp_rank), phase=phase)
|
||||
_CHILDREN[key] = child
|
||||
child.observe(int(tokens[dp_rank]) if 0 <= dp_rank < len(tokens) else int(mean))
|
||||
if mx > 0:
|
||||
hist["dp_step_padding_ratio"].labels(phase=phase, n_active=n_active).observe(
|
||||
1.0 - mean / mx
|
||||
)
|
||||
if mean > 0:
|
||||
hist["dp_step_spread"].labels(phase=phase, n_active=n_active).observe(
|
||||
(mx - min(sel)) / mean
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user