diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index bd72eaf906..2cb6e9cf1b 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.metrics_reporter import ( from sglang.srt.managers.scheduler_components.recv_skipper import ( SchedulerRecvSkipper, ) +from sglang.srt.observability import batch_telemetry from sglang.srt.managers.utils import is_health_check_generate_req from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache @@ -295,6 +296,14 @@ class MLPSyncBatchInfo: self.any_health_check = bool(tp0_info_cpu[:, 9].max()) self.global_attention_pairs = tp0_info_cpu[:, 10].tolist() self.sync_wait_seconds = time.perf_counter() - sync_start + if batch_telemetry.enabled(): + batch_telemetry.observe_dp_step( + self.global_num_tokens, + get_parallel().attn_dp_rank, + modes=tp0_info_cpu[:, 5].tolist(), + health=tp0_info_cpu[:, 9].tolist(), + phase="extend" if self.is_extend_in_batch else "decode", + ) if _ENABLE_METRICS_DP_ATTENTION: self.dp_cooperation_info = DPCooperationInfo.create( tp0_info_cpu[:, 5].tolist() diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index ece7752556..d2ece72289 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -21,6 +21,7 @@ from sglang.srt.observability.forward_pass_metrics import ( WelfordAccumulator, wrap_forward_with_fpm, ) +from sglang.srt.observability import batch_telemetry from sglang.srt.observability.metrics_collector import ( DPCooperationInfo, QueueCount, @@ -798,6 +799,21 @@ class SchedulerMetricsReporter: self.forward_pass_device_timer._report() msg += ", " + self.forward_occupancy_log_string() + if batch_telemetry.enabled(): + _batch_trace_id = batch_telemetry.observe_prefill( + dp_rank=get_parallel().attn_dp_rank, + iter_no=batch_iter, + n_reqs=prefill_stats.num_new_seqs, + n_new=prefill_stats.log_input_tokens, + n_cached=prefill_stats.log_hit_tokens, + queue_depth=len(self.scheduler.waiting_queue), + pending_tokens=prefill_stats.num_pending_tokens, + duration_s=gap_latency, + reqs=batch.reqs if batch is not None else None, + ) + if _batch_trace_id: + msg += f", batch-trace: {_batch_trace_id}" + if self.is_stats_logging_rank: logger.info(msg) if self.current_scheduler_metrics_enabled: @@ -940,6 +956,20 @@ class SchedulerMetricsReporter: self.last_decode_stats_tic = time.perf_counter() self.last_gen_throughput = self.num_generated_tokens / gap_latency + _decode_batch_trace = None + if batch_telemetry.enabled(): + _decode_batch_trace = batch_telemetry.observe_decode( + dp_rank=get_parallel().attn_dp_rank, + iter_no=self.forward_ct_decode, + n_reqs=len(batch.reqs) if batch is not None else 0, + n_new=num_generated_tokens, + n_cached=0, + queue_depth=len(self.scheduler.waiting_queue), + pending_tokens=0, + duration_s=gap_latency, + reqs=batch.reqs if batch is not None else None, + ) + self.num_generated_tokens = 0 num_running_reqs = len(batch.reqs) @@ -1064,6 +1094,9 @@ class SchedulerMetricsReporter: self.forward_pass_device_timer._report() msg += ", " + self.forward_occupancy_log_string() + if _decode_batch_trace: + msg += f", batch-trace: {_decode_batch_trace}" + if self.is_stats_logging_rank: logger.info(msg) if self.current_scheduler_metrics_enabled: diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py new file mode 100644 index 0000000000..a2c9b1107a --- /dev/null +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -0,0 +1,381 @@ +"""Per-batch scheduler telemetry (opt-in). + +Adds, at the *batch* granularity (one record per scheduler batch per DP rank): + + * Prometheus histograms (safe for the multiprocess collector): + sglang:batch_new_tokens{phase,dp_rank} + sglang:batch_cached_tokens{phase,dp_rank} + sglang:batch_duration_seconds{phase,dp_rank} + sglang:batch_queue_depth{phase,dp_rank} + sglang:batch_pending_tokens{phase,dp_rank} + sglang:batch_n_reqs{phase,dp_rank} + * An OTLP span ``scheduler.batch.{phase}`` with ``Link``s to the request + traces that took part in the batch (batch -> requests navigation), and a + ``sglang.batch_id`` attribute for reverse lookups / log correlation. + +Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs: + + SGLANG_BATCH_SPAN=0 histograms only (no spans) + SGLANG_BATCH_SPAN_MIN_MS=50 only emit spans for batches slower than this + SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0) + SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span + +Cost model (measured on our 4xDP4 / 4-GPU instances): + * histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU. + * span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s. + Everything is behind a single module-level bool, so when disabled the hot + path pays one ``if not _ENABLED: return``. + +NOTE: sglang runs prometheus_client in multiprocess mode, which does not +carry exemplars, so the span trace id is returned to the caller and appended +to the batch log line instead (see ``metrics_reporter``). +""" + +from __future__ import annotations + +import os +import random +import time +from typing import Any, Iterable, Optional + +_ENABLED = os.getenv("SGLANG_BATCH_TELEMETRY", "0").strip().lower() not in ( + "", + "0", + "false", + "no", + "off", +) +_SPAN_ENABLED = _ENABLED and os.getenv("SGLANG_BATCH_SPAN", "1").strip().lower() not in ( + "0", + "false", + "no", + "off", +) +_SPAN_MIN_S = float(os.getenv("SGLANG_BATCH_SPAN_MIN_MS", "0")) / 1000.0 +_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_SPAN_SAMPLE", "1.0")) +_MAX_LINKS = int(os.getenv("SGLANG_BATCH_SPAN_MAX_LINKS", "32")) + +_HISTOGRAMS = None +_CHILDREN: dict[tuple[str, str, int], Any] = {} +_TRACER = None + + +def enabled() -> bool: + return _ENABLED + + +def _ensure_histograms(): + global _HISTOGRAMS + if _HISTOGRAMS is None: + from prometheus_client import Histogram + + labels = ["phase", "dp_rank"] + _HISTOGRAMS = { + "batch_new_tokens": Histogram( + "sglang:batch_new_tokens", + "New (uncached) prompt tokens processed by one scheduler batch", + labels, + buckets=(128, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536), + ), + "batch_cached_tokens": Histogram( + "sglang:batch_cached_tokens", + "Cached prompt tokens carried by one scheduler batch", + labels, + buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576), + ), + "batch_duration_seconds": Histogram( + "sglang:batch_duration_seconds", + "Wall time spent in one scheduler batch", + labels, + buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0), + ), + "batch_queue_depth": Histogram( + "sglang:batch_queue_depth", + "Waiting-queue depth observed by one scheduler batch", + labels, + buckets=(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512), + ), + "batch_pending_tokens": Histogram( + "sglang:batch_pending_tokens", + "Prefill backlog (pending tokens) observed by one scheduler batch", + labels, + buckets=(0, 1024, 8192, 32768, 131072, 524288, 1048576), + ), + "batch_n_reqs": Histogram( + "sglang:batch_n_reqs", + "Requests carried by one scheduler batch", + labels, + buckets=(1, 2, 4, 8, 16, 32, 64, 128, 256), + ), + } + return _HISTOGRAMS + + +def _child(name: str, phase: str, dp_rank: int): + key = (name, phase, dp_rank) + child = _CHILDREN.get(key) + if child is None: + child = _ensure_histograms()[name].labels(phase=phase, dp_rank=str(dp_rank)) + _CHILDREN[key] = child + return child + + +def _request_span_context(req: Any): + """SpanContext of a request, from either a live span or a serialized copy. + + Scheduler processes receive a *copy* of the request trace context where + ``root_span`` is None (see ``trace_async.copy_for_thread``) and only + ``root_span_context`` (an OTel ``Context``) survives. Handle both. + """ + trace_ctx = getattr(getattr(req, "time_stats", None), "trace_ctx", None) + if trace_ctx is None: + return None + try: + root_span = getattr(trace_ctx, "root_span", None) + if root_span is not None: + span_ctx = root_span.get_span_context() + if span_ctx.is_valid: + return span_ctx + root_ctx = getattr(trace_ctx, "root_span_context", None) + if root_ctx is not None: + from opentelemetry import trace as otel_trace + + span_ctx = otel_trace.get_current_span(root_ctx).get_span_context() + if span_ctx.is_valid: + return span_ctx + except Exception: + return None + return None + + +def _emit_span( + phase: str, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]], +) -> Optional[str]: + """Create one batch span with links to the request traces. Returns trace id.""" + if not _SPAN_ENABLED: + return None + if _SPAN_MIN_S > 0 and duration_s < _SPAN_MIN_S: + return None + if _SPAN_SAMPLE < 1.0 and random.random() > _SPAN_SAMPLE: + return None + global _TRACER + try: + from opentelemetry import trace as otel_trace + from opentelemetry.trace import Link + + if _TRACER is None: + _TRACER = otel_trace.get_tracer("sglang.batch") + links = [] + if reqs: + for req in reqs: + if len(links) >= _MAX_LINKS: + break + ctx = _request_span_context(req) + if ctx is not None: + links.append(Link(ctx)) + end_ns = time.time_ns() + start_ns = end_ns - int(duration_s * 1e9) + span = _TRACER.start_span( + f"scheduler.batch.{phase}", + start_time=start_ns, + attributes={ + "sglang.batch_id": f"{dp_rank}:{iter_no}", + "sglang.dp_rank": int(dp_rank), + "sglang.phase": phase, + "sglang.n_reqs": int(n_reqs), + "sglang.n_new_tokens": int(n_new), + "sglang.n_cached_tokens": int(n_cached), + "sglang.queue_depth": int(queue_depth), + "sglang.pending_tokens": int(pending_tokens), + "sglang.batch_duration_s": float(duration_s), + }, + links=links, + ) + span.end(end_time=end_ns) + span_ctx = span.get_span_context() + if not span_ctx.is_valid: + # No TracerProvider configured (tracing disabled) -> NoOp span. + return None + return format(span_ctx.trace_id, "032x") + except Exception: + return None + + +def _observe( + phase: str, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]], +) -> Optional[str]: + if not _ENABLED: + return None + try: + trace_id = _emit_span( + phase, + dp_rank, + iter_no, + n_reqs, + n_new, + n_cached, + queue_depth, + pending_tokens, + duration_s, + reqs, + ) + _child("batch_new_tokens", phase, dp_rank).observe(n_new) + _child("batch_cached_tokens", phase, dp_rank).observe(n_cached) + _child("batch_duration_seconds", phase, dp_rank).observe(duration_s) + _child("batch_queue_depth", phase, dp_rank).observe(queue_depth) + _child("batch_pending_tokens", phase, dp_rank).observe(pending_tokens) + _child("batch_n_reqs", phase, dp_rank).observe(n_reqs) + return trace_id + except Exception: + # Telemetry must never break the scheduler loop. + return None + + +def observe_prefill( + *, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]] = None, +) -> Optional[str]: + return _observe( + "prefill", + dp_rank, + iter_no, + n_reqs, + n_new, + n_cached, + queue_depth, + pending_tokens, + duration_s, + reqs, + ) + + +def observe_decode( + *, + dp_rank: int, + iter_no: int, + n_reqs: int, + n_new: int, + n_cached: int, + queue_depth: int, + pending_tokens: int, + duration_s: float, + reqs: Optional[Iterable[Any]] = None, +) -> Optional[str]: + return _observe( + "decode", + dp_rank, + iter_no, + n_reqs, + n_new, + n_cached, + queue_depth, + pending_tokens, + duration_s, + reqs, + ) + +_DP_HISTS = None + + +def _ensure_dp_histograms(): + global _DP_HISTS + if _DP_HISTS is None: + from prometheus_client import Histogram + + _DP_HISTS = { + "dp_step_tokens": Histogram( + "sglang:dp_step_tokens", + "Per-rank token count in one DP-attention step", + ["dp_rank", "phase"], + buckets=(0, 256, 1024, 2048, 4096, 8192, 16384, 32768), + ), + "dp_step_padding_ratio": Histogram( + "sglang:dp_step_padding_ratio", + "1 - mean/max tokens across active DP ranks in one step (padding waste)", + ["phase", "n_active"], + buckets=(0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8), + ), + "dp_step_spread": Histogram( + "sglang:dp_step_spread", + "(max-min)/mean tokens across active DP ranks in one step", + ["phase", "n_active"], + buckets=(0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0), + ), + } + return _DP_HISTS + + +_DP_IDLE_MODE = 4 # ForwardMode.IDLE + + +def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = "unknown") -> None: + """Record per-step shape alignment across *active* DP ranks. + + ``tokens``/``modes``/``health`` are the per-rank vectors from the DP + all-gather (info tensor columns 0, 5, 9). IDLE ranks and health-check + steps are excluded so the padding ratio reflects real batch-shape + misalignment rather than idle-slot padding. + """ + if not _ENABLED: + return + try: + if health and any(bool(h) for h in health): + return + sel = [] + for i, t in enumerate(tokens): + if t is None: + continue + t = int(t) + if t <= 1: + continue + if modes is not None and i < len(modes) and int(modes[i]) == _DP_IDLE_MODE: + continue + sel.append(t) + if len(sel) < 2: + return + hist = _ensure_dp_histograms() + mx = max(sel) + mean = sum(sel) / len(sel) + n_active = str(len(sel)) + key = ("dp_step_tokens", phase, dp_rank) + child = _CHILDREN.get(key) + if child is None: + child = hist["dp_step_tokens"].labels(dp_rank=str(dp_rank), phase=phase) + _CHILDREN[key] = child + child.observe(int(tokens[dp_rank]) if 0 <= dp_rank < len(tokens) else int(mean)) + if mx > 0: + hist["dp_step_padding_ratio"].labels(phase=phase, n_active=n_active).observe( + 1.0 - mean / mx + ) + if mean > 0: + hist["dp_step_spread"].labels(phase=phase, n_active=n_active).observe( + (mx - min(sel)) / mean + ) + except Exception: + pass