observability: per-rank request counter for P/D RPS

The frontend sglang:num_requests_total has no dp_rank label (verified from the
series registry), so per-rank RPS was not derivable from any existing metric.

Add sglang:batch_requests_total{phase,dp_rank}, incremented by the scheduler
process that actually completed the request:

  phase=prefill  prefill compute finished and the request entered the PD
                 transfer queue (not a guarantee that the transfer succeeded)
  phase=decode   request finished decoding

Only the canonical scheduler rank reports (attn_tp_rank==0, attn_cp_rank==0,
last PP stage) — every TP/CP rank runs a scheduler and sees the same request,
so an ungated counter would be multiplied by TP*CP in the multiprocess
registry. Both hooks count each request exactly once: the decode path counts
after _handle_finish_state_updated_req and on the sampling-mask abort branch,
skipping rows that finished on an earlier step; the prefill path counts after
the request enters the disagg inflight queue (aborted/retried requests never
reach it). The per-request finished() check is skipped entirely when
SGLANG_BATCH_TELEMETRY is off.
This commit is contained in:
2026-11-12 10:30:56 +08:00
parent 17b922c84f
commit 7f42a0806a
4 changed files with 199 additions and 0 deletions
@@ -1273,6 +1273,7 @@ class SchedulerDisaggregationPrefillMixin:
self.checkpoint_disagg_prefill(req)
self.disagg_prefill_inflight_queue.append(req)
batch_telemetry.note_request_completed("prefill")
if self.spec_algorithm.is_eagle() and draft_input is not None:
req.output_topk_p = draft_input.topk_p[i]
req.output_topk_index = draft_input.topk_index[i]
@@ -29,6 +29,7 @@ from sglang.srt.managers.schedule_batch import (
ScheduleBatch,
mamba_lazy_spec_in_window,
)
from sglang.srt.observability import batch_telemetry
from sglang.srt.mem_cache.common import (
checkpoint_kv_cache,
release_kv_cache,
@@ -995,6 +996,10 @@ class SchedulerBatchResultProcessor:
if batch.spec_algorithm.is_none() and logits_output is not None:
newly_finished_beam_groups = self.beam_coordinator.commit_decode(batch)
# Keep the hot loop free when telemetry is off: read the gate once and
# skip the per-request finished() check entirely.
track_completions = batch_telemetry.enabled()
finished_reqs = 0
for i, req in enumerate(batch.reqs):
req: Req
@@ -1009,6 +1014,8 @@ class SchedulerBatchResultProcessor:
self._handle_finish_state_updated_req(
req, batch, result, i, logits_output
)
if track_completions and req.finished():
finished_reqs += 1
continue
if (self.enable_overlap or self.enable_overlap_mlx) and (
@@ -1041,10 +1048,16 @@ class SchedulerBatchResultProcessor:
req.update_finish_state(new_accept_len)
if sampling_mask_finish_reason is not None:
# This path finishes the request too (to_finish ->
# finished_reason); count it before the continue.
self._handle_sampling_mask_abort(req)
if track_completions and req.finished():
finished_reqs += 1
continue
self._handle_finish_state_updated_req(req, batch, result, i, logits_output)
if track_completions and req.finished():
finished_reqs += 1
if req.return_logprob:
self._apply_decode_logprobs(
@@ -1081,6 +1094,9 @@ class SchedulerBatchResultProcessor:
self._accept_grammar_tokens(req, next_token_id)
req.grammar.finished = req.finished()
if finished_reqs:
batch_telemetry.note_request_completed("decode", finished_reqs)
if auxiliary_output is not None:
self.consume_auxiliary_output(
batch,
@@ -75,6 +75,85 @@ def enabled() -> bool:
return _ENABLED
_REQUEST_COUNTER = None
_DP_RANK = None
_COMPLETION_RANK = None
def _should_report_completion() -> bool:
"""Whether this scheduler is the canonical reporter of request completions.
Every TP/CP rank runs a scheduler and observes the same requests, and a PP
stage only sees its own microbatches, so an ungated counter would be
multiplied across replicas once the multiprocess registry aggregates them.
Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same
shape the existing scheduler metrics use for rank-unique series.
"""
global _COMPLETION_RANK
if _COMPLETION_RANK is None:
try:
from sglang.srt.runtime_context import get_parallel
parallel = get_parallel()
_COMPLETION_RANK = bool(
parallel.attn_tp_rank == 0
and parallel.attn_cp_rank == 0
and parallel.pp_group.is_last_rank
)
except Exception:
# Unknown topology (e.g. a bare unit-test process): keep counting.
_COMPLETION_RANK = True
return _COMPLETION_RANK
def _current_dp_rank() -> int:
"""This process's attention-DP rank (0 for non-DP engines)."""
global _DP_RANK
if _DP_RANK is None:
try:
from sglang.srt.runtime_context import get_parallel
_DP_RANK = int(get_parallel().attn_dp_rank or 0)
except Exception:
_DP_RANK = 0
return _DP_RANK
def _ensure_request_counter():
global _REQUEST_COUNTER
if _REQUEST_COUNTER is None:
from prometheus_client import Counter
_REQUEST_COUNTER = Counter(
"sglang:batch_requests_total",
"Requests completed by the canonical scheduler rank, by phase. "
"phase=prefill: prefill compute finished and the request entered the "
"PD transfer queue (not a guarantee that KV transfer succeeded). "
"phase=decode: request finished decoding.",
["phase", "dp_rank"],
)
return _REQUEST_COUNTER
def note_request_completed(phase: str, n: int = 1) -> None:
"""Count finished requests per DP rank.
This is the per-rank RPS source: the frontend ``num_requests_total`` has no
rank label, while the scheduler knows exactly which DP shard completed the
request. Cheap (one counter increment per finished request) and gated by
SGLANG_BATCH_TELEMETRY like the other batch metrics.
"""
if not _ENABLED or n <= 0 or not _should_report_completion():
return
try:
_ensure_request_counter().labels(
phase=phase, dp_rank=str(_current_dp_rank())
).inc(int(n))
except Exception:
# Telemetry must never fail a request.
pass
def _ensure_histograms():
global _HISTOGRAMS
if _HISTOGRAMS is None:
@@ -0,0 +1,103 @@
"""Per-rank request counter used for P/D per-rank RPS.
``sglang:batch_requests_total{phase,dp_rank}`` is incremented by the scheduler
process (which knows its DP shard), because the frontend ``num_requests_total``
carries no rank label.
Skipped when prometheus_client is not installed.
"""
import importlib.util
import os
import pathlib
import unittest
from unittest import mock
try: # pragma: no cover - environment dependent
import prometheus_client # noqa: F401
_HAS_PROM = True
except Exception: # pragma: no cover
_HAS_PROM = False
def _module_path():
p = (
pathlib.Path(__file__).resolve().parents[4]
/ "python/sglang/srt/observability/batch_telemetry.py"
)
if p.exists():
return p
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
return pathlib.Path(spec.origin)
def _load(**env):
base = {
"SGLANG_BATCH_TELEMETRY": "1",
"SGLANG_BATCH_STAGE_TIMING": "1",
"SGLANG_TRACE_LOOP_SPAN": "1",
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
"SGLANG_TRACE_LOOP_DETAIL": "1",
}
base.update(env)
with mock.patch.dict(os.environ, base):
spec = importlib.util.spec_from_file_location(
"batch_telemetry_rank_rps_under_test", _module_path()
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
@unittest.skipUnless(_HAS_PROM, "requires prometheus_client")
class RankRequestCounterTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
# One module instance: prometheus_client forbids registering the same
# metric name twice in the default registry.
cls.mod = _load()
cls.mod._current_dp_rank = lambda: 2
# These tests assume the canonical (tp0/cp0/last-pp) rank; the gate
# itself is exercised by test_non_canonical_rank_does_not_report.
cls.mod._COMPLETION_RANK = True
def _value(self, phase, rank):
counter = self.mod._ensure_request_counter()
return counter.labels(phase=phase, dp_rank=str(rank))._value.get()
def test_counts_per_phase_and_rank(self):
before_2 = self._value("prefill", 2)
before_0 = self._value("prefill", 0)
self.mod.note_request_completed("prefill", 3)
self.mod.note_request_completed("prefill")
self.assertEqual(self._value("prefill", 2), before_2 + 4)
self.assertEqual(self._value("prefill", 0), before_0)
self.assertEqual(self._value("decode", 2), 0)
def test_non_canonical_rank_does_not_report(self):
# A TP/CP replica sees the same request; only the canonical scheduler
# may increment, otherwise multiprocess aggregation multiplies RPS.
before = self._value("prefill", 2)
saved = self.mod._COMPLETION_RANK
self.mod._COMPLETION_RANK = False
try:
self.mod.note_request_completed("prefill", 7)
finally:
self.mod._COMPLETION_RANK = saved
self.assertEqual(self._value("prefill", 2), before)
def test_ignores_non_positive_and_disabled(self):
before = self._value("decode", 2)
self.mod.note_request_completed("decode", 0)
self.assertEqual(self._value("decode", 2), before)
saved = self.mod._ENABLED
self.mod._ENABLED = False
try:
self.mod.note_request_completed("decode", 5)
finally:
self.mod._ENABLED = saved
self.assertEqual(self._value("decode", 2), before)
if __name__ == "__main__":
unittest.main()