observability: per-rank request counter for P/D RPS
The frontend sglang:num_requests_total has no dp_rank label (verified from the
series registry), so per-rank RPS was not derivable from any existing metric.
Add sglang:batch_requests_total{phase,dp_rank}, incremented by the scheduler
process that actually completed the request:
phase=prefill prefill compute finished and the request entered the PD
transfer queue (not a guarantee that the transfer succeeded)
phase=decode request finished decoding
Only the canonical scheduler rank reports (attn_tp_rank==0, attn_cp_rank==0,
last PP stage) — every TP/CP rank runs a scheduler and sees the same request,
so an ungated counter would be multiplied by TP*CP in the multiprocess
registry. Both hooks count each request exactly once: the decode path counts
after _handle_finish_state_updated_req and on the sampling-mask abort branch,
skipping rows that finished on an earlier step; the prefill path counts after
the request enters the disagg inflight queue (aborted/retried requests never
reach it). The per-request finished() check is skipped entirely when
SGLANG_BATCH_TELEMETRY is off.
This commit is contained in:
@@ -1273,6 +1273,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
batch_telemetry.note_request_completed("prefill")
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
req.output_topk_index = draft_input.topk_index[i]
|
||||
|
||||
@@ -29,6 +29,7 @@ from sglang.srt.managers.schedule_batch import (
|
||||
ScheduleBatch,
|
||||
mamba_lazy_spec_in_window,
|
||||
)
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.mem_cache.common import (
|
||||
checkpoint_kv_cache,
|
||||
release_kv_cache,
|
||||
@@ -995,6 +996,10 @@ class SchedulerBatchResultProcessor:
|
||||
if batch.spec_algorithm.is_none() and logits_output is not None:
|
||||
newly_finished_beam_groups = self.beam_coordinator.commit_decode(batch)
|
||||
|
||||
# Keep the hot loop free when telemetry is off: read the gate once and
|
||||
# skip the per-request finished() check entirely.
|
||||
track_completions = batch_telemetry.enabled()
|
||||
finished_reqs = 0
|
||||
for i, req in enumerate(batch.reqs):
|
||||
req: Req
|
||||
|
||||
@@ -1009,6 +1014,8 @@ class SchedulerBatchResultProcessor:
|
||||
self._handle_finish_state_updated_req(
|
||||
req, batch, result, i, logits_output
|
||||
)
|
||||
if track_completions and req.finished():
|
||||
finished_reqs += 1
|
||||
continue
|
||||
|
||||
if (self.enable_overlap or self.enable_overlap_mlx) and (
|
||||
@@ -1041,10 +1048,16 @@ class SchedulerBatchResultProcessor:
|
||||
req.update_finish_state(new_accept_len)
|
||||
|
||||
if sampling_mask_finish_reason is not None:
|
||||
# This path finishes the request too (to_finish ->
|
||||
# finished_reason); count it before the continue.
|
||||
self._handle_sampling_mask_abort(req)
|
||||
if track_completions and req.finished():
|
||||
finished_reqs += 1
|
||||
continue
|
||||
|
||||
self._handle_finish_state_updated_req(req, batch, result, i, logits_output)
|
||||
if track_completions and req.finished():
|
||||
finished_reqs += 1
|
||||
|
||||
if req.return_logprob:
|
||||
self._apply_decode_logprobs(
|
||||
@@ -1081,6 +1094,9 @@ class SchedulerBatchResultProcessor:
|
||||
self._accept_grammar_tokens(req, next_token_id)
|
||||
req.grammar.finished = req.finished()
|
||||
|
||||
if finished_reqs:
|
||||
batch_telemetry.note_request_completed("decode", finished_reqs)
|
||||
|
||||
if auxiliary_output is not None:
|
||||
self.consume_auxiliary_output(
|
||||
batch,
|
||||
|
||||
@@ -75,6 +75,85 @@ def enabled() -> bool:
|
||||
return _ENABLED
|
||||
|
||||
|
||||
_REQUEST_COUNTER = None
|
||||
_DP_RANK = None
|
||||
_COMPLETION_RANK = None
|
||||
|
||||
|
||||
def _should_report_completion() -> bool:
|
||||
"""Whether this scheduler is the canonical reporter of request completions.
|
||||
|
||||
Every TP/CP rank runs a scheduler and observes the same requests, and a PP
|
||||
stage only sees its own microbatches, so an ungated counter would be
|
||||
multiplied across replicas once the multiprocess registry aggregates them.
|
||||
Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same
|
||||
shape the existing scheduler metrics use for rank-unique series.
|
||||
"""
|
||||
global _COMPLETION_RANK
|
||||
if _COMPLETION_RANK is None:
|
||||
try:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
parallel = get_parallel()
|
||||
_COMPLETION_RANK = bool(
|
||||
parallel.attn_tp_rank == 0
|
||||
and parallel.attn_cp_rank == 0
|
||||
and parallel.pp_group.is_last_rank
|
||||
)
|
||||
except Exception:
|
||||
# Unknown topology (e.g. a bare unit-test process): keep counting.
|
||||
_COMPLETION_RANK = True
|
||||
return _COMPLETION_RANK
|
||||
|
||||
|
||||
def _current_dp_rank() -> int:
|
||||
"""This process's attention-DP rank (0 for non-DP engines)."""
|
||||
global _DP_RANK
|
||||
if _DP_RANK is None:
|
||||
try:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
_DP_RANK = int(get_parallel().attn_dp_rank or 0)
|
||||
except Exception:
|
||||
_DP_RANK = 0
|
||||
return _DP_RANK
|
||||
|
||||
|
||||
def _ensure_request_counter():
|
||||
global _REQUEST_COUNTER
|
||||
if _REQUEST_COUNTER is None:
|
||||
from prometheus_client import Counter
|
||||
|
||||
_REQUEST_COUNTER = Counter(
|
||||
"sglang:batch_requests_total",
|
||||
"Requests completed by the canonical scheduler rank, by phase. "
|
||||
"phase=prefill: prefill compute finished and the request entered the "
|
||||
"PD transfer queue (not a guarantee that KV transfer succeeded). "
|
||||
"phase=decode: request finished decoding.",
|
||||
["phase", "dp_rank"],
|
||||
)
|
||||
return _REQUEST_COUNTER
|
||||
|
||||
|
||||
def note_request_completed(phase: str, n: int = 1) -> None:
|
||||
"""Count finished requests per DP rank.
|
||||
|
||||
This is the per-rank RPS source: the frontend ``num_requests_total`` has no
|
||||
rank label, while the scheduler knows exactly which DP shard completed the
|
||||
request. Cheap (one counter increment per finished request) and gated by
|
||||
SGLANG_BATCH_TELEMETRY like the other batch metrics.
|
||||
"""
|
||||
if not _ENABLED or n <= 0 or not _should_report_completion():
|
||||
return
|
||||
try:
|
||||
_ensure_request_counter().labels(
|
||||
phase=phase, dp_rank=str(_current_dp_rank())
|
||||
).inc(int(n))
|
||||
except Exception:
|
||||
# Telemetry must never fail a request.
|
||||
pass
|
||||
|
||||
|
||||
def _ensure_histograms():
|
||||
global _HISTOGRAMS
|
||||
if _HISTOGRAMS is None:
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Per-rank request counter used for P/D per-rank RPS.
|
||||
|
||||
``sglang:batch_requests_total{phase,dp_rank}`` is incremented by the scheduler
|
||||
process (which knows its DP shard), because the frontend ``num_requests_total``
|
||||
carries no rank label.
|
||||
|
||||
Skipped when prometheus_client is not installed.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
try: # pragma: no cover - environment dependent
|
||||
import prometheus_client # noqa: F401
|
||||
|
||||
_HAS_PROM = True
|
||||
except Exception: # pragma: no cover
|
||||
_HAS_PROM = False
|
||||
|
||||
|
||||
def _module_path():
|
||||
p = (
|
||||
pathlib.Path(__file__).resolve().parents[4]
|
||||
/ "python/sglang/srt/observability/batch_telemetry.py"
|
||||
)
|
||||
if p.exists():
|
||||
return p
|
||||
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
|
||||
return pathlib.Path(spec.origin)
|
||||
|
||||
|
||||
def _load(**env):
|
||||
base = {
|
||||
"SGLANG_BATCH_TELEMETRY": "1",
|
||||
"SGLANG_BATCH_STAGE_TIMING": "1",
|
||||
"SGLANG_TRACE_LOOP_SPAN": "1",
|
||||
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
|
||||
"SGLANG_TRACE_LOOP_DETAIL": "1",
|
||||
}
|
||||
base.update(env)
|
||||
with mock.patch.dict(os.environ, base):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"batch_telemetry_rank_rps_under_test", _module_path()
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
@unittest.skipUnless(_HAS_PROM, "requires prometheus_client")
|
||||
class RankRequestCounterTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# One module instance: prometheus_client forbids registering the same
|
||||
# metric name twice in the default registry.
|
||||
cls.mod = _load()
|
||||
cls.mod._current_dp_rank = lambda: 2
|
||||
# These tests assume the canonical (tp0/cp0/last-pp) rank; the gate
|
||||
# itself is exercised by test_non_canonical_rank_does_not_report.
|
||||
cls.mod._COMPLETION_RANK = True
|
||||
|
||||
def _value(self, phase, rank):
|
||||
counter = self.mod._ensure_request_counter()
|
||||
return counter.labels(phase=phase, dp_rank=str(rank))._value.get()
|
||||
|
||||
def test_counts_per_phase_and_rank(self):
|
||||
before_2 = self._value("prefill", 2)
|
||||
before_0 = self._value("prefill", 0)
|
||||
self.mod.note_request_completed("prefill", 3)
|
||||
self.mod.note_request_completed("prefill")
|
||||
self.assertEqual(self._value("prefill", 2), before_2 + 4)
|
||||
self.assertEqual(self._value("prefill", 0), before_0)
|
||||
self.assertEqual(self._value("decode", 2), 0)
|
||||
|
||||
def test_non_canonical_rank_does_not_report(self):
|
||||
# A TP/CP replica sees the same request; only the canonical scheduler
|
||||
# may increment, otherwise multiprocess aggregation multiplies RPS.
|
||||
before = self._value("prefill", 2)
|
||||
saved = self.mod._COMPLETION_RANK
|
||||
self.mod._COMPLETION_RANK = False
|
||||
try:
|
||||
self.mod.note_request_completed("prefill", 7)
|
||||
finally:
|
||||
self.mod._COMPLETION_RANK = saved
|
||||
self.assertEqual(self._value("prefill", 2), before)
|
||||
|
||||
def test_ignores_non_positive_and_disabled(self):
|
||||
before = self._value("decode", 2)
|
||||
self.mod.note_request_completed("decode", 0)
|
||||
self.assertEqual(self._value("decode", 2), before)
|
||||
saved = self.mod._ENABLED
|
||||
self.mod._ENABLED = False
|
||||
try:
|
||||
self.mod.note_request_completed("decode", 5)
|
||||
finally:
|
||||
self.mod._ENABLED = saved
|
||||
self.assertEqual(self._value("decode", 2), before)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user