observability: per-rank request counter for P/D RPS

The frontend sglang:num_requests_total has no dp_rank label (verified from the
series registry), so per-rank RPS was not derivable from any existing metric.

Add sglang:batch_requests_total{phase,dp_rank}, incremented by the scheduler
process that actually completed the request:

  phase=prefill  prefill compute finished and the request entered the PD
                 transfer queue (not a guarantee that the transfer succeeded)
  phase=decode   request finished decoding

Only the canonical scheduler rank reports (attn_tp_rank==0, attn_cp_rank==0,
last PP stage) — every TP/CP rank runs a scheduler and sees the same request,
so an ungated counter would be multiplied by TP*CP in the multiprocess
registry. Both hooks count each request exactly once: the decode path counts
after _handle_finish_state_updated_req and on the sampling-mask abort branch,
skipping rows that finished on an earlier step; the prefill path counts after
the request enters the disagg inflight queue (aborted/retried requests never
reach it). The per-request finished() check is skipped entirely when
SGLANG_BATCH_TELEMETRY is off.
This commit is contained in:
2026-11-12 10:30:56 +08:00
parent 17b922c84f
commit 7f42a0806a
4 changed files with 199 additions and 0 deletions
@@ -1273,6 +1273,7 @@ class SchedulerDisaggregationPrefillMixin:
self.checkpoint_disagg_prefill(req)
self.disagg_prefill_inflight_queue.append(req)
batch_telemetry.note_request_completed("prefill")
if self.spec_algorithm.is_eagle() and draft_input is not None:
req.output_topk_p = draft_input.topk_p[i]
req.output_topk_index = draft_input.topk_index[i]
@@ -29,6 +29,7 @@ from sglang.srt.managers.schedule_batch import (
ScheduleBatch,
mamba_lazy_spec_in_window,
)
from sglang.srt.observability import batch_telemetry
from sglang.srt.mem_cache.common import (
checkpoint_kv_cache,
release_kv_cache,
@@ -995,6 +996,10 @@ class SchedulerBatchResultProcessor:
if batch.spec_algorithm.is_none() and logits_output is not None:
newly_finished_beam_groups = self.beam_coordinator.commit_decode(batch)
# Keep the hot loop free when telemetry is off: read the gate once and
# skip the per-request finished() check entirely.
track_completions = batch_telemetry.enabled()
finished_reqs = 0
for i, req in enumerate(batch.reqs):
req: Req
@@ -1009,6 +1014,8 @@ class SchedulerBatchResultProcessor:
self._handle_finish_state_updated_req(
req, batch, result, i, logits_output
)
if track_completions and req.finished():
finished_reqs += 1
continue
if (self.enable_overlap or self.enable_overlap_mlx) and (
@@ -1041,10 +1048,16 @@ class SchedulerBatchResultProcessor:
req.update_finish_state(new_accept_len)
if sampling_mask_finish_reason is not None:
# This path finishes the request too (to_finish ->
# finished_reason); count it before the continue.
self._handle_sampling_mask_abort(req)
if track_completions and req.finished():
finished_reqs += 1
continue
self._handle_finish_state_updated_req(req, batch, result, i, logits_output)
if track_completions and req.finished():
finished_reqs += 1
if req.return_logprob:
self._apply_decode_logprobs(
@@ -1081,6 +1094,9 @@ class SchedulerBatchResultProcessor:
self._accept_grammar_tokens(req, next_token_id)
req.grammar.finished = req.finished()
if finished_reqs:
batch_telemetry.note_request_completed("decode", finished_reqs)
if auxiliary_output is not None:
self.consume_auxiliary_output(
batch,
@@ -75,6 +75,85 @@ def enabled() -> bool:
return _ENABLED
_REQUEST_COUNTER = None
_DP_RANK = None
_COMPLETION_RANK = None
def _should_report_completion() -> bool:
"""Whether this scheduler is the canonical reporter of request completions.
Every TP/CP rank runs a scheduler and observes the same requests, and a PP
stage only sees its own microbatches, so an ungated counter would be
multiplied across replicas once the multiprocess registry aggregates them.
Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same
shape the existing scheduler metrics use for rank-unique series.
"""
global _COMPLETION_RANK
if _COMPLETION_RANK is None:
try:
from sglang.srt.runtime_context import get_parallel
parallel = get_parallel()
_COMPLETION_RANK = bool(
parallel.attn_tp_rank == 0
and parallel.attn_cp_rank == 0
and parallel.pp_group.is_last_rank
)
except Exception:
# Unknown topology (e.g. a bare unit-test process): keep counting.
_COMPLETION_RANK = True
return _COMPLETION_RANK
def _current_dp_rank() -> int:
"""This process's attention-DP rank (0 for non-DP engines)."""
global _DP_RANK
if _DP_RANK is None:
try:
from sglang.srt.runtime_context import get_parallel
_DP_RANK = int(get_parallel().attn_dp_rank or 0)
except Exception:
_DP_RANK = 0
return _DP_RANK
def _ensure_request_counter():
global _REQUEST_COUNTER
if _REQUEST_COUNTER is None:
from prometheus_client import Counter
_REQUEST_COUNTER = Counter(
"sglang:batch_requests_total",
"Requests completed by the canonical scheduler rank, by phase. "
"phase=prefill: prefill compute finished and the request entered the "
"PD transfer queue (not a guarantee that KV transfer succeeded). "
"phase=decode: request finished decoding.",
["phase", "dp_rank"],
)
return _REQUEST_COUNTER
def note_request_completed(phase: str, n: int = 1) -> None:
"""Count finished requests per DP rank.
This is the per-rank RPS source: the frontend ``num_requests_total`` has no
rank label, while the scheduler knows exactly which DP shard completed the
request. Cheap (one counter increment per finished request) and gated by
SGLANG_BATCH_TELEMETRY like the other batch metrics.
"""
if not _ENABLED or n <= 0 or not _should_report_completion():
return
try:
_ensure_request_counter().labels(
phase=phase, dp_rank=str(_current_dp_rank())
).inc(int(n))
except Exception:
# Telemetry must never fail a request.
pass
def _ensure_histograms():
global _HISTOGRAMS
if _HISTOGRAMS is None: