observability: per-rank request counter for P/D RPS
The frontend sglang:num_requests_total has no dp_rank label (verified from the
series registry), so per-rank RPS was not derivable from any existing metric.
Add sglang:batch_requests_total{phase,dp_rank}, incremented by the scheduler
process that actually completed the request:
phase=prefill prefill compute finished and the request entered the PD
transfer queue (not a guarantee that the transfer succeeded)
phase=decode request finished decoding
Only the canonical scheduler rank reports (attn_tp_rank==0, attn_cp_rank==0,
last PP stage) — every TP/CP rank runs a scheduler and sees the same request,
so an ungated counter would be multiplied by TP*CP in the multiprocess
registry. Both hooks count each request exactly once: the decode path counts
after _handle_finish_state_updated_req and on the sampling-mask abort branch,
skipping rows that finished on an earlier step; the prefill path counts after
the request enters the disagg inflight queue (aborted/retried requests never
reach it). The per-request finished() check is skipped entirely when
SGLANG_BATCH_TELEMETRY is off.
This commit is contained in:
@@ -1273,6 +1273,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
batch_telemetry.note_request_completed("prefill")
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
req.output_topk_index = draft_input.topk_index[i]
|
||||
|
||||
@@ -29,6 +29,7 @@ from sglang.srt.managers.schedule_batch import (
|
||||
ScheduleBatch,
|
||||
mamba_lazy_spec_in_window,
|
||||
)
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.mem_cache.common import (
|
||||
checkpoint_kv_cache,
|
||||
release_kv_cache,
|
||||
@@ -995,6 +996,10 @@ class SchedulerBatchResultProcessor:
|
||||
if batch.spec_algorithm.is_none() and logits_output is not None:
|
||||
newly_finished_beam_groups = self.beam_coordinator.commit_decode(batch)
|
||||
|
||||
# Keep the hot loop free when telemetry is off: read the gate once and
|
||||
# skip the per-request finished() check entirely.
|
||||
track_completions = batch_telemetry.enabled()
|
||||
finished_reqs = 0
|
||||
for i, req in enumerate(batch.reqs):
|
||||
req: Req
|
||||
|
||||
@@ -1009,6 +1014,8 @@ class SchedulerBatchResultProcessor:
|
||||
self._handle_finish_state_updated_req(
|
||||
req, batch, result, i, logits_output
|
||||
)
|
||||
if track_completions and req.finished():
|
||||
finished_reqs += 1
|
||||
continue
|
||||
|
||||
if (self.enable_overlap or self.enable_overlap_mlx) and (
|
||||
@@ -1041,10 +1048,16 @@ class SchedulerBatchResultProcessor:
|
||||
req.update_finish_state(new_accept_len)
|
||||
|
||||
if sampling_mask_finish_reason is not None:
|
||||
# This path finishes the request too (to_finish ->
|
||||
# finished_reason); count it before the continue.
|
||||
self._handle_sampling_mask_abort(req)
|
||||
if track_completions and req.finished():
|
||||
finished_reqs += 1
|
||||
continue
|
||||
|
||||
self._handle_finish_state_updated_req(req, batch, result, i, logits_output)
|
||||
if track_completions and req.finished():
|
||||
finished_reqs += 1
|
||||
|
||||
if req.return_logprob:
|
||||
self._apply_decode_logprobs(
|
||||
@@ -1081,6 +1094,9 @@ class SchedulerBatchResultProcessor:
|
||||
self._accept_grammar_tokens(req, next_token_id)
|
||||
req.grammar.finished = req.finished()
|
||||
|
||||
if finished_reqs:
|
||||
batch_telemetry.note_request_completed("decode", finished_reqs)
|
||||
|
||||
if auxiliary_output is not None:
|
||||
self.consume_auxiliary_output(
|
||||
batch,
|
||||
|
||||
@@ -75,6 +75,85 @@ def enabled() -> bool:
|
||||
return _ENABLED
|
||||
|
||||
|
||||
_REQUEST_COUNTER = None
|
||||
_DP_RANK = None
|
||||
_COMPLETION_RANK = None
|
||||
|
||||
|
||||
def _should_report_completion() -> bool:
|
||||
"""Whether this scheduler is the canonical reporter of request completions.
|
||||
|
||||
Every TP/CP rank runs a scheduler and observes the same requests, and a PP
|
||||
stage only sees its own microbatches, so an ungated counter would be
|
||||
multiplied across replicas once the multiprocess registry aggregates them.
|
||||
Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same
|
||||
shape the existing scheduler metrics use for rank-unique series.
|
||||
"""
|
||||
global _COMPLETION_RANK
|
||||
if _COMPLETION_RANK is None:
|
||||
try:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
parallel = get_parallel()
|
||||
_COMPLETION_RANK = bool(
|
||||
parallel.attn_tp_rank == 0
|
||||
and parallel.attn_cp_rank == 0
|
||||
and parallel.pp_group.is_last_rank
|
||||
)
|
||||
except Exception:
|
||||
# Unknown topology (e.g. a bare unit-test process): keep counting.
|
||||
_COMPLETION_RANK = True
|
||||
return _COMPLETION_RANK
|
||||
|
||||
|
||||
def _current_dp_rank() -> int:
|
||||
"""This process's attention-DP rank (0 for non-DP engines)."""
|
||||
global _DP_RANK
|
||||
if _DP_RANK is None:
|
||||
try:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
_DP_RANK = int(get_parallel().attn_dp_rank or 0)
|
||||
except Exception:
|
||||
_DP_RANK = 0
|
||||
return _DP_RANK
|
||||
|
||||
|
||||
def _ensure_request_counter():
|
||||
global _REQUEST_COUNTER
|
||||
if _REQUEST_COUNTER is None:
|
||||
from prometheus_client import Counter
|
||||
|
||||
_REQUEST_COUNTER = Counter(
|
||||
"sglang:batch_requests_total",
|
||||
"Requests completed by the canonical scheduler rank, by phase. "
|
||||
"phase=prefill: prefill compute finished and the request entered the "
|
||||
"PD transfer queue (not a guarantee that KV transfer succeeded). "
|
||||
"phase=decode: request finished decoding.",
|
||||
["phase", "dp_rank"],
|
||||
)
|
||||
return _REQUEST_COUNTER
|
||||
|
||||
|
||||
def note_request_completed(phase: str, n: int = 1) -> None:
|
||||
"""Count finished requests per DP rank.
|
||||
|
||||
This is the per-rank RPS source: the frontend ``num_requests_total`` has no
|
||||
rank label, while the scheduler knows exactly which DP shard completed the
|
||||
request. Cheap (one counter increment per finished request) and gated by
|
||||
SGLANG_BATCH_TELEMETRY like the other batch metrics.
|
||||
"""
|
||||
if not _ENABLED or n <= 0 or not _should_report_completion():
|
||||
return
|
||||
try:
|
||||
_ensure_request_counter().labels(
|
||||
phase=phase, dp_rank=str(_current_dp_rank())
|
||||
).inc(int(n))
|
||||
except Exception:
|
||||
# Telemetry must never fail a request.
|
||||
pass
|
||||
|
||||
|
||||
def _ensure_histograms():
|
||||
global _HISTOGRAMS
|
||||
if _HISTOGRAMS is None:
|
||||
|
||||
Reference in New Issue
Block a user