diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index cecc957537..b36d6c3fd7 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1273,6 +1273,7 @@ class SchedulerDisaggregationPrefillMixin: self.checkpoint_disagg_prefill(req) self.disagg_prefill_inflight_queue.append(req) + batch_telemetry.note_request_completed("prefill") if self.spec_algorithm.is_eagle() and draft_input is not None: req.output_topk_p = draft_input.topk_p[i] req.output_topk_index = draft_input.topk_index[i] diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index 59aa51413d..4cffb6801e 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -29,6 +29,7 @@ from sglang.srt.managers.schedule_batch import ( ScheduleBatch, mamba_lazy_spec_in_window, ) +from sglang.srt.observability import batch_telemetry from sglang.srt.mem_cache.common import ( checkpoint_kv_cache, release_kv_cache, @@ -995,6 +996,10 @@ class SchedulerBatchResultProcessor: if batch.spec_algorithm.is_none() and logits_output is not None: newly_finished_beam_groups = self.beam_coordinator.commit_decode(batch) + # Keep the hot loop free when telemetry is off: read the gate once and + # skip the per-request finished() check entirely. + track_completions = batch_telemetry.enabled() + finished_reqs = 0 for i, req in enumerate(batch.reqs): req: Req @@ -1009,6 +1014,8 @@ class SchedulerBatchResultProcessor: self._handle_finish_state_updated_req( req, batch, result, i, logits_output ) + if track_completions and req.finished(): + finished_reqs += 1 continue if (self.enable_overlap or self.enable_overlap_mlx) and ( @@ -1041,10 +1048,16 @@ class SchedulerBatchResultProcessor: req.update_finish_state(new_accept_len) if sampling_mask_finish_reason is not None: + # This path finishes the request too (to_finish -> + # finished_reason); count it before the continue. self._handle_sampling_mask_abort(req) + if track_completions and req.finished(): + finished_reqs += 1 continue self._handle_finish_state_updated_req(req, batch, result, i, logits_output) + if track_completions and req.finished(): + finished_reqs += 1 if req.return_logprob: self._apply_decode_logprobs( @@ -1081,6 +1094,9 @@ class SchedulerBatchResultProcessor: self._accept_grammar_tokens(req, next_token_id) req.grammar.finished = req.finished() + if finished_reqs: + batch_telemetry.note_request_completed("decode", finished_reqs) + if auxiliary_output is not None: self.consume_auxiliary_output( batch, diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index 9e56d8c8bd..7a733605f6 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -75,6 +75,85 @@ def enabled() -> bool: return _ENABLED +_REQUEST_COUNTER = None +_DP_RANK = None +_COMPLETION_RANK = None + + +def _should_report_completion() -> bool: + """Whether this scheduler is the canonical reporter of request completions. + + Every TP/CP rank runs a scheduler and observes the same requests, and a PP + stage only sees its own microbatches, so an ungated counter would be + multiplied across replicas once the multiprocess registry aggregates them. + Report from attn_tp_rank==0 / attn_cp_rank==0 / last PP stage, the same + shape the existing scheduler metrics use for rank-unique series. + """ + global _COMPLETION_RANK + if _COMPLETION_RANK is None: + try: + from sglang.srt.runtime_context import get_parallel + + parallel = get_parallel() + _COMPLETION_RANK = bool( + parallel.attn_tp_rank == 0 + and parallel.attn_cp_rank == 0 + and parallel.pp_group.is_last_rank + ) + except Exception: + # Unknown topology (e.g. a bare unit-test process): keep counting. + _COMPLETION_RANK = True + return _COMPLETION_RANK + + +def _current_dp_rank() -> int: + """This process's attention-DP rank (0 for non-DP engines).""" + global _DP_RANK + if _DP_RANK is None: + try: + from sglang.srt.runtime_context import get_parallel + + _DP_RANK = int(get_parallel().attn_dp_rank or 0) + except Exception: + _DP_RANK = 0 + return _DP_RANK + + +def _ensure_request_counter(): + global _REQUEST_COUNTER + if _REQUEST_COUNTER is None: + from prometheus_client import Counter + + _REQUEST_COUNTER = Counter( + "sglang:batch_requests_total", + "Requests completed by the canonical scheduler rank, by phase. " + "phase=prefill: prefill compute finished and the request entered the " + "PD transfer queue (not a guarantee that KV transfer succeeded). " + "phase=decode: request finished decoding.", + ["phase", "dp_rank"], + ) + return _REQUEST_COUNTER + + +def note_request_completed(phase: str, n: int = 1) -> None: + """Count finished requests per DP rank. + + This is the per-rank RPS source: the frontend ``num_requests_total`` has no + rank label, while the scheduler knows exactly which DP shard completed the + request. Cheap (one counter increment per finished request) and gated by + SGLANG_BATCH_TELEMETRY like the other batch metrics. + """ + if not _ENABLED or n <= 0 or not _should_report_completion(): + return + try: + _ensure_request_counter().labels( + phase=phase, dp_rank=str(_current_dp_rank()) + ).inc(int(n)) + except Exception: + # Telemetry must never fail a request. + pass + + def _ensure_histograms(): global _HISTOGRAMS if _HISTOGRAMS is None: diff --git a/test/registered/unit/observability/test_rank_requests.py b/test/registered/unit/observability/test_rank_requests.py new file mode 100644 index 0000000000..52bcc526fc --- /dev/null +++ b/test/registered/unit/observability/test_rank_requests.py @@ -0,0 +1,103 @@ +"""Per-rank request counter used for P/D per-rank RPS. + +``sglang:batch_requests_total{phase,dp_rank}`` is incremented by the scheduler +process (which knows its DP shard), because the frontend ``num_requests_total`` +carries no rank label. + +Skipped when prometheus_client is not installed. +""" +import importlib.util +import os +import pathlib +import unittest +from unittest import mock + +try: # pragma: no cover - environment dependent + import prometheus_client # noqa: F401 + + _HAS_PROM = True +except Exception: # pragma: no cover + _HAS_PROM = False + + +def _module_path(): + p = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/observability/batch_telemetry.py" + ) + if p.exists(): + return p + spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry") + return pathlib.Path(spec.origin) + + +def _load(**env): + base = { + "SGLANG_BATCH_TELEMETRY": "1", + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_TRACE_LOOP_SPAN": "1", + "SGLANG_TRACE_LOOP_SAMPLE": "1.0", + "SGLANG_TRACE_LOOP_DETAIL": "1", + } + base.update(env) + with mock.patch.dict(os.environ, base): + spec = importlib.util.spec_from_file_location( + "batch_telemetry_rank_rps_under_test", _module_path() + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@unittest.skipUnless(_HAS_PROM, "requires prometheus_client") +class RankRequestCounterTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + # One module instance: prometheus_client forbids registering the same + # metric name twice in the default registry. + cls.mod = _load() + cls.mod._current_dp_rank = lambda: 2 + # These tests assume the canonical (tp0/cp0/last-pp) rank; the gate + # itself is exercised by test_non_canonical_rank_does_not_report. + cls.mod._COMPLETION_RANK = True + + def _value(self, phase, rank): + counter = self.mod._ensure_request_counter() + return counter.labels(phase=phase, dp_rank=str(rank))._value.get() + + def test_counts_per_phase_and_rank(self): + before_2 = self._value("prefill", 2) + before_0 = self._value("prefill", 0) + self.mod.note_request_completed("prefill", 3) + self.mod.note_request_completed("prefill") + self.assertEqual(self._value("prefill", 2), before_2 + 4) + self.assertEqual(self._value("prefill", 0), before_0) + self.assertEqual(self._value("decode", 2), 0) + + def test_non_canonical_rank_does_not_report(self): + # A TP/CP replica sees the same request; only the canonical scheduler + # may increment, otherwise multiprocess aggregation multiplies RPS. + before = self._value("prefill", 2) + saved = self.mod._COMPLETION_RANK + self.mod._COMPLETION_RANK = False + try: + self.mod.note_request_completed("prefill", 7) + finally: + self.mod._COMPLETION_RANK = saved + self.assertEqual(self._value("prefill", 2), before) + + def test_ignores_non_positive_and_disabled(self): + before = self._value("decode", 2) + self.mod.note_request_completed("decode", 0) + self.assertEqual(self._value("decode", 2), before) + saved = self.mod._ENABLED + self.mod._ENABLED = False + try: + self.mod.note_request_completed("decode", 5) + finally: + self.mod._ENABLED = saved + self.assertEqual(self._value("decode", 2), before) + + +if __name__ == "__main__": + unittest.main()