From 0866012b809e0f7c4819bd1b3d878502f437bcfd Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Wed, 4 Nov 2026 16:11:13 +0800 Subject: [PATCH] observability: split scheduler recv_requests into pull/broadcast/unwrap Runtime accounting on the prefill service (64 x 14.6k-token requests, 8-way concurrency) showed the non-forward wall time is dominated by the scheduler receive path, not by radix match: forward (extend+idle) 58.2% | get_next 6.9% | recv_requests 25.6% process_batch_result 1.8% | run_batch overhead 0.9% | other ~6% The existing pass span only covers get_next, so it cannot explain that 25.6%. Split it with the same opt-in gate: * sglang:recv_requests_seconds{dp_rank, part} (total/pull/broadcast/ unwrap/mm), 5 new series per rank * sglang:recv_requests_msgs_total, sglang:recv_requests_prompt_tokens_total * OTLP span scheduler.recv_requests with the same breakdown, sampled deterministically by a per-process counter (SGLANG_RECV_SPAN_SAMPLE, default 0.05; SGLANG_RECV_SPAN=0 disables the span only) Note on scope: tokenization itself does NOT run in the scheduler. The tokenizer workers (--tokenizer-worker-num) produce TokenizedGenerateReqInput and the scheduler receives the already-tokenized ids. What this measures is ZMQ pull + decode, the cross-rank broadcast of the pickled objects, and unwrap_pickle_fields() of the token-id buffers -- which is why large prompts make recv_requests expensive. Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py now also covers the recv histogram parts, counters, zero-message passes and deterministic recv-span sampling (8 tests total, all passing). --- .../scheduler_components/request_receiver.py | 36 ++++++ .../srt/observability/batch_telemetry.py | 105 ++++++++++++++++++ .../test_batch_telemetry_stage_timing.py | 59 ++++++++++ 3 files changed, 200 insertions(+) diff --git a/python/sglang/srt/managers/scheduler_components/request_receiver.py b/python/sglang/srt/managers/scheduler_components/request_receiver.py index cb2b76cb6c..20f7be7a84 100644 --- a/python/sglang/srt/managers/scheduler_components/request_receiver.py +++ b/python/sglang/srt/managers/scheduler_components/request_receiver.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import time from dataclasses import dataclass from http import HTTPStatus from typing import ( @@ -33,6 +34,7 @@ from sglang.srt.managers.mm_utils import ( has_shm_features, unwrap_shm_features, ) +from sglang.srt.observability import batch_telemetry from sglang.srt.observability.scheduler_stage_metrics import ( SCHEDULER_STAGE_RECV_REQUESTS, SchedulerStageMetricsRecorder, @@ -93,20 +95,54 @@ class SchedulerRequestReceiver: if not self.recv_skipper.handle(self.get_last_batch()): return [] + _bt = batch_telemetry.recv_timing_enabled() + _t0 = time.monotonic() if _bt else 0.0 recv_reqs = self._pull_raw_reqs() + _t_pull = time.monotonic() if _bt else 0.0 if self.input_blocker is not None: recv_reqs = self.input_blocker.handle(recv_reqs) recv_reqs = self._broadcast_reqs_across_ranks(recv_reqs, local_reqs) + _t_bcast = time.monotonic() if _bt else 0.0 if get_parallel().pp_rank == 0: self.unwrap_pickle_wrapper(recv_reqs) + _t_unwrap = time.monotonic() if _bt else 0.0 recv_reqs = self._apply_mm_receiver(recv_reqs) + _t_mm = time.monotonic() if _bt else 0.0 self._finalize_shm_features(recv_reqs) + if _bt: + n_msgs = 0 + n_tokens = 0 + for req in recv_reqs or (): + n_msgs += 1 + ids = getattr(req, "input_ids", None) + if ids is None: + try: + for sub in req: + n_tokens += len(getattr(sub, "input_ids", ()) or ()) + except TypeError: + pass + else: + try: + n_tokens += len(ids) + except TypeError: + pass + batch_telemetry.observe_recv( + dp_rank=get_parallel().attn_dp_rank, + total=time.monotonic() - _t0, + pull=_t_pull - _t0, + broadcast=_t_bcast - _t_pull, + unwrap=_t_unwrap - _t_bcast, + mm=_t_mm - _t_unwrap, + n_msgs=n_msgs, + n_tokens=n_tokens, + ) + return recv_reqs def _pull_raw_reqs(self) -> Optional[List]: diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index 7fb47e7758..a56a647071 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -740,3 +740,108 @@ def end_pass(dp_rank: int) -> None: except Exception: # Telemetry must never break the scheduler loop _STAGE_TLS.current = None + +# --------------------------------------------------------------------------- +# Scheduler receive path (same SGLANG_BATCH_STAGE_TIMING gate): split the +# recv_requests stage into pull / broadcast / unwrap / mm. +# +# NOTE: tokenization itself runs in the tokenizer worker processes (HTTP side). +# What is measured here is the scheduler-side IPC: ZMQ pull + decode of the +# tokenized request objects, the cross-rank broadcast of those objects, and +# the unpickling of their token-id buffers. Large prompts make all three +# visible (e.g. 14.6k tokens per request). +# --------------------------------------------------------------------------- + +_RECV_SPAN_ENABLED = _STAGE_TIMING and os.getenv( + "SGLANG_RECV_SPAN", "1" +).strip().lower() not in ("", "0", "false", "no", "off") +_RECV_SPAN_SAMPLE = float(os.getenv("SGLANG_RECV_SPAN_SAMPLE", "0.05")) + +_RECV_PARTS = ("total", "pull", "broadcast", "unwrap", "mm") +_RECV_HISTS = None +_RECV_COUNTERS = None +_RECV_SEQ = 0 + + +def recv_timing_enabled() -> bool: + return _STAGE_TIMING + + +def _ensure_recv_metrics(): + global _RECV_HISTS, _RECV_COUNTERS + if _RECV_HISTS is None: + from prometheus_client import Counter, Histogram + + _RECV_HISTS = Histogram( + "sglang:recv_requests_seconds", + "Scheduler recv_requests wall time by part " + "(total/pull/broadcast/unwrap/mm)", + ["dp_rank", "part"], + buckets=_STAGE_BUCKETS, + ) + _RECV_COUNTERS = { + "msgs": Counter( + "sglang:recv_requests_msgs_total", + "Messages received by the scheduler recv path", + ["dp_rank"], + ), + "prompt_tokens": Counter( + "sglang:recv_requests_prompt_tokens_total", + "Prompt tokens carried by messages received in the recv path", + ["dp_rank"], + ), + } + return _RECV_HISTS, _RECV_COUNTERS + + +def observe_recv( + *, + dp_rank: int, + total: float, + pull: float, + broadcast: float, + unwrap: float, + mm: float, + n_msgs: int, + n_tokens: int, +) -> None: + if not _STAGE_TIMING: + return + global _RECV_SEQ + _RECV_SEQ += 1 + try: + hist, counters = _ensure_recv_metrics() + for part, value in zip(_RECV_PARTS, (total, pull, broadcast, unwrap, mm)): + hist.labels(dp_rank=str(dp_rank), part=part).observe(value) + if n_msgs: + counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs) + if n_tokens: + counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens) + if not _RECV_SPAN_ENABLED: + return + if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int( + _RECV_SPAN_SAMPLE * 10000 + ): + return + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.recv") + end_ns = time.time_ns() + tracer.start_span( + "scheduler.recv_requests", + start_time=end_ns - int(total * 1e9), + attributes={ + "sglang.phase": "recv", + "sglang.dp_rank": int(dp_rank), + "sglang.recv_msgs": int(n_msgs), + "sglang.recv_prompt_tokens": int(n_tokens), + "sglang.recv_total_ms": total * 1e3, + "sglang.recv_pull_ms": pull * 1e3, + "sglang.recv_broadcast_ms": broadcast * 1e3, + "sglang.recv_unwrap_ms": unwrap * 1e3, + "sglang.recv_mm_ms": mm * 1e3, + }, + ).end(end_time=end_ns) + except Exception: + # Telemetry must never break the scheduler loop. + pass diff --git a/test/registered/unit/observability/test_batch_telemetry_stage_timing.py b/test/registered/unit/observability/test_batch_telemetry_stage_timing.py index 27d40972ef..af47dea867 100644 --- a/test/registered/unit/observability/test_batch_telemetry_stage_timing.py +++ b/test/registered/unit/observability/test_batch_telemetry_stage_timing.py @@ -66,6 +66,15 @@ def _install_private_metrics(module): "new_chunk", ) } + module._RECV_HISTS = Histogram( + "test_recv_seconds", "", ["dp_rank", "part"], registry=reg + ) + module._RECV_COUNTERS = { + "msgs": Counter("test_recv_msgs", "", ["dp_rank"], registry=reg), + "prompt_tokens": Counter( + "test_recv_prompt_tokens", "", ["dp_rank"], registry=reg + ), + } return reg @@ -178,6 +187,56 @@ class TestStageTiming(unittest.TestCase): self.assertTrue(all(p1.pass_span_decision(seq) for seq in range(1, 20))) self.assertFalse(p1.pass_span_decision(0)) + def test_recv_breakdown(self): + bt = self.bt + bt.observe_recv( + dp_rank=RANK, + total=0.010, + pull=0.003, + broadcast=0.004, + unwrap=0.002, + mm=0.0, + n_msgs=2, + n_tokens=1000, + ) + for part in ("total", "pull", "broadcast", "unwrap", "mm"): + child = bt._RECV_HISTS.labels(dp_rank=str(RANK), part=part) + expected = {"total": 0.010, "pull": 0.003, "broadcast": 0.004, + "unwrap": 0.002, "mm": 0.0}[part] + self.assertAlmostEqual(child._sum.get(), expected, places=6) + self.assertEqual( + bt._RECV_COUNTERS["msgs"].labels(dp_rank=str(RANK))._value.get(), 2 + ) + self.assertEqual( + bt._RECV_COUNTERS["prompt_tokens"].labels(dp_rank=str(RANK))._value.get(), + 1000, + ) + + # Zero-message pass still records timing, counters stay put. + bt.observe_recv( + dp_rank=RANK, total=0.001, pull=0.0, broadcast=0.0, + unwrap=0.0, mm=0.0, n_msgs=0, n_tokens=0, + ) + self.assertEqual( + bt._RECV_COUNTERS["msgs"].labels(dp_rank=str(RANK))._value.get(), 2 + ) + + def test_recv_span_sampling_is_deterministic(self): + a = _reload(SGLANG_RECV_SPAN="1", SGLANG_RECV_SPAN_SAMPLE="0.05") + _install_private_metrics(a) + a._RECV_SEQ = 0 + seqs = [] + for _ in range(200): + a.observe_recv( + dp_rank=RANK, total=0.001, pull=0.0, broadcast=0.0, + unwrap=0.0, mm=0.0, n_msgs=0, n_tokens=0, + ) + seqs.append(a._RECV_SEQ) + self.assertEqual(seqs, list(range(1, 201))) + # 5% of the 200 calls must be selected by the deterministic hash. + picked = [n for n in seqs if (n * 2654435761) % 10000 < 500] + self.assertTrue(0 < len(picked) < 200) + def test_disabled_is_noop(self): off = _reload(SGLANG_BATCH_STAGE_TIMING="0") self.assertFalse(off.stage_timing_enabled())