observability: split scheduler recv_requests into pull/broadcast/unwrap

Runtime accounting on the prefill service (64 x 14.6k-token requests,
8-way concurrency) showed the non-forward wall time is dominated by the
scheduler receive path, not by radix match:

  forward (extend+idle) 58.2% | get_next 6.9% | recv_requests 25.6%
  process_batch_result 1.8% | run_batch overhead 0.9% | other ~6%

The existing pass span only covers get_next, so it cannot explain that
25.6%.  Split it with the same opt-in gate:

  * sglang:recv_requests_seconds{dp_rank, part} (total/pull/broadcast/
    unwrap/mm), 5 new series per rank
  * sglang:recv_requests_msgs_total, sglang:recv_requests_prompt_tokens_total
  * OTLP span scheduler.recv_requests with the same breakdown, sampled
    deterministically by a per-process counter (SGLANG_RECV_SPAN_SAMPLE,
    default 0.05; SGLANG_RECV_SPAN=0 disables the span only)

Note on scope: tokenization itself does NOT run in the scheduler. The
tokenizer workers (--tokenizer-worker-num) produce TokenizedGenerateReqInput
and the scheduler receives the already-tokenized ids. What this measures is
ZMQ pull + decode, the cross-rank broadcast of the pickled objects, and
unwrap_pickle_fields() of the token-id buffers -- which is why large prompts
make recv_requests expensive.

Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py
now also covers the recv histogram parts, counters, zero-message passes and
deterministic recv-span sampling (8 tests total, all passing).
This commit is contained in:
2026-11-04 16:11:13 +08:00
parent 85f4f9576c
commit 0866012b80
3 changed files with 200 additions and 0 deletions
@@ -1,6 +1,7 @@
from __future__ import annotations
import logging
import time
from dataclasses import dataclass
from http import HTTPStatus
from typing import (
@@ -33,6 +34,7 @@ from sglang.srt.managers.mm_utils import (
has_shm_features,
unwrap_shm_features,
)
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.scheduler_stage_metrics import (
SCHEDULER_STAGE_RECV_REQUESTS,
SchedulerStageMetricsRecorder,
@@ -93,20 +95,54 @@ class SchedulerRequestReceiver:
if not self.recv_skipper.handle(self.get_last_batch()):
return []
_bt = batch_telemetry.recv_timing_enabled()
_t0 = time.monotonic() if _bt else 0.0
recv_reqs = self._pull_raw_reqs()
_t_pull = time.monotonic() if _bt else 0.0
if self.input_blocker is not None:
recv_reqs = self.input_blocker.handle(recv_reqs)
recv_reqs = self._broadcast_reqs_across_ranks(recv_reqs, local_reqs)
_t_bcast = time.monotonic() if _bt else 0.0
if get_parallel().pp_rank == 0:
self.unwrap_pickle_wrapper(recv_reqs)
_t_unwrap = time.monotonic() if _bt else 0.0
recv_reqs = self._apply_mm_receiver(recv_reqs)
_t_mm = time.monotonic() if _bt else 0.0
self._finalize_shm_features(recv_reqs)
if _bt:
n_msgs = 0
n_tokens = 0
for req in recv_reqs or ():
n_msgs += 1
ids = getattr(req, "input_ids", None)
if ids is None:
try:
for sub in req:
n_tokens += len(getattr(sub, "input_ids", ()) or ())
except TypeError:
pass
else:
try:
n_tokens += len(ids)
except TypeError:
pass
batch_telemetry.observe_recv(
dp_rank=get_parallel().attn_dp_rank,
total=time.monotonic() - _t0,
pull=_t_pull - _t0,
broadcast=_t_bcast - _t_pull,
unwrap=_t_unwrap - _t_bcast,
mm=_t_mm - _t_unwrap,
n_msgs=n_msgs,
n_tokens=n_tokens,
)
return recv_reqs
def _pull_raw_reqs(self) -> Optional[List]:
@@ -740,3 +740,108 @@ def end_pass(dp_rank: int) -> None:
except Exception:
# Telemetry must never break the scheduler loop
_STAGE_TLS.current = None
# ---------------------------------------------------------------------------
# Scheduler receive path (same SGLANG_BATCH_STAGE_TIMING gate): split the
# recv_requests stage into pull / broadcast / unwrap / mm.
#
# NOTE: tokenization itself runs in the tokenizer worker processes (HTTP side).
# What is measured here is the scheduler-side IPC: ZMQ pull + decode of the
# tokenized request objects, the cross-rank broadcast of those objects, and
# the unpickling of their token-id buffers. Large prompts make all three
# visible (e.g. 14.6k tokens per request).
# ---------------------------------------------------------------------------
_RECV_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
"SGLANG_RECV_SPAN", "1"
).strip().lower() not in ("", "0", "false", "no", "off")
_RECV_SPAN_SAMPLE = float(os.getenv("SGLANG_RECV_SPAN_SAMPLE", "0.05"))
_RECV_PARTS = ("total", "pull", "broadcast", "unwrap", "mm")
_RECV_HISTS = None
_RECV_COUNTERS = None
_RECV_SEQ = 0
def recv_timing_enabled() -> bool:
return _STAGE_TIMING
def _ensure_recv_metrics():
global _RECV_HISTS, _RECV_COUNTERS
if _RECV_HISTS is None:
from prometheus_client import Counter, Histogram
_RECV_HISTS = Histogram(
"sglang:recv_requests_seconds",
"Scheduler recv_requests wall time by part "
"(total/pull/broadcast/unwrap/mm)",
["dp_rank", "part"],
buckets=_STAGE_BUCKETS,
)
_RECV_COUNTERS = {
"msgs": Counter(
"sglang:recv_requests_msgs_total",
"Messages received by the scheduler recv path",
["dp_rank"],
),
"prompt_tokens": Counter(
"sglang:recv_requests_prompt_tokens_total",
"Prompt tokens carried by messages received in the recv path",
["dp_rank"],
),
}
return _RECV_HISTS, _RECV_COUNTERS
def observe_recv(
*,
dp_rank: int,
total: float,
pull: float,
broadcast: float,
unwrap: float,
mm: float,
n_msgs: int,
n_tokens: int,
) -> None:
if not _STAGE_TIMING:
return
global _RECV_SEQ
_RECV_SEQ += 1
try:
hist, counters = _ensure_recv_metrics()
for part, value in zip(_RECV_PARTS, (total, pull, broadcast, unwrap, mm)):
hist.labels(dp_rank=str(dp_rank), part=part).observe(value)
if n_msgs:
counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs)
if n_tokens:
counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens)
if not _RECV_SPAN_ENABLED:
return
if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int(
_RECV_SPAN_SAMPLE * 10000
):
return
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.recv")
end_ns = time.time_ns()
tracer.start_span(
"scheduler.recv_requests",
start_time=end_ns - int(total * 1e9),
attributes={
"sglang.phase": "recv",
"sglang.dp_rank": int(dp_rank),
"sglang.recv_msgs": int(n_msgs),
"sglang.recv_prompt_tokens": int(n_tokens),
"sglang.recv_total_ms": total * 1e3,
"sglang.recv_pull_ms": pull * 1e3,
"sglang.recv_broadcast_ms": broadcast * 1e3,
"sglang.recv_unwrap_ms": unwrap * 1e3,
"sglang.recv_mm_ms": mm * 1e3,
},
).end(end_time=end_ns)
except Exception:
# Telemetry must never break the scheduler loop.
pass