observability: split scheduler recv_requests into pull/broadcast/unwrap
Runtime accounting on the prefill service (64 x 14.6k-token requests,
8-way concurrency) showed the non-forward wall time is dominated by the
scheduler receive path, not by radix match:
forward (extend+idle) 58.2% | get_next 6.9% | recv_requests 25.6%
process_batch_result 1.8% | run_batch overhead 0.9% | other ~6%
The existing pass span only covers get_next, so it cannot explain that
25.6%. Split it with the same opt-in gate:
* sglang:recv_requests_seconds{dp_rank, part} (total/pull/broadcast/
unwrap/mm), 5 new series per rank
* sglang:recv_requests_msgs_total, sglang:recv_requests_prompt_tokens_total
* OTLP span scheduler.recv_requests with the same breakdown, sampled
deterministically by a per-process counter (SGLANG_RECV_SPAN_SAMPLE,
default 0.05; SGLANG_RECV_SPAN=0 disables the span only)
Note on scope: tokenization itself does NOT run in the scheduler. The
tokenizer workers (--tokenizer-worker-num) produce TokenizedGenerateReqInput
and the scheduler receives the already-tokenized ids. What this measures is
ZMQ pull + decode, the cross-rank broadcast of the pickled objects, and
unwrap_pickle_fields() of the token-id buffers -- which is why large prompts
make recv_requests expensive.
Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py
now also covers the recv histogram parts, counters, zero-message passes and
deterministic recv-span sampling (8 tests total, all passing).
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import (
|
||||
@@ -33,6 +34,7 @@ from sglang.srt.managers.mm_utils import (
|
||||
has_shm_features,
|
||||
unwrap_shm_features,
|
||||
)
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.scheduler_stage_metrics import (
|
||||
SCHEDULER_STAGE_RECV_REQUESTS,
|
||||
SchedulerStageMetricsRecorder,
|
||||
@@ -93,20 +95,54 @@ class SchedulerRequestReceiver:
|
||||
if not self.recv_skipper.handle(self.get_last_batch()):
|
||||
return []
|
||||
|
||||
_bt = batch_telemetry.recv_timing_enabled()
|
||||
_t0 = time.monotonic() if _bt else 0.0
|
||||
recv_reqs = self._pull_raw_reqs()
|
||||
_t_pull = time.monotonic() if _bt else 0.0
|
||||
|
||||
if self.input_blocker is not None:
|
||||
recv_reqs = self.input_blocker.handle(recv_reqs)
|
||||
|
||||
recv_reqs = self._broadcast_reqs_across_ranks(recv_reqs, local_reqs)
|
||||
_t_bcast = time.monotonic() if _bt else 0.0
|
||||
|
||||
if get_parallel().pp_rank == 0:
|
||||
self.unwrap_pickle_wrapper(recv_reqs)
|
||||
_t_unwrap = time.monotonic() if _bt else 0.0
|
||||
|
||||
recv_reqs = self._apply_mm_receiver(recv_reqs)
|
||||
_t_mm = time.monotonic() if _bt else 0.0
|
||||
|
||||
self._finalize_shm_features(recv_reqs)
|
||||
|
||||
if _bt:
|
||||
n_msgs = 0
|
||||
n_tokens = 0
|
||||
for req in recv_reqs or ():
|
||||
n_msgs += 1
|
||||
ids = getattr(req, "input_ids", None)
|
||||
if ids is None:
|
||||
try:
|
||||
for sub in req:
|
||||
n_tokens += len(getattr(sub, "input_ids", ()) or ())
|
||||
except TypeError:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
n_tokens += len(ids)
|
||||
except TypeError:
|
||||
pass
|
||||
batch_telemetry.observe_recv(
|
||||
dp_rank=get_parallel().attn_dp_rank,
|
||||
total=time.monotonic() - _t0,
|
||||
pull=_t_pull - _t0,
|
||||
broadcast=_t_bcast - _t_pull,
|
||||
unwrap=_t_unwrap - _t_bcast,
|
||||
mm=_t_mm - _t_unwrap,
|
||||
n_msgs=n_msgs,
|
||||
n_tokens=n_tokens,
|
||||
)
|
||||
|
||||
return recv_reqs
|
||||
|
||||
def _pull_raw_reqs(self) -> Optional[List]:
|
||||
|
||||
@@ -740,3 +740,108 @@ def end_pass(dp_rank: int) -> None:
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop
|
||||
_STAGE_TLS.current = None
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler receive path (same SGLANG_BATCH_STAGE_TIMING gate): split the
|
||||
# recv_requests stage into pull / broadcast / unwrap / mm.
|
||||
#
|
||||
# NOTE: tokenization itself runs in the tokenizer worker processes (HTTP side).
|
||||
# What is measured here is the scheduler-side IPC: ZMQ pull + decode of the
|
||||
# tokenized request objects, the cross-rank broadcast of those objects, and
|
||||
# the unpickling of their token-id buffers. Large prompts make all three
|
||||
# visible (e.g. 14.6k tokens per request).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_RECV_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
|
||||
"SGLANG_RECV_SPAN", "1"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
_RECV_SPAN_SAMPLE = float(os.getenv("SGLANG_RECV_SPAN_SAMPLE", "0.05"))
|
||||
|
||||
_RECV_PARTS = ("total", "pull", "broadcast", "unwrap", "mm")
|
||||
_RECV_HISTS = None
|
||||
_RECV_COUNTERS = None
|
||||
_RECV_SEQ = 0
|
||||
|
||||
|
||||
def recv_timing_enabled() -> bool:
|
||||
return _STAGE_TIMING
|
||||
|
||||
|
||||
def _ensure_recv_metrics():
|
||||
global _RECV_HISTS, _RECV_COUNTERS
|
||||
if _RECV_HISTS is None:
|
||||
from prometheus_client import Counter, Histogram
|
||||
|
||||
_RECV_HISTS = Histogram(
|
||||
"sglang:recv_requests_seconds",
|
||||
"Scheduler recv_requests wall time by part "
|
||||
"(total/pull/broadcast/unwrap/mm)",
|
||||
["dp_rank", "part"],
|
||||
buckets=_STAGE_BUCKETS,
|
||||
)
|
||||
_RECV_COUNTERS = {
|
||||
"msgs": Counter(
|
||||
"sglang:recv_requests_msgs_total",
|
||||
"Messages received by the scheduler recv path",
|
||||
["dp_rank"],
|
||||
),
|
||||
"prompt_tokens": Counter(
|
||||
"sglang:recv_requests_prompt_tokens_total",
|
||||
"Prompt tokens carried by messages received in the recv path",
|
||||
["dp_rank"],
|
||||
),
|
||||
}
|
||||
return _RECV_HISTS, _RECV_COUNTERS
|
||||
|
||||
|
||||
def observe_recv(
|
||||
*,
|
||||
dp_rank: int,
|
||||
total: float,
|
||||
pull: float,
|
||||
broadcast: float,
|
||||
unwrap: float,
|
||||
mm: float,
|
||||
n_msgs: int,
|
||||
n_tokens: int,
|
||||
) -> None:
|
||||
if not _STAGE_TIMING:
|
||||
return
|
||||
global _RECV_SEQ
|
||||
_RECV_SEQ += 1
|
||||
try:
|
||||
hist, counters = _ensure_recv_metrics()
|
||||
for part, value in zip(_RECV_PARTS, (total, pull, broadcast, unwrap, mm)):
|
||||
hist.labels(dp_rank=str(dp_rank), part=part).observe(value)
|
||||
if n_msgs:
|
||||
counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs)
|
||||
if n_tokens:
|
||||
counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens)
|
||||
if not _RECV_SPAN_ENABLED:
|
||||
return
|
||||
if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int(
|
||||
_RECV_SPAN_SAMPLE * 10000
|
||||
):
|
||||
return
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.recv")
|
||||
end_ns = time.time_ns()
|
||||
tracer.start_span(
|
||||
"scheduler.recv_requests",
|
||||
start_time=end_ns - int(total * 1e9),
|
||||
attributes={
|
||||
"sglang.phase": "recv",
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.recv_msgs": int(n_msgs),
|
||||
"sglang.recv_prompt_tokens": int(n_tokens),
|
||||
"sglang.recv_total_ms": total * 1e3,
|
||||
"sglang.recv_pull_ms": pull * 1e3,
|
||||
"sglang.recv_broadcast_ms": broadcast * 1e3,
|
||||
"sglang.recv_unwrap_ms": unwrap * 1e3,
|
||||
"sglang.recv_mm_ms": mm * 1e3,
|
||||
},
|
||||
).end(end_time=end_ns)
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user