From 85f4f9576c3b9f34f8d9e0830ced0c505dde7958 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Wed, 4 Nov 2026 14:48:46 +0800 Subject: [PATCH] scheduler: add opt-in per-pass prefill stage timing (SGLANG_BATCH_STAGE_TIMING) Motivation: on the PD prefill node the wall-clock budget is dominated by get_next_batch_to_run (~50% of the hour on 2026-11-02 16:00-17:00), and forward_overlap=partial is only 21-87 s/h, i.e. it does not overlap the GPU forward. We know the total but not which segment inside it costs: radix match, PrefillAdder admission, the HiCache event collective, the bootstrap all-reduce, or the DP metadata gather. Measurement design (one sample per scheduling pass, including passes that produce no batch; see handoff/prefill-batch-stage-timing-20261103.md v2/v3): L1 7 per-pass histograms: sglang:batch_prefill_{get_next,match,adder,hicache_events, bootstrap,mlp_sync,other}_seconds other = total - (match + adder + hicache + bootstrap + mlp_sync), clamped to >= 0. Nested stages are not double counted. L3 5 per-pass counters: candidates / skipped_prefetch / admitted / resumed_chunk / new_chunk L2 a dedicated per-pass span scheduler.prefill.pass (not the batch span), emitted from end_pass() on every pass. Its sampling decision is a pure function of mlp_sync_seq, so a sampled gather round is sampled on all DP ranks. Idle ranks and ranks without a local batch still emit, which is exactly what the cross-rank skew analysis needs. No links. Hooks are installed in both scheduling loops: * get_next_disagg_prefill_batch_to_run() for --disaggregation-mode prefill (hicache / bootstrap / mlp_sync segments), and * get_next_batch_to_run() for standalone prefill (hicache / mlp_sync), so the instrumentation can be validated on ds41-prefill-dp4 without a decode peer. match/adder/counters live in the shared get_new_batch_prefill path. Semantics: * match times only init_next_round_input; check_prefetch_progress counts as skipped_prefetch, never as match. * bootstrap and hicache_events are cross-rank sync points measured at the outer call, not attributed to local CPU work. * dp_gather_ms comes from the existing MLPSyncBatchInfo.sync_wait_seconds; sglang:dp_attention_sync_wait_seconds is reused, no duplicate metric. The value is cleared before each outer call and mlp_sync_seq only increments when a gather actually ran, so stale values cannot leak. * resumed vs new chunked requests are captured separately: had_chunked is recorded at pass start (before process_prefill_chunk/admission can consume the last chunk) and new_chunk from adder.new_chunked_req. This fixes both the first-chunk and last-chunk mislabels. Opt-in: * SGLANG_BATCH_STAGE_TIMING=1 (default 0); SGLANG_BATCH_PASS_SPAN=1 / SGLANG_BATCH_PASS_SPAN_SAMPLE=0.05. * When off every hook is one module-level bool check; no logging, no CUDA calls, no new collectives. Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py covers first/middle/last chunk labelling, all-prefetch-skip passes without a batch, stale-gather clearing, rank-deterministic pass-span sampling, and the disabled no-op path. py_compile passes on all touched files. --- python/sglang/srt/disaggregation/prefill.py | 25 ++ python/sglang/srt/managers/scheduler.py | 34 ++ .../managers/scheduler_components/dp_attn.py | 6 + .../srt/observability/batch_telemetry.py | 361 ++++++++++++++++++ .../test_batch_telemetry_stage_timing.py | 194 ++++++++++ 5 files changed, 620 insertions(+) create mode 100644 test/registered/unit/observability/test_batch_telemetry_stage_timing.py diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 96b95faa0b..4ad4febe40 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -84,6 +84,7 @@ from sglang.srt.mem_cache.common import ( from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache +from sglang.srt.observability import batch_telemetry from sglang.srt.observability.req_time_stats import set_schedule_time_batch from sglang.srt.observability.scheduler_stage_metrics import ( SCHEDULER_STAGE_GET_NEXT_BATCH, @@ -647,16 +648,30 @@ class SchedulerDisaggregationPrefillMixin: last_batch: Optional[ScheduleBatch], should_retry_storage_prefetch: bool = True, ) -> NextBatchPlan: + bt_pass = batch_telemetry.begin_pass() + if bt_pass is not None: + # Capture before process_prefill_chunk()/admission can consume it: + # the last chunk of a resumed request clears self.chunked_req. + batch_telemetry.note_had_chunked(self.chunked_req is not None) + self.process_pending_chunked_abort() + if bt_pass is not None: + _bt_t = time.monotonic() self._process_hicache_events( should_retry_storage_prefetch=should_retry_storage_prefetch ) + if bt_pass is not None: + batch_telemetry.add_hicache(time.monotonic() - _bt_t) # HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it # Otherwise, it hangs under high concurrency running_batch.batch_is_full = False + if bt_pass is not None: + _bt_t = time.monotonic() self.resolve_waiting_queue_bootstrap() + if bt_pass is not None: + batch_telemetry.add_bootstrap(time.monotonic() - _bt_t) # Skip repeated chunk processing while polling a pending batch. if last_batch is None or not last_batch.prefill_chunk_processed: @@ -669,11 +684,21 @@ class SchedulerDisaggregationPrefillMixin: prefill_plan = self.get_new_batch_prefill(running_batch) batch = prefill_plan.batch_to_run running_batch = prefill_plan.running_batch + if bt_pass is not None: + # Drop any gather value left by an earlier pass; only a gather that + # actually runs inside the call below may be attributed to this one. + batch_telemetry.note_dp_gather(None) + _bt_t = time.monotonic() batch = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(batch) + if bt_pass is not None: + batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t) if batch: set_schedule_time_batch(batch) + if bt_pass is not None: + batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank) + return NextBatchPlan(batch_to_run=batch, running_batch=running_batch) @torch.no_grad() diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index a57924af00..9bf630348a 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -304,6 +304,7 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow from sglang.srt.model_loader.utils import get_resolved_model_impl from sglang.srt.multiplex.multiplexing_mixin import SchedulerMultiplexMixin +from sglang.srt.observability import batch_telemetry from sglang.srt.observability.metrics_collector import SchedulerMetricsCollector from sglang.srt.observability.req_time_stats import ( flush_trace_batch, @@ -3856,8 +3857,17 @@ class Scheduler( def get_next_batch_to_run( self, running_batch: ScheduleBatch, last_batch: Optional[ScheduleBatch] ) -> NextBatchPlan: + bt_pass = batch_telemetry.begin_pass() + if bt_pass is not None: + # Capture before the chunked request can be consumed by this pass. + batch_telemetry.note_had_chunked(self.chunked_req is not None) + self.process_pending_chunked_abort() + if bt_pass is not None: + _bt_t = time.monotonic() self._process_hicache_events() + if bt_pass is not None: + batch_telemetry.add_hicache(time.monotonic() - _bt_t) if self.dllm_config is not None: self.dllm_manager.filter_finished_reqs() @@ -3948,7 +3958,11 @@ class Scheduler( # Before merging the new batch into running batch: # 1. All new batches are none -> need_mlp_sync remains true (sync is needed for decode batch). # 2. All new batches are some (prefill / idle) -> we do not need prepare mlp sync one more time. + if bt_pass is not None: + _bt_t = time.monotonic() new_batch = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(new_batch) + if bt_pass is not None: + batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t) need_mlp_sync = new_batch is None if new_batch is not None: @@ -3967,9 +3981,13 @@ class Scheduler( ret = None # Handle DP attention and log stats + if bt_pass is not None: + _bt_t = time.monotonic() ret = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch( ret, need_sync=need_mlp_sync ) + if bt_pass is not None: + batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t) # Decode->extend conversion keeps a heterogeneous dp step replayable. # A converted running batch stays the running batch; next step it comes # back as last_batch and is filtered in place, not merged with itself. @@ -3984,6 +4002,9 @@ class Scheduler( if ret: set_schedule_time_batch(ret) + if bt_pass is not None: + batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank) + return NextBatchPlan(batch_to_run=ret, running_batch=running_batch) def get_num_allocatable_reqs( @@ -4168,6 +4189,7 @@ class Scheduler( # candidate: the resumed chunk or mixed decode rows can spend it. if adder.budget_state() == AddReqResult.OTHER: break + batch_telemetry.mark_candidate() if self.enable_lora and not self.can_schedule_lora_req(req, running_loras): continue @@ -4206,6 +4228,7 @@ class Scheduler( req.time_stats.observe_storage_prefetch_ready(prefetch_done) if not prefetch_done: # skip staging requests that are ongoing prefetch + batch_telemetry.mark_skipped_prefetch() continue # Pop the L3-loaded span. Unified cache exposes its absolute # start so cache-mode L2/L3 attribution survives L3-tail eviction. @@ -4219,7 +4242,11 @@ class Scheduler( # marks the staged span below once it is surfaced. req.host_hit_is_storage = False + if batch_telemetry.stage_timing_enabled(): + _bt_t = time.monotonic() req.init_next_round_input(self.tree_cache) + if batch_telemetry.stage_timing_enabled(): + batch_telemetry.add_match(time.monotonic() - _bt_t) if self.enable_hicache_storage and ( self._prefetch_after_device_hit_loss(req) ): @@ -4229,11 +4256,15 @@ class Scheduler( req.time_stats.observe_storage_staging_ready(staging_ready) if not staging_ready: continue + if batch_telemetry.stage_timing_enabled(): + _bt_t = time.monotonic() res = adder.add_one_req( req, has_chunked_req=(self.chunked_req is not None), truncation_align_size=self.truncation_align_size, ) + if batch_telemetry.stage_timing_enabled(): + batch_telemetry.add_adder(time.monotonic() - _bt_t) if self.enable_lora: running_loras.add(req.lora_id) @@ -4271,6 +4302,7 @@ class Scheduler( # though the request they made room for was not admitted. for req in adder.preempt_list: self._add_request_to_queue(req) + batch_telemetry.set_admitted(0) return None, running_batch can_run_set = set(can_run_list) @@ -4373,6 +4405,8 @@ class Scheduler( else: new_batch.decoding_reqs = None + batch_telemetry.set_admitted(len(can_run_list)) + batch_telemetry.note_new_chunk(adder.new_chunked_req is not None) return new_batch, running_batch def can_schedule_lora_req( diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 2cb6e9cf1b..f3d50a943a 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -723,6 +723,12 @@ def prepare_mlp_sync_batch_raw( # No batch yet: this iteration may gather again, or end in on_idle. sync_wait_carry.seconds = sync_wait_seconds + # Stage timing: publish this pass's DP all-gather wait (None = no gather ran). + if metadata_ready and not skip_all_gather and num_dp_ranks > 1: + batch_telemetry.note_dp_gather(mlp_sync_info.sync_wait_seconds) + else: + batch_telemetry.note_dp_gather(None) + return local_batch diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index a2c9b1107a..7fb47e7758 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -35,6 +35,7 @@ from __future__ import annotations import os import random +import threading import time from typing import Any, Iterable, Optional @@ -379,3 +380,363 @@ def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str = ) except Exception: pass + + +# --------------------------------------------------------------------------- +# Per-pass stage timing (SGLANG_BATCH_STAGE_TIMING=1) -- P (disagg prefill). +# +# One sample per get_next_disagg_prefill_batch_to_run() pass, including passes +# that produce no batch. Disabled by default; when off every hook below is a +# single module-level bool check. No CUDA calls, no per-pass logging. +# +# Cross-rank analysis uses a dedicated, independently sampled span +# ``scheduler.prefill.pass`` emitted from end_pass(): the sampling decision is a +# pure function of ``mlp_sync_seq`` (identical on every DP rank that took part +# in the same gather), so a sampled gather round is sampled on *all* ranks -- +# unlike the per-rank random sampling of the batch span. Idle ranks and ranks +# with no local batch therefore still emit, which is exactly the case the +# skew analysis needs. +# --------------------------------------------------------------------------- + +_STAGE_TIMING = os.getenv("SGLANG_BATCH_STAGE_TIMING", "0").strip().lower() not in ( + "", + "0", + "false", + "no", + "off", +) + +_STAGE_LABELS = ["phase", "dp_rank"] +_STAGE_BUCKETS = ( + 1e-5, + 3e-5, + 1e-4, + 3e-4, + 1e-3, + 3e-3, + 1e-2, + 3e-2, + 1e-1, + 3e-1, + 1.0, + 3.0, +) + +_STAGE_HISTS = None +_STAGE_COUNTERS = None +_STAGE_CHILDREN: dict = {} +_STAGE_TLS = threading.local() + +# Set by the DP all-gather path (dp_attn.prepare_mlp_sync_batch_raw). +_LAST_DP_GATHER = None +_DP_GATHER_SEQ = 0 + +_PASS_SPAN_ENABLED = _STAGE_TIMING and os.getenv( + "SGLANG_BATCH_PASS_SPAN", "1" +).strip().lower() not in ("", "0", "false", "no", "off") +_PASS_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_PASS_SPAN_SAMPLE", "0.05")) + +_STAGE_HIST_DOCS = { + "get_next": "Wall time of one get_next_disagg_prefill_batch_to_run pass", + "match": "Per-pass sum of init_next_round_input (prefix match) over candidates", + "adder": "Per-pass sum of PrefillAdder.add_one_req (budget/admission)", + "hicache_events": "Per-pass time in _process_hicache_events (includes its collectives)", + "bootstrap": "Per-pass time in resolve_waiting_queue_bootstrap (poll + all-reduce)", + "mlp_sync": "Per-pass outer maybe_prepare_mlp_sync_batch time (not pure gather wait)", + "other": "get_next total minus the instrumented segments", +} + + +def stage_timing_enabled() -> bool: + return _STAGE_TIMING + + +def pass_span_enabled() -> bool: + return _PASS_SPAN_ENABLED + + +def pass_span_decision(seq: int) -> bool: + """Deterministic, rank-identical sampling decision for one gather round. + + ``seq`` is the DP all-gather sequence number, which is the same on every + rank that participated in that round. Therefore one decision covers the + whole round, instead of four independent ``random.random()`` draws. + """ + if not _PASS_SPAN_ENABLED or seq <= 0: + return False + if _PASS_SPAN_SAMPLE >= 1.0: + return True + if _PASS_SPAN_SAMPLE <= 0.0: + return False + return (seq * 2654435761) % 10000 < int(_PASS_SPAN_SAMPLE * 10000) + + +def _ensure_stage_metrics(): + global _STAGE_HISTS, _STAGE_COUNTERS + if _STAGE_HISTS is None: + from prometheus_client import Counter, Histogram + + _STAGE_HISTS = { + name: Histogram( + f"sglang:batch_prefill_{name}_seconds", + doc, + _STAGE_LABELS, + buckets=_STAGE_BUCKETS, + ) + for name, doc in _STAGE_HIST_DOCS.items() + } + _STAGE_COUNTERS = { + "candidates": Counter( + "sglang:batch_prefill_candidates_total", + "Requests examined by the per-pass admission candidate loop", + _STAGE_LABELS, + ), + "skipped_prefetch": Counter( + "sglang:batch_prefill_skipped_prefetch_total", + "Candidates skipped because their L2/L3 prefetch was not ready", + _STAGE_LABELS, + ), + "admitted": Counter( + "sglang:batch_prefill_admitted_total", + "Requests admitted into can_run_list by one pass", + _STAGE_LABELS, + ), + "resumed_chunk": Counter( + "sglang:batch_prefill_resumed_chunk_total", + "Passes whose batch continues a chunked-prefill request " + "(including its last chunk)", + _STAGE_LABELS, + ), + "new_chunk": Counter( + "sglang:batch_prefill_new_chunk_total", + "Passes that start a new chunked-prefill request", + _STAGE_LABELS, + ), + } + return _STAGE_HISTS, _STAGE_COUNTERS + + +class PassTimings: + __slots__ = ( + "phase", + "t0", + "match", + "adder", + "hicache", + "bootstrap", + "mlp_sync", + "dp_gather", + "candidates", + "skipped_prefetch", + "admitted", + "had_chunked", + "new_chunk", + "seq", + ) + + def __init__(self, phase: str): + self.phase = phase + self.t0 = time.monotonic() + self.match = 0.0 + self.adder = 0.0 + self.hicache = 0.0 + self.bootstrap = 0.0 + self.mlp_sync = 0.0 + self.dp_gather = None + self.candidates = 0 + self.skipped_prefetch = 0 + self.admitted = 0 + self.had_chunked = False + self.new_chunk = False + self.seq = 0 + + +def begin_pass(phase: str = "prefill") -> Optional[PassTimings]: + """Start a new pass. Returns None when stage timing is disabled.""" + if not _STAGE_TIMING: + return None + timings = PassTimings(phase) + _STAGE_TLS.current = timings + return timings + + +def _current() -> Optional[PassTimings]: + if not _STAGE_TIMING: + return None + return getattr(_STAGE_TLS, "current", None) + + +def add_match(seconds: float) -> None: + t = _current() + if t is not None: + t.match += seconds + + +def add_adder(seconds: float) -> None: + t = _current() + if t is not None: + t.adder += seconds + + +def add_hicache(seconds: float) -> None: + t = _current() + if t is not None: + t.hicache += seconds + + +def add_bootstrap(seconds: float) -> None: + t = _current() + if t is not None: + t.bootstrap += seconds + + +def mark_candidate(n: int = 1) -> None: + t = _current() + if t is not None: + t.candidates += n + + +def mark_skipped_prefetch(n: int = 1) -> None: + t = _current() + if t is not None: + t.skipped_prefetch += n + + +def set_admitted(n: int) -> None: + t = _current() + if t is not None: + t.admitted = int(n) + + +def note_had_chunked(flag: bool) -> None: + """Record whether a chunked request was already in flight when the pass began. + + Must be called at pass start, before process_prefill_chunk()/admission can + consume the chunked request (the last chunk clears self.chunked_req). + """ + t = _current() + if t is not None: + t.had_chunked = bool(flag) + + +def note_new_chunk(flag: bool) -> None: + """Record whether this pass started a new chunked request (first chunk).""" + t = _current() + if t is not None: + t.new_chunk = bool(flag) + + +def note_dp_gather(seconds: Optional[float]) -> None: + """Called by the DP all-gather path; None means no gather ran this pass.""" + global _LAST_DP_GATHER, _DP_GATHER_SEQ + if not _STAGE_TIMING: + return + if seconds is None: + _LAST_DP_GATHER = None + return + _LAST_DP_GATHER = float(seconds) + _DP_GATHER_SEQ += 1 + + +def _take_dp_gather(): + global _LAST_DP_GATHER + value = _LAST_DP_GATHER + _LAST_DP_GATHER = None + return value + + +def add_mlp_sync(seconds: float) -> None: + t = _current() + if t is None: + return + t.mlp_sync += seconds + gather = _take_dp_gather() + if gather is not None: + t.dp_gather = gather + t.seq = _DP_GATHER_SEQ + + +def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None: + if not pass_span_decision(t.seq): + return + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.prefill.pass") + end_ns = time.time_ns() + start_ns = end_ns - int(total * 1e9) + tracer.start_span( + "scheduler.prefill.pass", + start_time=start_ns, + attributes={ + "sglang.phase": t.phase, + "sglang.dp_rank": int(dp_rank), + "sglang.mlp_sync_seq": int(t.seq), + "sglang.get_next_ms": total * 1e3, + "sglang.match_ms": t.match * 1e3, + "sglang.adder_ms": t.adder * 1e3, + "sglang.hicache_ms": t.hicache * 1e3, + "sglang.bootstrap_ms": t.bootstrap * 1e3, + "sglang.mlp_sync_ms": t.mlp_sync * 1e3, + "sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3, + "sglang.other_ms": other * 1e3, + "sglang.candidates": int(t.candidates), + "sglang.skipped_prefetch": int(t.skipped_prefetch), + "sglang.admitted": int(t.admitted), + "sglang.resumed_chunked": int(bool(t.had_chunked)), + "sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked), + }, + ).end(end_time=end_ns) + except Exception: + # Tracing must never break the scheduler loop. + pass + + +def end_pass(dp_rank: int) -> None: + """Close the current pass: observe histograms/counters and emit the pass span.""" + t = _current() + if t is None: + return + _STAGE_TLS.current = None + try: + total = time.monotonic() - t.t0 + measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync + other = total - measured + if other < 0.0: + other = 0.0 + hist, counters = _ensure_stage_metrics() + child_key = (t.phase, dp_rank) + keyed = _STAGE_CHILDREN.get(child_key) + if keyed is None: + keyed = { + name: h.labels(phase=t.phase, dp_rank=str(dp_rank)) + for name, h in hist.items() + } + keyed.update( + { + name: c.labels(phase=t.phase, dp_rank=str(dp_rank)) + for name, c in counters.items() + } + ) + _STAGE_CHILDREN[child_key] = keyed + keyed["get_next"].observe(total) + keyed["match"].observe(t.match) + keyed["adder"].observe(t.adder) + keyed["hicache_events"].observe(t.hicache) + keyed["bootstrap"].observe(t.bootstrap) + keyed["mlp_sync"].observe(t.mlp_sync) + keyed["other"].observe(other) + if t.candidates: + keyed["candidates"].inc(t.candidates) + if t.skipped_prefetch: + keyed["skipped_prefetch"].inc(t.skipped_prefetch) + if t.admitted: + keyed["admitted"].inc(t.admitted) + if t.had_chunked: + keyed["resumed_chunk"].inc(1) + elif t.new_chunk: + keyed["new_chunk"].inc(1) + _emit_pass_span(dp_rank, t, total, other) + except Exception: + # Telemetry must never break the scheduler loop + _STAGE_TLS.current = None diff --git a/test/registered/unit/observability/test_batch_telemetry_stage_timing.py b/test/registered/unit/observability/test_batch_telemetry_stage_timing.py new file mode 100644 index 0000000000..27d40972ef --- /dev/null +++ b/test/registered/unit/observability/test_batch_telemetry_stage_timing.py @@ -0,0 +1,194 @@ +"""Unit tests for the opt-in per-pass prefill stage timing. + +Covers the review points: + * first / middle / last chunk are labelled correctly (resumed vs new), + * a pass with no batch (all prefetch-skipped) still records L1/L3, + * a stale DP gather value is never attributed to a later pass, + * the pass-span sampling decision is deterministic for a given + ``mlp_sync_seq`` (rank-identical), unlike per-rank random sampling. +""" +import importlib.util +import os +import pathlib +import unittest +from unittest import mock + +from prometheus_client import CollectorRegistry, Counter, Histogram + + +def _module_path(): + """Resolve the module under test without importing the sglang package.""" + p = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/observability/batch_telemetry.py" + ) + if p.exists(): + return p + spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry") + return pathlib.Path(spec.origin) + + +def _reload(**env): + """Load a fresh module instance with the requested env (no reload/sys.modules).""" + base = { + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_BATCH_PASS_SPAN": "0", + "SGLANG_BATCH_PASS_SPAN_SAMPLE": "0.05", + } + base.update(env) + with mock.patch.dict(os.environ, base): + spec = importlib.util.spec_from_file_location( + "batch_telemetry_t", _module_path() + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PHASE = "prefill" +RANK = 0 + + +def _install_private_metrics(module): + """Give the module a private registry so reloads do not collide.""" + reg = CollectorRegistry() + module._STAGE_HISTS = { + name: Histogram(f"test_{name}", "", ["phase", "dp_rank"], registry=reg) + for name in module._STAGE_HIST_DOCS + } + module._STAGE_COUNTERS = { + name: Counter(f"test_{name}", "", ["phase", "dp_rank"], registry=reg) + for name in ( + "candidates", + "skipped_prefetch", + "admitted", + "resumed_chunk", + "new_chunk", + ) + } + return reg + + +def _counter(module, name): + child = module._STAGE_COUNTERS[name].labels(phase=PHASE, dp_rank=str(RANK)) + return child._value.get() + + +def _hist_sum(module, name): + child = module._STAGE_HISTS[name].labels(phase=PHASE, dp_rank=str(RANK)) + return child._sum.get() + + +class TestStageTiming(unittest.TestCase): + def setUp(self): + self.bt = _reload() + _install_private_metrics(self.bt) + + def tearDown(self): + os.environ.pop("SGLANG_BATCH_STAGE_TIMING", None) + _reload(SGLANG_BATCH_STAGE_TIMING="0") + + def _run_pass(self, had_chunked=False, new_chunk=False, gather=0.005): + bt = self.bt + bt.begin_pass() + bt.note_had_chunked(had_chunked) + bt.note_new_chunk(new_chunk) + if gather is not None: + bt.note_dp_gather(gather) + bt.add_mlp_sync(0.001) + bt.end_pass(dp_rank=RANK) + + def test_all_segments_and_counters_recorded(self): + bt = self.bt + bt.begin_pass() + bt.mark_candidate() + bt.mark_candidate() + bt.mark_skipped_prefetch() + bt.add_hicache(0.001) + bt.add_bootstrap(0.002) + bt.add_match(0.003) + bt.add_adder(0.004) + bt.note_dp_gather(0.005) + bt.add_mlp_sync(0.006) + bt.set_admitted(2) + bt.end_pass(dp_rank=RANK) + + for name in bt._STAGE_HIST_DOCS: + self.assertGreater( + _hist_sum(bt, name) + (1 if name == "other" else 0), + 0.0, + f"{name} not observed", + ) + self.assertEqual(_counter(bt, "candidates"), 2) + self.assertEqual(_counter(bt, "skipped_prefetch"), 1) + self.assertEqual(_counter(bt, "admitted"), 2) + + def test_chunk_first_middle_last(self): + bt = self.bt + self._run_pass(had_chunked=False, new_chunk=True) # first chunk + self.assertEqual(_counter(bt, "new_chunk"), 1) + self.assertEqual(_counter(bt, "resumed_chunk"), 0) + + self._run_pass(had_chunked=True, new_chunk=False) # middle chunk + self.assertEqual(_counter(bt, "new_chunk"), 1) + self.assertEqual(_counter(bt, "resumed_chunk"), 1) + + # Last chunk: had_chunked is captured before admission consumes it. + self._run_pass(had_chunked=True, new_chunk=False) + self.assertEqual(_counter(bt, "resumed_chunk"), 2) + + def test_no_batch_pass_still_records(self): + bt = self.bt + before = _hist_sum(bt, "get_next") + self._run_pass(had_chunked=False, new_chunk=False) # no batch formed + self.assertGreater(_hist_sum(bt, "get_next"), before) + self.assertEqual(_counter(bt, "skipped_prefetch"), 0) + + def test_stale_gather_not_leaked(self): + bt = self.bt + bt.begin_pass() + bt.note_dp_gather(0.004) + bt.add_mlp_sync(0.001) + bt.end_pass(dp_rank=RANK) + + bt.begin_pass() + bt.add_mlp_sync(0.001) # no gather noted for this pass + t = bt._current() + self.assertIsNone(t.dp_gather) + self.assertEqual(t.seq, 0) + bt.end_pass(dp_rank=RANK) + + def test_pass_span_sampling_is_rank_deterministic(self): + bt = self.bt + p5 = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="0.05") + _install_private_metrics(p5) + decisions = [p5.pass_span_decision(seq) for seq in range(1, 200)] + self.assertTrue(any(decisions)) + self.assertFalse(all(decisions)) + + p5b = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="0.05") + self.assertEqual( + [p5b.pass_span_decision(seq) for seq in range(1, 200)], decisions + ) + + p0 = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="0.0") + self.assertFalse(any(p0.pass_span_decision(seq) for seq in range(1, 200))) + + p1 = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="1.0") + self.assertTrue(all(p1.pass_span_decision(seq) for seq in range(1, 20))) + self.assertFalse(p1.pass_span_decision(0)) + + def test_disabled_is_noop(self): + off = _reload(SGLANG_BATCH_STAGE_TIMING="0") + self.assertFalse(off.stage_timing_enabled()) + self.assertIsNone(off.begin_pass()) + self.assertIsNone(off._current()) + off.add_match(1.0) + off.mark_candidate() + off.note_had_chunked(True) + off.end_pass(dp_rank=RANK) + self.assertFalse(off.pass_span_decision(1)) + + +if __name__ == "__main__": + unittest.main()