scheduler: add opt-in per-pass prefill stage timing (SGLANG_BATCH_STAGE_TIMING)

Motivation: on the PD prefill node the wall-clock budget is dominated by
get_next_batch_to_run (~50% of the hour on 2026-11-02 16:00-17:00), and
forward_overlap=partial is only 21-87 s/h, i.e. it does not overlap the GPU
forward. We know the total but not which segment inside it costs: radix
match, PrefillAdder admission, the HiCache event collective, the bootstrap
all-reduce, or the DP metadata gather.

Measurement design (one sample per scheduling pass, including passes that
produce no batch; see handoff/prefill-batch-stage-timing-20261103.md v2/v3):

  L1  7 per-pass histograms:
      sglang:batch_prefill_{get_next,match,adder,hicache_events,
      bootstrap,mlp_sync,other}_seconds
      other = total - (match + adder + hicache + bootstrap + mlp_sync),
      clamped to >= 0. Nested stages are not double counted.

  L3  5 per-pass counters:
      candidates / skipped_prefetch / admitted / resumed_chunk / new_chunk

  L2  a dedicated per-pass span scheduler.prefill.pass (not the batch span),
      emitted from end_pass() on every pass. Its sampling decision is a pure
      function of mlp_sync_seq, so a sampled gather round is sampled on all
      DP ranks. Idle ranks and ranks without a local batch still emit, which
      is exactly what the cross-rank skew analysis needs. No links.

Hooks are installed in both scheduling loops:
  * get_next_disagg_prefill_batch_to_run() for --disaggregation-mode prefill
    (hicache / bootstrap / mlp_sync segments), and
  * get_next_batch_to_run() for standalone prefill (hicache / mlp_sync),
so the instrumentation can be validated on ds41-prefill-dp4 without a decode
peer. match/adder/counters live in the shared get_new_batch_prefill path.

Semantics:
  * match times only init_next_round_input; check_prefetch_progress counts as
    skipped_prefetch, never as match.
  * bootstrap and hicache_events are cross-rank sync points measured at the
    outer call, not attributed to local CPU work.
  * dp_gather_ms comes from the existing MLPSyncBatchInfo.sync_wait_seconds;
    sglang:dp_attention_sync_wait_seconds is reused, no duplicate metric.
    The value is cleared before each outer call and mlp_sync_seq only
    increments when a gather actually ran, so stale values cannot leak.
  * resumed vs new chunked requests are captured separately: had_chunked is
    recorded at pass start (before process_prefill_chunk/admission can
    consume the last chunk) and new_chunk from adder.new_chunked_req. This
    fixes both the first-chunk and last-chunk mislabels.

Opt-in:
  * SGLANG_BATCH_STAGE_TIMING=1 (default 0);
    SGLANG_BATCH_PASS_SPAN=1 / SGLANG_BATCH_PASS_SPAN_SAMPLE=0.05.
  * When off every hook is one module-level bool check; no logging, no CUDA
    calls, no new collectives.

Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py
covers first/middle/last chunk labelling, all-prefetch-skip passes without a
batch, stale-gather clearing, rank-deterministic pass-span sampling, and the
disabled no-op path. py_compile passes on all touched files.
This commit is contained in:
2026-11-04 14:53:29 +08:00
parent 2434d09848
commit 85f4f9576c
5 changed files with 620 additions and 0 deletions
@@ -84,6 +84,7 @@ from sglang.srt.mem_cache.common import (
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.req_time_stats import set_schedule_time_batch
from sglang.srt.observability.scheduler_stage_metrics import (
SCHEDULER_STAGE_GET_NEXT_BATCH,
@@ -647,16 +648,30 @@ class SchedulerDisaggregationPrefillMixin:
last_batch: Optional[ScheduleBatch],
should_retry_storage_prefetch: bool = True,
) -> NextBatchPlan:
bt_pass = batch_telemetry.begin_pass()
if bt_pass is not None:
# Capture before process_prefill_chunk()/admission can consume it:
# the last chunk of a resumed request clears self.chunked_req.
batch_telemetry.note_had_chunked(self.chunked_req is not None)
self.process_pending_chunked_abort()
if bt_pass is not None:
_bt_t = time.monotonic()
self._process_hicache_events(
should_retry_storage_prefetch=should_retry_storage_prefetch
)
if bt_pass is not None:
batch_telemetry.add_hicache(time.monotonic() - _bt_t)
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
# Otherwise, it hangs under high concurrency
running_batch.batch_is_full = False
if bt_pass is not None:
_bt_t = time.monotonic()
self.resolve_waiting_queue_bootstrap()
if bt_pass is not None:
batch_telemetry.add_bootstrap(time.monotonic() - _bt_t)
# Skip repeated chunk processing while polling a pending batch.
if last_batch is None or not last_batch.prefill_chunk_processed:
@@ -669,11 +684,21 @@ class SchedulerDisaggregationPrefillMixin:
prefill_plan = self.get_new_batch_prefill(running_batch)
batch = prefill_plan.batch_to_run
running_batch = prefill_plan.running_batch
if bt_pass is not None:
# Drop any gather value left by an earlier pass; only a gather that
# actually runs inside the call below may be attributed to this one.
batch_telemetry.note_dp_gather(None)
_bt_t = time.monotonic()
batch = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(batch)
if bt_pass is not None:
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
if batch:
set_schedule_time_batch(batch)
if bt_pass is not None:
batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank)
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
@torch.no_grad()
+34
View File
@@ -304,6 +304,7 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
from sglang.srt.model_loader.utils import get_resolved_model_impl
from sglang.srt.multiplex.multiplexing_mixin import SchedulerMultiplexMixin
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.metrics_collector import SchedulerMetricsCollector
from sglang.srt.observability.req_time_stats import (
flush_trace_batch,
@@ -3856,8 +3857,17 @@ class Scheduler(
def get_next_batch_to_run(
self, running_batch: ScheduleBatch, last_batch: Optional[ScheduleBatch]
) -> NextBatchPlan:
bt_pass = batch_telemetry.begin_pass()
if bt_pass is not None:
# Capture before the chunked request can be consumed by this pass.
batch_telemetry.note_had_chunked(self.chunked_req is not None)
self.process_pending_chunked_abort()
if bt_pass is not None:
_bt_t = time.monotonic()
self._process_hicache_events()
if bt_pass is not None:
batch_telemetry.add_hicache(time.monotonic() - _bt_t)
if self.dllm_config is not None:
self.dllm_manager.filter_finished_reqs()
@@ -3948,7 +3958,11 @@ class Scheduler(
# Before merging the new batch into running batch:
# 1. All new batches are none -> need_mlp_sync remains true (sync is needed for decode batch).
# 2. All new batches are some (prefill / idle) -> we do not need prepare mlp sync one more time.
if bt_pass is not None:
_bt_t = time.monotonic()
new_batch = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(new_batch)
if bt_pass is not None:
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
need_mlp_sync = new_batch is None
if new_batch is not None:
@@ -3967,9 +3981,13 @@ class Scheduler(
ret = None
# Handle DP attention and log stats
if bt_pass is not None:
_bt_t = time.monotonic()
ret = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(
ret, need_sync=need_mlp_sync
)
if bt_pass is not None:
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
# Decode->extend conversion keeps a heterogeneous dp step replayable.
# A converted running batch stays the running batch; next step it comes
# back as last_batch and is filtered in place, not merged with itself.
@@ -3984,6 +4002,9 @@ class Scheduler(
if ret:
set_schedule_time_batch(ret)
if bt_pass is not None:
batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank)
return NextBatchPlan(batch_to_run=ret, running_batch=running_batch)
def get_num_allocatable_reqs(
@@ -4168,6 +4189,7 @@ class Scheduler(
# candidate: the resumed chunk or mixed decode rows can spend it.
if adder.budget_state() == AddReqResult.OTHER:
break
batch_telemetry.mark_candidate()
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
continue
@@ -4206,6 +4228,7 @@ class Scheduler(
req.time_stats.observe_storage_prefetch_ready(prefetch_done)
if not prefetch_done:
# skip staging requests that are ongoing prefetch
batch_telemetry.mark_skipped_prefetch()
continue
# Pop the L3-loaded span. Unified cache exposes its absolute
# start so cache-mode L2/L3 attribution survives L3-tail eviction.
@@ -4219,7 +4242,11 @@ class Scheduler(
# marks the staged span below once it is surfaced.
req.host_hit_is_storage = False
if batch_telemetry.stage_timing_enabled():
_bt_t = time.monotonic()
req.init_next_round_input(self.tree_cache)
if batch_telemetry.stage_timing_enabled():
batch_telemetry.add_match(time.monotonic() - _bt_t)
if self.enable_hicache_storage and (
self._prefetch_after_device_hit_loss(req)
):
@@ -4229,11 +4256,15 @@ class Scheduler(
req.time_stats.observe_storage_staging_ready(staging_ready)
if not staging_ready:
continue
if batch_telemetry.stage_timing_enabled():
_bt_t = time.monotonic()
res = adder.add_one_req(
req,
has_chunked_req=(self.chunked_req is not None),
truncation_align_size=self.truncation_align_size,
)
if batch_telemetry.stage_timing_enabled():
batch_telemetry.add_adder(time.monotonic() - _bt_t)
if self.enable_lora:
running_loras.add(req.lora_id)
@@ -4271,6 +4302,7 @@ class Scheduler(
# though the request they made room for was not admitted.
for req in adder.preempt_list:
self._add_request_to_queue(req)
batch_telemetry.set_admitted(0)
return None, running_batch
can_run_set = set(can_run_list)
@@ -4373,6 +4405,8 @@ class Scheduler(
else:
new_batch.decoding_reqs = None
batch_telemetry.set_admitted(len(can_run_list))
batch_telemetry.note_new_chunk(adder.new_chunked_req is not None)
return new_batch, running_batch
def can_schedule_lora_req(
@@ -723,6 +723,12 @@ def prepare_mlp_sync_batch_raw(
# No batch yet: this iteration may gather again, or end in on_idle.
sync_wait_carry.seconds = sync_wait_seconds
# Stage timing: publish this pass's DP all-gather wait (None = no gather ran).
if metadata_ready and not skip_all_gather and num_dp_ranks > 1:
batch_telemetry.note_dp_gather(mlp_sync_info.sync_wait_seconds)
else:
batch_telemetry.note_dp_gather(None)
return local_batch
@@ -35,6 +35,7 @@ from __future__ import annotations
import os
import random
import threading
import time
from typing import Any, Iterable, Optional
@@ -379,3 +380,363 @@ def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str =
)
except Exception:
pass
# ---------------------------------------------------------------------------
# Per-pass stage timing (SGLANG_BATCH_STAGE_TIMING=1) -- P (disagg prefill).
#
# One sample per get_next_disagg_prefill_batch_to_run() pass, including passes
# that produce no batch. Disabled by default; when off every hook below is a
# single module-level bool check. No CUDA calls, no per-pass logging.
#
# Cross-rank analysis uses a dedicated, independently sampled span
# ``scheduler.prefill.pass`` emitted from end_pass(): the sampling decision is a
# pure function of ``mlp_sync_seq`` (identical on every DP rank that took part
# in the same gather), so a sampled gather round is sampled on *all* ranks --
# unlike the per-rank random sampling of the batch span. Idle ranks and ranks
# with no local batch therefore still emit, which is exactly the case the
# skew analysis needs.
# ---------------------------------------------------------------------------
_STAGE_TIMING = os.getenv("SGLANG_BATCH_STAGE_TIMING", "0").strip().lower() not in (
"",
"0",
"false",
"no",
"off",
)
_STAGE_LABELS = ["phase", "dp_rank"]
_STAGE_BUCKETS = (
1e-5,
3e-5,
1e-4,
3e-4,
1e-3,
3e-3,
1e-2,
3e-2,
1e-1,
3e-1,
1.0,
3.0,
)
_STAGE_HISTS = None
_STAGE_COUNTERS = None
_STAGE_CHILDREN: dict = {}
_STAGE_TLS = threading.local()
# Set by the DP all-gather path (dp_attn.prepare_mlp_sync_batch_raw).
_LAST_DP_GATHER = None
_DP_GATHER_SEQ = 0
_PASS_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
"SGLANG_BATCH_PASS_SPAN", "1"
).strip().lower() not in ("", "0", "false", "no", "off")
_PASS_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_PASS_SPAN_SAMPLE", "0.05"))
_STAGE_HIST_DOCS = {
"get_next": "Wall time of one get_next_disagg_prefill_batch_to_run pass",
"match": "Per-pass sum of init_next_round_input (prefix match) over candidates",
"adder": "Per-pass sum of PrefillAdder.add_one_req (budget/admission)",
"hicache_events": "Per-pass time in _process_hicache_events (includes its collectives)",
"bootstrap": "Per-pass time in resolve_waiting_queue_bootstrap (poll + all-reduce)",
"mlp_sync": "Per-pass outer maybe_prepare_mlp_sync_batch time (not pure gather wait)",
"other": "get_next total minus the instrumented segments",
}
def stage_timing_enabled() -> bool:
return _STAGE_TIMING
def pass_span_enabled() -> bool:
return _PASS_SPAN_ENABLED
def pass_span_decision(seq: int) -> bool:
"""Deterministic, rank-identical sampling decision for one gather round.
``seq`` is the DP all-gather sequence number, which is the same on every
rank that participated in that round. Therefore one decision covers the
whole round, instead of four independent ``random.random()`` draws.
"""
if not _PASS_SPAN_ENABLED or seq <= 0:
return False
if _PASS_SPAN_SAMPLE >= 1.0:
return True
if _PASS_SPAN_SAMPLE <= 0.0:
return False
return (seq * 2654435761) % 10000 < int(_PASS_SPAN_SAMPLE * 10000)
def _ensure_stage_metrics():
global _STAGE_HISTS, _STAGE_COUNTERS
if _STAGE_HISTS is None:
from prometheus_client import Counter, Histogram
_STAGE_HISTS = {
name: Histogram(
f"sglang:batch_prefill_{name}_seconds",
doc,
_STAGE_LABELS,
buckets=_STAGE_BUCKETS,
)
for name, doc in _STAGE_HIST_DOCS.items()
}
_STAGE_COUNTERS = {
"candidates": Counter(
"sglang:batch_prefill_candidates_total",
"Requests examined by the per-pass admission candidate loop",
_STAGE_LABELS,
),
"skipped_prefetch": Counter(
"sglang:batch_prefill_skipped_prefetch_total",
"Candidates skipped because their L2/L3 prefetch was not ready",
_STAGE_LABELS,
),
"admitted": Counter(
"sglang:batch_prefill_admitted_total",
"Requests admitted into can_run_list by one pass",
_STAGE_LABELS,
),
"resumed_chunk": Counter(
"sglang:batch_prefill_resumed_chunk_total",
"Passes whose batch continues a chunked-prefill request "
"(including its last chunk)",
_STAGE_LABELS,
),
"new_chunk": Counter(
"sglang:batch_prefill_new_chunk_total",
"Passes that start a new chunked-prefill request",
_STAGE_LABELS,
),
}
return _STAGE_HISTS, _STAGE_COUNTERS
class PassTimings:
__slots__ = (
"phase",
"t0",
"match",
"adder",
"hicache",
"bootstrap",
"mlp_sync",
"dp_gather",
"candidates",
"skipped_prefetch",
"admitted",
"had_chunked",
"new_chunk",
"seq",
)
def __init__(self, phase: str):
self.phase = phase
self.t0 = time.monotonic()
self.match = 0.0
self.adder = 0.0
self.hicache = 0.0
self.bootstrap = 0.0
self.mlp_sync = 0.0
self.dp_gather = None
self.candidates = 0
self.skipped_prefetch = 0
self.admitted = 0
self.had_chunked = False
self.new_chunk = False
self.seq = 0
def begin_pass(phase: str = "prefill") -> Optional[PassTimings]:
"""Start a new pass. Returns None when stage timing is disabled."""
if not _STAGE_TIMING:
return None
timings = PassTimings(phase)
_STAGE_TLS.current = timings
return timings
def _current() -> Optional[PassTimings]:
if not _STAGE_TIMING:
return None
return getattr(_STAGE_TLS, "current", None)
def add_match(seconds: float) -> None:
t = _current()
if t is not None:
t.match += seconds
def add_adder(seconds: float) -> None:
t = _current()
if t is not None:
t.adder += seconds
def add_hicache(seconds: float) -> None:
t = _current()
if t is not None:
t.hicache += seconds
def add_bootstrap(seconds: float) -> None:
t = _current()
if t is not None:
t.bootstrap += seconds
def mark_candidate(n: int = 1) -> None:
t = _current()
if t is not None:
t.candidates += n
def mark_skipped_prefetch(n: int = 1) -> None:
t = _current()
if t is not None:
t.skipped_prefetch += n
def set_admitted(n: int) -> None:
t = _current()
if t is not None:
t.admitted = int(n)
def note_had_chunked(flag: bool) -> None:
"""Record whether a chunked request was already in flight when the pass began.
Must be called at pass start, before process_prefill_chunk()/admission can
consume the chunked request (the last chunk clears self.chunked_req).
"""
t = _current()
if t is not None:
t.had_chunked = bool(flag)
def note_new_chunk(flag: bool) -> None:
"""Record whether this pass started a new chunked request (first chunk)."""
t = _current()
if t is not None:
t.new_chunk = bool(flag)
def note_dp_gather(seconds: Optional[float]) -> None:
"""Called by the DP all-gather path; None means no gather ran this pass."""
global _LAST_DP_GATHER, _DP_GATHER_SEQ
if not _STAGE_TIMING:
return
if seconds is None:
_LAST_DP_GATHER = None
return
_LAST_DP_GATHER = float(seconds)
_DP_GATHER_SEQ += 1
def _take_dp_gather():
global _LAST_DP_GATHER
value = _LAST_DP_GATHER
_LAST_DP_GATHER = None
return value
def add_mlp_sync(seconds: float) -> None:
t = _current()
if t is None:
return
t.mlp_sync += seconds
gather = _take_dp_gather()
if gather is not None:
t.dp_gather = gather
t.seq = _DP_GATHER_SEQ
def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None:
if not pass_span_decision(t.seq):
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.prefill.pass")
end_ns = time.time_ns()
start_ns = end_ns - int(total * 1e9)
tracer.start_span(
"scheduler.prefill.pass",
start_time=start_ns,
attributes={
"sglang.phase": t.phase,
"sglang.dp_rank": int(dp_rank),
"sglang.mlp_sync_seq": int(t.seq),
"sglang.get_next_ms": total * 1e3,
"sglang.match_ms": t.match * 1e3,
"sglang.adder_ms": t.adder * 1e3,
"sglang.hicache_ms": t.hicache * 1e3,
"sglang.bootstrap_ms": t.bootstrap * 1e3,
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
"sglang.other_ms": other * 1e3,
"sglang.candidates": int(t.candidates),
"sglang.skipped_prefetch": int(t.skipped_prefetch),
"sglang.admitted": int(t.admitted),
"sglang.resumed_chunked": int(bool(t.had_chunked)),
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
},
).end(end_time=end_ns)
except Exception:
# Tracing must never break the scheduler loop.
pass
def end_pass(dp_rank: int) -> None:
"""Close the current pass: observe histograms/counters and emit the pass span."""
t = _current()
if t is None:
return
_STAGE_TLS.current = None
try:
total = time.monotonic() - t.t0
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
other = total - measured
if other < 0.0:
other = 0.0
hist, counters = _ensure_stage_metrics()
child_key = (t.phase, dp_rank)
keyed = _STAGE_CHILDREN.get(child_key)
if keyed is None:
keyed = {
name: h.labels(phase=t.phase, dp_rank=str(dp_rank))
for name, h in hist.items()
}
keyed.update(
{
name: c.labels(phase=t.phase, dp_rank=str(dp_rank))
for name, c in counters.items()
}
)
_STAGE_CHILDREN[child_key] = keyed
keyed["get_next"].observe(total)
keyed["match"].observe(t.match)
keyed["adder"].observe(t.adder)
keyed["hicache_events"].observe(t.hicache)
keyed["bootstrap"].observe(t.bootstrap)
keyed["mlp_sync"].observe(t.mlp_sync)
keyed["other"].observe(other)
if t.candidates:
keyed["candidates"].inc(t.candidates)
if t.skipped_prefetch:
keyed["skipped_prefetch"].inc(t.skipped_prefetch)
if t.admitted:
keyed["admitted"].inc(t.admitted)
if t.had_chunked:
keyed["resumed_chunk"].inc(1)
elif t.new_chunk:
keyed["new_chunk"].inc(1)
_emit_pass_span(dp_rank, t, total, other)
except Exception:
# Telemetry must never break the scheduler loop
_STAGE_TLS.current = None
@@ -0,0 +1,194 @@
"""Unit tests for the opt-in per-pass prefill stage timing.
Covers the review points:
* first / middle / last chunk are labelled correctly (resumed vs new),
* a pass with no batch (all prefetch-skipped) still records L1/L3,
* a stale DP gather value is never attributed to a later pass,
* the pass-span sampling decision is deterministic for a given
``mlp_sync_seq`` (rank-identical), unlike per-rank random sampling.
"""
import importlib.util
import os
import pathlib
import unittest
from unittest import mock
from prometheus_client import CollectorRegistry, Counter, Histogram
def _module_path():
"""Resolve the module under test without importing the sglang package."""
p = (
pathlib.Path(__file__).resolve().parents[4]
/ "python/sglang/srt/observability/batch_telemetry.py"
)
if p.exists():
return p
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
return pathlib.Path(spec.origin)
def _reload(**env):
"""Load a fresh module instance with the requested env (no reload/sys.modules)."""
base = {
"SGLANG_BATCH_STAGE_TIMING": "1",
"SGLANG_BATCH_PASS_SPAN": "0",
"SGLANG_BATCH_PASS_SPAN_SAMPLE": "0.05",
}
base.update(env)
with mock.patch.dict(os.environ, base):
spec = importlib.util.spec_from_file_location(
"batch_telemetry_t", _module_path()
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
PHASE = "prefill"
RANK = 0
def _install_private_metrics(module):
"""Give the module a private registry so reloads do not collide."""
reg = CollectorRegistry()
module._STAGE_HISTS = {
name: Histogram(f"test_{name}", "", ["phase", "dp_rank"], registry=reg)
for name in module._STAGE_HIST_DOCS
}
module._STAGE_COUNTERS = {
name: Counter(f"test_{name}", "", ["phase", "dp_rank"], registry=reg)
for name in (
"candidates",
"skipped_prefetch",
"admitted",
"resumed_chunk",
"new_chunk",
)
}
return reg
def _counter(module, name):
child = module._STAGE_COUNTERS[name].labels(phase=PHASE, dp_rank=str(RANK))
return child._value.get()
def _hist_sum(module, name):
child = module._STAGE_HISTS[name].labels(phase=PHASE, dp_rank=str(RANK))
return child._sum.get()
class TestStageTiming(unittest.TestCase):
def setUp(self):
self.bt = _reload()
_install_private_metrics(self.bt)
def tearDown(self):
os.environ.pop("SGLANG_BATCH_STAGE_TIMING", None)
_reload(SGLANG_BATCH_STAGE_TIMING="0")
def _run_pass(self, had_chunked=False, new_chunk=False, gather=0.005):
bt = self.bt
bt.begin_pass()
bt.note_had_chunked(had_chunked)
bt.note_new_chunk(new_chunk)
if gather is not None:
bt.note_dp_gather(gather)
bt.add_mlp_sync(0.001)
bt.end_pass(dp_rank=RANK)
def test_all_segments_and_counters_recorded(self):
bt = self.bt
bt.begin_pass()
bt.mark_candidate()
bt.mark_candidate()
bt.mark_skipped_prefetch()
bt.add_hicache(0.001)
bt.add_bootstrap(0.002)
bt.add_match(0.003)
bt.add_adder(0.004)
bt.note_dp_gather(0.005)
bt.add_mlp_sync(0.006)
bt.set_admitted(2)
bt.end_pass(dp_rank=RANK)
for name in bt._STAGE_HIST_DOCS:
self.assertGreater(
_hist_sum(bt, name) + (1 if name == "other" else 0),
0.0,
f"{name} not observed",
)
self.assertEqual(_counter(bt, "candidates"), 2)
self.assertEqual(_counter(bt, "skipped_prefetch"), 1)
self.assertEqual(_counter(bt, "admitted"), 2)
def test_chunk_first_middle_last(self):
bt = self.bt
self._run_pass(had_chunked=False, new_chunk=True) # first chunk
self.assertEqual(_counter(bt, "new_chunk"), 1)
self.assertEqual(_counter(bt, "resumed_chunk"), 0)
self._run_pass(had_chunked=True, new_chunk=False) # middle chunk
self.assertEqual(_counter(bt, "new_chunk"), 1)
self.assertEqual(_counter(bt, "resumed_chunk"), 1)
# Last chunk: had_chunked is captured before admission consumes it.
self._run_pass(had_chunked=True, new_chunk=False)
self.assertEqual(_counter(bt, "resumed_chunk"), 2)
def test_no_batch_pass_still_records(self):
bt = self.bt
before = _hist_sum(bt, "get_next")
self._run_pass(had_chunked=False, new_chunk=False) # no batch formed
self.assertGreater(_hist_sum(bt, "get_next"), before)
self.assertEqual(_counter(bt, "skipped_prefetch"), 0)
def test_stale_gather_not_leaked(self):
bt = self.bt
bt.begin_pass()
bt.note_dp_gather(0.004)
bt.add_mlp_sync(0.001)
bt.end_pass(dp_rank=RANK)
bt.begin_pass()
bt.add_mlp_sync(0.001) # no gather noted for this pass
t = bt._current()
self.assertIsNone(t.dp_gather)
self.assertEqual(t.seq, 0)
bt.end_pass(dp_rank=RANK)
def test_pass_span_sampling_is_rank_deterministic(self):
bt = self.bt
p5 = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="0.05")
_install_private_metrics(p5)
decisions = [p5.pass_span_decision(seq) for seq in range(1, 200)]
self.assertTrue(any(decisions))
self.assertFalse(all(decisions))
p5b = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="0.05")
self.assertEqual(
[p5b.pass_span_decision(seq) for seq in range(1, 200)], decisions
)
p0 = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="0.0")
self.assertFalse(any(p0.pass_span_decision(seq) for seq in range(1, 200)))
p1 = _reload(SGLANG_BATCH_PASS_SPAN="1", SGLANG_BATCH_PASS_SPAN_SAMPLE="1.0")
self.assertTrue(all(p1.pass_span_decision(seq) for seq in range(1, 20)))
self.assertFalse(p1.pass_span_decision(0))
def test_disabled_is_noop(self):
off = _reload(SGLANG_BATCH_STAGE_TIMING="0")
self.assertFalse(off.stage_timing_enabled())
self.assertIsNone(off.begin_pass())
self.assertIsNone(off._current())
off.add_match(1.0)
off.mark_candidate()
off.note_had_chunked(True)
off.end_pass(dp_rank=RANK)
self.assertFalse(off.pass_span_decision(1))
if __name__ == "__main__":
unittest.main()