scheduler: add opt-in per-pass prefill stage timing (SGLANG_BATCH_STAGE_TIMING)
Motivation: on the PD prefill node the wall-clock budget is dominated by
get_next_batch_to_run (~50% of the hour on 2026-11-02 16:00-17:00), and
forward_overlap=partial is only 21-87 s/h, i.e. it does not overlap the GPU
forward. We know the total but not which segment inside it costs: radix
match, PrefillAdder admission, the HiCache event collective, the bootstrap
all-reduce, or the DP metadata gather.
Measurement design (one sample per scheduling pass, including passes that
produce no batch; see handoff/prefill-batch-stage-timing-20261103.md v2/v3):
L1 7 per-pass histograms:
sglang:batch_prefill_{get_next,match,adder,hicache_events,
bootstrap,mlp_sync,other}_seconds
other = total - (match + adder + hicache + bootstrap + mlp_sync),
clamped to >= 0. Nested stages are not double counted.
L3 5 per-pass counters:
candidates / skipped_prefetch / admitted / resumed_chunk / new_chunk
L2 a dedicated per-pass span scheduler.prefill.pass (not the batch span),
emitted from end_pass() on every pass. Its sampling decision is a pure
function of mlp_sync_seq, so a sampled gather round is sampled on all
DP ranks. Idle ranks and ranks without a local batch still emit, which
is exactly what the cross-rank skew analysis needs. No links.
Hooks are installed in both scheduling loops:
* get_next_disagg_prefill_batch_to_run() for --disaggregation-mode prefill
(hicache / bootstrap / mlp_sync segments), and
* get_next_batch_to_run() for standalone prefill (hicache / mlp_sync),
so the instrumentation can be validated on ds41-prefill-dp4 without a decode
peer. match/adder/counters live in the shared get_new_batch_prefill path.
Semantics:
* match times only init_next_round_input; check_prefetch_progress counts as
skipped_prefetch, never as match.
* bootstrap and hicache_events are cross-rank sync points measured at the
outer call, not attributed to local CPU work.
* dp_gather_ms comes from the existing MLPSyncBatchInfo.sync_wait_seconds;
sglang:dp_attention_sync_wait_seconds is reused, no duplicate metric.
The value is cleared before each outer call and mlp_sync_seq only
increments when a gather actually ran, so stale values cannot leak.
* resumed vs new chunked requests are captured separately: had_chunked is
recorded at pass start (before process_prefill_chunk/admission can
consume the last chunk) and new_chunk from adder.new_chunked_req. This
fixes both the first-chunk and last-chunk mislabels.
Opt-in:
* SGLANG_BATCH_STAGE_TIMING=1 (default 0);
SGLANG_BATCH_PASS_SPAN=1 / SGLANG_BATCH_PASS_SPAN_SAMPLE=0.05.
* When off every hook is one module-level bool check; no logging, no CUDA
calls, no new collectives.
Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py
covers first/middle/last chunk labelling, all-prefetch-skip passes without a
batch, stale-gather clearing, rank-deterministic pass-span sampling, and the
disabled no-op path. py_compile passes on all touched files.
This commit is contained in:
@@ -84,6 +84,7 @@ from sglang.srt.mem_cache.common import (
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.req_time_stats import set_schedule_time_batch
|
||||
from sglang.srt.observability.scheduler_stage_metrics import (
|
||||
SCHEDULER_STAGE_GET_NEXT_BATCH,
|
||||
@@ -647,16 +648,30 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
last_batch: Optional[ScheduleBatch],
|
||||
should_retry_storage_prefetch: bool = True,
|
||||
) -> NextBatchPlan:
|
||||
bt_pass = batch_telemetry.begin_pass()
|
||||
if bt_pass is not None:
|
||||
# Capture before process_prefill_chunk()/admission can consume it:
|
||||
# the last chunk of a resumed request clears self.chunked_req.
|
||||
batch_telemetry.note_had_chunked(self.chunked_req is not None)
|
||||
|
||||
self.process_pending_chunked_abort()
|
||||
if bt_pass is not None:
|
||||
_bt_t = time.monotonic()
|
||||
self._process_hicache_events(
|
||||
should_retry_storage_prefetch=should_retry_storage_prefetch
|
||||
)
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.add_hicache(time.monotonic() - _bt_t)
|
||||
|
||||
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
|
||||
# Otherwise, it hangs under high concurrency
|
||||
running_batch.batch_is_full = False
|
||||
|
||||
if bt_pass is not None:
|
||||
_bt_t = time.monotonic()
|
||||
self.resolve_waiting_queue_bootstrap()
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.add_bootstrap(time.monotonic() - _bt_t)
|
||||
|
||||
# Skip repeated chunk processing while polling a pending batch.
|
||||
if last_batch is None or not last_batch.prefill_chunk_processed:
|
||||
@@ -669,11 +684,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
prefill_plan = self.get_new_batch_prefill(running_batch)
|
||||
batch = prefill_plan.batch_to_run
|
||||
running_batch = prefill_plan.running_batch
|
||||
if bt_pass is not None:
|
||||
# Drop any gather value left by an earlier pass; only a gather that
|
||||
# actually runs inside the call below may be attributed to this one.
|
||||
batch_telemetry.note_dp_gather(None)
|
||||
_bt_t = time.monotonic()
|
||||
batch = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(batch)
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
|
||||
|
||||
if batch:
|
||||
set_schedule_time_batch(batch)
|
||||
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank)
|
||||
|
||||
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
|
||||
|
||||
@torch.no_grad()
|
||||
|
||||
@@ -304,6 +304,7 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
|
||||
from sglang.srt.model_loader.utils import get_resolved_model_impl
|
||||
from sglang.srt.multiplex.multiplexing_mixin import SchedulerMultiplexMixin
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.metrics_collector import SchedulerMetricsCollector
|
||||
from sglang.srt.observability.req_time_stats import (
|
||||
flush_trace_batch,
|
||||
@@ -3856,8 +3857,17 @@ class Scheduler(
|
||||
def get_next_batch_to_run(
|
||||
self, running_batch: ScheduleBatch, last_batch: Optional[ScheduleBatch]
|
||||
) -> NextBatchPlan:
|
||||
bt_pass = batch_telemetry.begin_pass()
|
||||
if bt_pass is not None:
|
||||
# Capture before the chunked request can be consumed by this pass.
|
||||
batch_telemetry.note_had_chunked(self.chunked_req is not None)
|
||||
|
||||
self.process_pending_chunked_abort()
|
||||
if bt_pass is not None:
|
||||
_bt_t = time.monotonic()
|
||||
self._process_hicache_events()
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.add_hicache(time.monotonic() - _bt_t)
|
||||
|
||||
if self.dllm_config is not None:
|
||||
self.dllm_manager.filter_finished_reqs()
|
||||
@@ -3948,7 +3958,11 @@ class Scheduler(
|
||||
# Before merging the new batch into running batch:
|
||||
# 1. All new batches are none -> need_mlp_sync remains true (sync is needed for decode batch).
|
||||
# 2. All new batches are some (prefill / idle) -> we do not need prepare mlp sync one more time.
|
||||
if bt_pass is not None:
|
||||
_bt_t = time.monotonic()
|
||||
new_batch = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(new_batch)
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
|
||||
need_mlp_sync = new_batch is None
|
||||
|
||||
if new_batch is not None:
|
||||
@@ -3967,9 +3981,13 @@ class Scheduler(
|
||||
ret = None
|
||||
|
||||
# Handle DP attention and log stats
|
||||
if bt_pass is not None:
|
||||
_bt_t = time.monotonic()
|
||||
ret = self.dp_attn_adapter.maybe_prepare_mlp_sync_batch(
|
||||
ret, need_sync=need_mlp_sync
|
||||
)
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
|
||||
# Decode->extend conversion keeps a heterogeneous dp step replayable.
|
||||
# A converted running batch stays the running batch; next step it comes
|
||||
# back as last_batch and is filtered in place, not merged with itself.
|
||||
@@ -3984,6 +4002,9 @@ class Scheduler(
|
||||
if ret:
|
||||
set_schedule_time_batch(ret)
|
||||
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank)
|
||||
|
||||
return NextBatchPlan(batch_to_run=ret, running_batch=running_batch)
|
||||
|
||||
def get_num_allocatable_reqs(
|
||||
@@ -4168,6 +4189,7 @@ class Scheduler(
|
||||
# candidate: the resumed chunk or mixed decode rows can spend it.
|
||||
if adder.budget_state() == AddReqResult.OTHER:
|
||||
break
|
||||
batch_telemetry.mark_candidate()
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
|
||||
@@ -4206,6 +4228,7 @@ class Scheduler(
|
||||
req.time_stats.observe_storage_prefetch_ready(prefetch_done)
|
||||
if not prefetch_done:
|
||||
# skip staging requests that are ongoing prefetch
|
||||
batch_telemetry.mark_skipped_prefetch()
|
||||
continue
|
||||
# Pop the L3-loaded span. Unified cache exposes its absolute
|
||||
# start so cache-mode L2/L3 attribution survives L3-tail eviction.
|
||||
@@ -4219,7 +4242,11 @@ class Scheduler(
|
||||
# marks the staged span below once it is surfaced.
|
||||
req.host_hit_is_storage = False
|
||||
|
||||
if batch_telemetry.stage_timing_enabled():
|
||||
_bt_t = time.monotonic()
|
||||
req.init_next_round_input(self.tree_cache)
|
||||
if batch_telemetry.stage_timing_enabled():
|
||||
batch_telemetry.add_match(time.monotonic() - _bt_t)
|
||||
if self.enable_hicache_storage and (
|
||||
self._prefetch_after_device_hit_loss(req)
|
||||
):
|
||||
@@ -4229,11 +4256,15 @@ class Scheduler(
|
||||
req.time_stats.observe_storage_staging_ready(staging_ready)
|
||||
if not staging_ready:
|
||||
continue
|
||||
if batch_telemetry.stage_timing_enabled():
|
||||
_bt_t = time.monotonic()
|
||||
res = adder.add_one_req(
|
||||
req,
|
||||
has_chunked_req=(self.chunked_req is not None),
|
||||
truncation_align_size=self.truncation_align_size,
|
||||
)
|
||||
if batch_telemetry.stage_timing_enabled():
|
||||
batch_telemetry.add_adder(time.monotonic() - _bt_t)
|
||||
|
||||
if self.enable_lora:
|
||||
running_loras.add(req.lora_id)
|
||||
@@ -4271,6 +4302,7 @@ class Scheduler(
|
||||
# though the request they made room for was not admitted.
|
||||
for req in adder.preempt_list:
|
||||
self._add_request_to_queue(req)
|
||||
batch_telemetry.set_admitted(0)
|
||||
return None, running_batch
|
||||
|
||||
can_run_set = set(can_run_list)
|
||||
@@ -4373,6 +4405,8 @@ class Scheduler(
|
||||
else:
|
||||
new_batch.decoding_reqs = None
|
||||
|
||||
batch_telemetry.set_admitted(len(can_run_list))
|
||||
batch_telemetry.note_new_chunk(adder.new_chunked_req is not None)
|
||||
return new_batch, running_batch
|
||||
|
||||
def can_schedule_lora_req(
|
||||
|
||||
@@ -723,6 +723,12 @@ def prepare_mlp_sync_batch_raw(
|
||||
# No batch yet: this iteration may gather again, or end in on_idle.
|
||||
sync_wait_carry.seconds = sync_wait_seconds
|
||||
|
||||
# Stage timing: publish this pass's DP all-gather wait (None = no gather ran).
|
||||
if metadata_ready and not skip_all_gather and num_dp_ranks > 1:
|
||||
batch_telemetry.note_dp_gather(mlp_sync_info.sync_wait_seconds)
|
||||
else:
|
||||
batch_telemetry.note_dp_gather(None)
|
||||
|
||||
return local_batch
|
||||
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
@@ -379,3 +380,363 @@ def observe_dp_step(tokens, dp_rank: int, modes=None, health=None, phase: str =
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-pass stage timing (SGLANG_BATCH_STAGE_TIMING=1) -- P (disagg prefill).
|
||||
#
|
||||
# One sample per get_next_disagg_prefill_batch_to_run() pass, including passes
|
||||
# that produce no batch. Disabled by default; when off every hook below is a
|
||||
# single module-level bool check. No CUDA calls, no per-pass logging.
|
||||
#
|
||||
# Cross-rank analysis uses a dedicated, independently sampled span
|
||||
# ``scheduler.prefill.pass`` emitted from end_pass(): the sampling decision is a
|
||||
# pure function of ``mlp_sync_seq`` (identical on every DP rank that took part
|
||||
# in the same gather), so a sampled gather round is sampled on *all* ranks --
|
||||
# unlike the per-rank random sampling of the batch span. Idle ranks and ranks
|
||||
# with no local batch therefore still emit, which is exactly the case the
|
||||
# skew analysis needs.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_STAGE_TIMING = os.getenv("SGLANG_BATCH_STAGE_TIMING", "0").strip().lower() not in (
|
||||
"",
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
)
|
||||
|
||||
_STAGE_LABELS = ["phase", "dp_rank"]
|
||||
_STAGE_BUCKETS = (
|
||||
1e-5,
|
||||
3e-5,
|
||||
1e-4,
|
||||
3e-4,
|
||||
1e-3,
|
||||
3e-3,
|
||||
1e-2,
|
||||
3e-2,
|
||||
1e-1,
|
||||
3e-1,
|
||||
1.0,
|
||||
3.0,
|
||||
)
|
||||
|
||||
_STAGE_HISTS = None
|
||||
_STAGE_COUNTERS = None
|
||||
_STAGE_CHILDREN: dict = {}
|
||||
_STAGE_TLS = threading.local()
|
||||
|
||||
# Set by the DP all-gather path (dp_attn.prepare_mlp_sync_batch_raw).
|
||||
_LAST_DP_GATHER = None
|
||||
_DP_GATHER_SEQ = 0
|
||||
|
||||
_PASS_SPAN_ENABLED = _STAGE_TIMING and os.getenv(
|
||||
"SGLANG_BATCH_PASS_SPAN", "1"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
_PASS_SPAN_SAMPLE = float(os.getenv("SGLANG_BATCH_PASS_SPAN_SAMPLE", "0.05"))
|
||||
|
||||
_STAGE_HIST_DOCS = {
|
||||
"get_next": "Wall time of one get_next_disagg_prefill_batch_to_run pass",
|
||||
"match": "Per-pass sum of init_next_round_input (prefix match) over candidates",
|
||||
"adder": "Per-pass sum of PrefillAdder.add_one_req (budget/admission)",
|
||||
"hicache_events": "Per-pass time in _process_hicache_events (includes its collectives)",
|
||||
"bootstrap": "Per-pass time in resolve_waiting_queue_bootstrap (poll + all-reduce)",
|
||||
"mlp_sync": "Per-pass outer maybe_prepare_mlp_sync_batch time (not pure gather wait)",
|
||||
"other": "get_next total minus the instrumented segments",
|
||||
}
|
||||
|
||||
|
||||
def stage_timing_enabled() -> bool:
|
||||
return _STAGE_TIMING
|
||||
|
||||
|
||||
def pass_span_enabled() -> bool:
|
||||
return _PASS_SPAN_ENABLED
|
||||
|
||||
|
||||
def pass_span_decision(seq: int) -> bool:
|
||||
"""Deterministic, rank-identical sampling decision for one gather round.
|
||||
|
||||
``seq`` is the DP all-gather sequence number, which is the same on every
|
||||
rank that participated in that round. Therefore one decision covers the
|
||||
whole round, instead of four independent ``random.random()`` draws.
|
||||
"""
|
||||
if not _PASS_SPAN_ENABLED or seq <= 0:
|
||||
return False
|
||||
if _PASS_SPAN_SAMPLE >= 1.0:
|
||||
return True
|
||||
if _PASS_SPAN_SAMPLE <= 0.0:
|
||||
return False
|
||||
return (seq * 2654435761) % 10000 < int(_PASS_SPAN_SAMPLE * 10000)
|
||||
|
||||
|
||||
def _ensure_stage_metrics():
|
||||
global _STAGE_HISTS, _STAGE_COUNTERS
|
||||
if _STAGE_HISTS is None:
|
||||
from prometheus_client import Counter, Histogram
|
||||
|
||||
_STAGE_HISTS = {
|
||||
name: Histogram(
|
||||
f"sglang:batch_prefill_{name}_seconds",
|
||||
doc,
|
||||
_STAGE_LABELS,
|
||||
buckets=_STAGE_BUCKETS,
|
||||
)
|
||||
for name, doc in _STAGE_HIST_DOCS.items()
|
||||
}
|
||||
_STAGE_COUNTERS = {
|
||||
"candidates": Counter(
|
||||
"sglang:batch_prefill_candidates_total",
|
||||
"Requests examined by the per-pass admission candidate loop",
|
||||
_STAGE_LABELS,
|
||||
),
|
||||
"skipped_prefetch": Counter(
|
||||
"sglang:batch_prefill_skipped_prefetch_total",
|
||||
"Candidates skipped because their L2/L3 prefetch was not ready",
|
||||
_STAGE_LABELS,
|
||||
),
|
||||
"admitted": Counter(
|
||||
"sglang:batch_prefill_admitted_total",
|
||||
"Requests admitted into can_run_list by one pass",
|
||||
_STAGE_LABELS,
|
||||
),
|
||||
"resumed_chunk": Counter(
|
||||
"sglang:batch_prefill_resumed_chunk_total",
|
||||
"Passes whose batch continues a chunked-prefill request "
|
||||
"(including its last chunk)",
|
||||
_STAGE_LABELS,
|
||||
),
|
||||
"new_chunk": Counter(
|
||||
"sglang:batch_prefill_new_chunk_total",
|
||||
"Passes that start a new chunked-prefill request",
|
||||
_STAGE_LABELS,
|
||||
),
|
||||
}
|
||||
return _STAGE_HISTS, _STAGE_COUNTERS
|
||||
|
||||
|
||||
class PassTimings:
|
||||
__slots__ = (
|
||||
"phase",
|
||||
"t0",
|
||||
"match",
|
||||
"adder",
|
||||
"hicache",
|
||||
"bootstrap",
|
||||
"mlp_sync",
|
||||
"dp_gather",
|
||||
"candidates",
|
||||
"skipped_prefetch",
|
||||
"admitted",
|
||||
"had_chunked",
|
||||
"new_chunk",
|
||||
"seq",
|
||||
)
|
||||
|
||||
def __init__(self, phase: str):
|
||||
self.phase = phase
|
||||
self.t0 = time.monotonic()
|
||||
self.match = 0.0
|
||||
self.adder = 0.0
|
||||
self.hicache = 0.0
|
||||
self.bootstrap = 0.0
|
||||
self.mlp_sync = 0.0
|
||||
self.dp_gather = None
|
||||
self.candidates = 0
|
||||
self.skipped_prefetch = 0
|
||||
self.admitted = 0
|
||||
self.had_chunked = False
|
||||
self.new_chunk = False
|
||||
self.seq = 0
|
||||
|
||||
|
||||
def begin_pass(phase: str = "prefill") -> Optional[PassTimings]:
|
||||
"""Start a new pass. Returns None when stage timing is disabled."""
|
||||
if not _STAGE_TIMING:
|
||||
return None
|
||||
timings = PassTimings(phase)
|
||||
_STAGE_TLS.current = timings
|
||||
return timings
|
||||
|
||||
|
||||
def _current() -> Optional[PassTimings]:
|
||||
if not _STAGE_TIMING:
|
||||
return None
|
||||
return getattr(_STAGE_TLS, "current", None)
|
||||
|
||||
|
||||
def add_match(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.match += seconds
|
||||
|
||||
|
||||
def add_adder(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.adder += seconds
|
||||
|
||||
|
||||
def add_hicache(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.hicache += seconds
|
||||
|
||||
|
||||
def add_bootstrap(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.bootstrap += seconds
|
||||
|
||||
|
||||
def mark_candidate(n: int = 1) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.candidates += n
|
||||
|
||||
|
||||
def mark_skipped_prefetch(n: int = 1) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.skipped_prefetch += n
|
||||
|
||||
|
||||
def set_admitted(n: int) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.admitted = int(n)
|
||||
|
||||
|
||||
def note_had_chunked(flag: bool) -> None:
|
||||
"""Record whether a chunked request was already in flight when the pass began.
|
||||
|
||||
Must be called at pass start, before process_prefill_chunk()/admission can
|
||||
consume the chunked request (the last chunk clears self.chunked_req).
|
||||
"""
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.had_chunked = bool(flag)
|
||||
|
||||
|
||||
def note_new_chunk(flag: bool) -> None:
|
||||
"""Record whether this pass started a new chunked request (first chunk)."""
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.new_chunk = bool(flag)
|
||||
|
||||
|
||||
def note_dp_gather(seconds: Optional[float]) -> None:
|
||||
"""Called by the DP all-gather path; None means no gather ran this pass."""
|
||||
global _LAST_DP_GATHER, _DP_GATHER_SEQ
|
||||
if not _STAGE_TIMING:
|
||||
return
|
||||
if seconds is None:
|
||||
_LAST_DP_GATHER = None
|
||||
return
|
||||
_LAST_DP_GATHER = float(seconds)
|
||||
_DP_GATHER_SEQ += 1
|
||||
|
||||
|
||||
def _take_dp_gather():
|
||||
global _LAST_DP_GATHER
|
||||
value = _LAST_DP_GATHER
|
||||
_LAST_DP_GATHER = None
|
||||
return value
|
||||
|
||||
|
||||
def add_mlp_sync(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is None:
|
||||
return
|
||||
t.mlp_sync += seconds
|
||||
gather = _take_dp_gather()
|
||||
if gather is not None:
|
||||
t.dp_gather = gather
|
||||
t.seq = _DP_GATHER_SEQ
|
||||
|
||||
|
||||
def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None:
|
||||
if not pass_span_decision(t.seq):
|
||||
return
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.pass")
|
||||
end_ns = time.time_ns()
|
||||
start_ns = end_ns - int(total * 1e9)
|
||||
tracer.start_span(
|
||||
"scheduler.prefill.pass",
|
||||
start_time=start_ns,
|
||||
attributes={
|
||||
"sglang.phase": t.phase,
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.mlp_sync_seq": int(t.seq),
|
||||
"sglang.get_next_ms": total * 1e3,
|
||||
"sglang.match_ms": t.match * 1e3,
|
||||
"sglang.adder_ms": t.adder * 1e3,
|
||||
"sglang.hicache_ms": t.hicache * 1e3,
|
||||
"sglang.bootstrap_ms": t.bootstrap * 1e3,
|
||||
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
|
||||
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
|
||||
"sglang.other_ms": other * 1e3,
|
||||
"sglang.candidates": int(t.candidates),
|
||||
"sglang.skipped_prefetch": int(t.skipped_prefetch),
|
||||
"sglang.admitted": int(t.admitted),
|
||||
"sglang.resumed_chunked": int(bool(t.had_chunked)),
|
||||
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
|
||||
},
|
||||
).end(end_time=end_ns)
|
||||
except Exception:
|
||||
# Tracing must never break the scheduler loop.
|
||||
pass
|
||||
|
||||
|
||||
def end_pass(dp_rank: int) -> None:
|
||||
"""Close the current pass: observe histograms/counters and emit the pass span."""
|
||||
t = _current()
|
||||
if t is None:
|
||||
return
|
||||
_STAGE_TLS.current = None
|
||||
try:
|
||||
total = time.monotonic() - t.t0
|
||||
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
|
||||
other = total - measured
|
||||
if other < 0.0:
|
||||
other = 0.0
|
||||
hist, counters = _ensure_stage_metrics()
|
||||
child_key = (t.phase, dp_rank)
|
||||
keyed = _STAGE_CHILDREN.get(child_key)
|
||||
if keyed is None:
|
||||
keyed = {
|
||||
name: h.labels(phase=t.phase, dp_rank=str(dp_rank))
|
||||
for name, h in hist.items()
|
||||
}
|
||||
keyed.update(
|
||||
{
|
||||
name: c.labels(phase=t.phase, dp_rank=str(dp_rank))
|
||||
for name, c in counters.items()
|
||||
}
|
||||
)
|
||||
_STAGE_CHILDREN[child_key] = keyed
|
||||
keyed["get_next"].observe(total)
|
||||
keyed["match"].observe(t.match)
|
||||
keyed["adder"].observe(t.adder)
|
||||
keyed["hicache_events"].observe(t.hicache)
|
||||
keyed["bootstrap"].observe(t.bootstrap)
|
||||
keyed["mlp_sync"].observe(t.mlp_sync)
|
||||
keyed["other"].observe(other)
|
||||
if t.candidates:
|
||||
keyed["candidates"].inc(t.candidates)
|
||||
if t.skipped_prefetch:
|
||||
keyed["skipped_prefetch"].inc(t.skipped_prefetch)
|
||||
if t.admitted:
|
||||
keyed["admitted"].inc(t.admitted)
|
||||
if t.had_chunked:
|
||||
keyed["resumed_chunk"].inc(1)
|
||||
elif t.new_chunk:
|
||||
keyed["new_chunk"].inc(1)
|
||||
_emit_pass_span(dp_rank, t, total, other)
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop
|
||||
_STAGE_TLS.current = None
|
||||
|
||||
Reference in New Issue
Block a user