observability: split get_next residual into its four phases

Adds the Phase A instrumentation from the async-backup handoff (§7.1):

* PassTimings: pending_abort / chunk / new_batch / set_sched (+ chunk_count)
* prefill.get_next_disagg_prefill_batch_to_run: time
  process_pending_chunked_abort, process_prefill_chunk,
  get_new_batch_prefill and set_schedule_time_batch
* scheduler.prefill.pass gains sglang.{pending_abort,chunk,new_batch,set_sched}_ms,
  sglang.get_next_residual_ms (= other - these four) and sglang.chunk_count;
  the existing sglang.other_ms is unchanged for dashboard compatibility.

~5 monotonic calls per pass; enabled by SGLANG_BATCH_STAGE_TIMING (already on).
This commit is contained in:
2026-11-08 00:20:55 +08:00
parent d6f1e1efbd
commit a5fbe834e8
2 changed files with 67 additions and 2 deletions
@@ -697,8 +697,11 @@ class SchedulerDisaggregationPrefillMixin:
# the last chunk of a resumed request clears self.chunked_req.
batch_telemetry.note_had_chunked(self.chunked_req is not None)
if bt_pass is not None:
_bt_t = time.monotonic()
self.process_pending_chunked_abort()
if bt_pass is not None:
batch_telemetry.add_pending_abort(time.monotonic() - _bt_t)
_bt_t = time.monotonic()
self._process_hicache_events(
should_retry_storage_prefetch=should_retry_storage_prefetch
@@ -717,16 +720,25 @@ class SchedulerDisaggregationPrefillMixin:
batch_telemetry.add_bootstrap(time.monotonic() - _bt_t)
# Skip repeated chunk processing while polling a pending batch.
_chunk_t0 = time.monotonic() if bt_pass is not None else None
_had_chunk = self.chunked_req is not None
if last_batch is None or not last_batch.prefill_chunk_processed:
self.process_prefill_chunk(
last_batch=last_batch, running_batch=running_batch
)
if last_batch is not None:
last_batch.prefill_chunk_processed = True
if _chunk_t0 is not None:
batch_telemetry.add_chunk(
time.monotonic() - _chunk_t0, 1 if _had_chunk else 0
)
_nb_t0 = time.monotonic() if bt_pass is not None else None
prefill_plan = self.get_new_batch_prefill(running_batch)
batch = prefill_plan.batch_to_run
running_batch = prefill_plan.running_batch
if _nb_t0 is not None:
batch_telemetry.add_new_batch(time.monotonic() - _nb_t0)
if bt_pass is not None:
# Drop any gather value left by an earlier pass; only a gather that
# actually runs inside the call below may be attributed to this one.
@@ -737,7 +749,10 @@ class SchedulerDisaggregationPrefillMixin:
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
if batch:
_ss_t0 = time.monotonic() if bt_pass is not None else None
set_schedule_time_batch(batch)
if _ss_t0 is not None:
batch_telemetry.add_set_sched(time.monotonic() - _ss_t0)
if bt_pass is not None:
batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank)
@@ -535,6 +535,11 @@ class PassTimings:
"hicache",
"bootstrap",
"mlp_sync",
"pending_abort",
"chunk",
"new_batch",
"set_sched",
"chunk_count",
"dp_gather",
"candidates",
"skipped_prefetch",
@@ -554,6 +559,11 @@ class PassTimings:
self.hicache = 0.0
self.bootstrap = 0.0
self.mlp_sync = 0.0
self.pending_abort = 0.0
self.chunk = 0.0
self.new_batch = 0.0
self.set_sched = 0.0
self.chunk_count = 0
self.dp_gather = None
self.candidates = 0
self.skipped_prefetch = 0
@@ -604,6 +614,31 @@ def add_bootstrap(seconds: float) -> None:
t.bootstrap += seconds
def add_pending_abort(seconds: float) -> None:
t = _current()
if t is not None:
t.pending_abort += seconds
def add_chunk(seconds: float, count: int = 0) -> None:
t = _current()
if t is not None:
t.chunk += seconds
t.chunk_count += int(count)
def add_new_batch(seconds: float) -> None:
t = _current()
if t is not None:
t.new_batch += seconds
def add_set_sched(seconds: float) -> None:
t = _current()
if t is not None:
t.set_sched += seconds
def mark_candidate(n: int = 1) -> None:
t = _current()
if t is not None:
@@ -678,7 +713,9 @@ def add_mlp_sync(seconds: float) -> None:
t.seq = _DP_GATHER_SEQ
def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None:
def _emit_pass_span(
dp_rank: int, t: PassTimings, total: float, other: float, residual: float
) -> None:
if not pass_span_decision(t.seq):
return
loop = getattr(_LOOP_TLS, "current", None)
@@ -709,6 +746,12 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) ->
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
"sglang.other_ms": other * 1e3,
"sglang.pending_abort_ms": t.pending_abort * 1e3,
"sglang.chunk_ms": t.chunk * 1e3,
"sglang.new_batch_ms": t.new_batch * 1e3,
"sglang.set_sched_ms": t.set_sched * 1e3,
"sglang.get_next_residual_ms": residual * 1e3,
"sglang.chunk_count": int(t.chunk_count),
"sglang.candidates": int(t.candidates),
"sglang.skipped_prefetch": int(t.skipped_prefetch),
"sglang.admitted": int(t.admitted),
@@ -735,6 +778,13 @@ def end_pass(dp_rank: int) -> None:
other = total - measured
if other < 0.0:
other = 0.0
# Residual after also charging the get_next sub-phases.
measured2 = measured + (
t.pending_abort + t.chunk + t.new_batch + t.set_sched
)
residual = total - measured2
if residual < 0.0:
residual = 0.0
hist, counters = _ensure_stage_metrics()
child_key = (t.phase, dp_rank)
keyed = _STAGE_CHILDREN.get(child_key)
@@ -768,7 +818,7 @@ def end_pass(dp_rank: int) -> None:
elif t.new_chunk:
keyed["new_chunk"].inc(1)
note_pass_snapshot(t, total)
_emit_pass_span(dp_rank, t, total, other)
_emit_pass_span(dp_rank, t, total, other, residual)
except Exception:
# Telemetry must never break the scheduler loop
_STAGE_TLS.current = None