observability: split get_next residual into its four phases
Adds the Phase A instrumentation from the async-backup handoff (§7.1):
* PassTimings: pending_abort / chunk / new_batch / set_sched (+ chunk_count)
* prefill.get_next_disagg_prefill_batch_to_run: time
process_pending_chunked_abort, process_prefill_chunk,
get_new_batch_prefill and set_schedule_time_batch
* scheduler.prefill.pass gains sglang.{pending_abort,chunk,new_batch,set_sched}_ms,
sglang.get_next_residual_ms (= other - these four) and sglang.chunk_count;
the existing sglang.other_ms is unchanged for dashboard compatibility.
~5 monotonic calls per pass; enabled by SGLANG_BATCH_STAGE_TIMING (already on).
This commit is contained in:
@@ -697,8 +697,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# the last chunk of a resumed request clears self.chunked_req.
|
||||
batch_telemetry.note_had_chunked(self.chunked_req is not None)
|
||||
|
||||
if bt_pass is not None:
|
||||
_bt_t = time.monotonic()
|
||||
self.process_pending_chunked_abort()
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.add_pending_abort(time.monotonic() - _bt_t)
|
||||
_bt_t = time.monotonic()
|
||||
self._process_hicache_events(
|
||||
should_retry_storage_prefetch=should_retry_storage_prefetch
|
||||
@@ -717,16 +720,25 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
batch_telemetry.add_bootstrap(time.monotonic() - _bt_t)
|
||||
|
||||
# Skip repeated chunk processing while polling a pending batch.
|
||||
_chunk_t0 = time.monotonic() if bt_pass is not None else None
|
||||
_had_chunk = self.chunked_req is not None
|
||||
if last_batch is None or not last_batch.prefill_chunk_processed:
|
||||
self.process_prefill_chunk(
|
||||
last_batch=last_batch, running_batch=running_batch
|
||||
)
|
||||
if last_batch is not None:
|
||||
last_batch.prefill_chunk_processed = True
|
||||
if _chunk_t0 is not None:
|
||||
batch_telemetry.add_chunk(
|
||||
time.monotonic() - _chunk_t0, 1 if _had_chunk else 0
|
||||
)
|
||||
|
||||
_nb_t0 = time.monotonic() if bt_pass is not None else None
|
||||
prefill_plan = self.get_new_batch_prefill(running_batch)
|
||||
batch = prefill_plan.batch_to_run
|
||||
running_batch = prefill_plan.running_batch
|
||||
if _nb_t0 is not None:
|
||||
batch_telemetry.add_new_batch(time.monotonic() - _nb_t0)
|
||||
if bt_pass is not None:
|
||||
# Drop any gather value left by an earlier pass; only a gather that
|
||||
# actually runs inside the call below may be attributed to this one.
|
||||
@@ -737,7 +749,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t)
|
||||
|
||||
if batch:
|
||||
_ss_t0 = time.monotonic() if bt_pass is not None else None
|
||||
set_schedule_time_batch(batch)
|
||||
if _ss_t0 is not None:
|
||||
batch_telemetry.add_set_sched(time.monotonic() - _ss_t0)
|
||||
|
||||
if bt_pass is not None:
|
||||
batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank)
|
||||
|
||||
@@ -535,6 +535,11 @@ class PassTimings:
|
||||
"hicache",
|
||||
"bootstrap",
|
||||
"mlp_sync",
|
||||
"pending_abort",
|
||||
"chunk",
|
||||
"new_batch",
|
||||
"set_sched",
|
||||
"chunk_count",
|
||||
"dp_gather",
|
||||
"candidates",
|
||||
"skipped_prefetch",
|
||||
@@ -554,6 +559,11 @@ class PassTimings:
|
||||
self.hicache = 0.0
|
||||
self.bootstrap = 0.0
|
||||
self.mlp_sync = 0.0
|
||||
self.pending_abort = 0.0
|
||||
self.chunk = 0.0
|
||||
self.new_batch = 0.0
|
||||
self.set_sched = 0.0
|
||||
self.chunk_count = 0
|
||||
self.dp_gather = None
|
||||
self.candidates = 0
|
||||
self.skipped_prefetch = 0
|
||||
@@ -604,6 +614,31 @@ def add_bootstrap(seconds: float) -> None:
|
||||
t.bootstrap += seconds
|
||||
|
||||
|
||||
def add_pending_abort(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.pending_abort += seconds
|
||||
|
||||
|
||||
def add_chunk(seconds: float, count: int = 0) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.chunk += seconds
|
||||
t.chunk_count += int(count)
|
||||
|
||||
|
||||
def add_new_batch(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.new_batch += seconds
|
||||
|
||||
|
||||
def add_set_sched(seconds: float) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.set_sched += seconds
|
||||
|
||||
|
||||
def mark_candidate(n: int = 1) -> None:
|
||||
t = _current()
|
||||
if t is not None:
|
||||
@@ -678,7 +713,9 @@ def add_mlp_sync(seconds: float) -> None:
|
||||
t.seq = _DP_GATHER_SEQ
|
||||
|
||||
|
||||
def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None:
|
||||
def _emit_pass_span(
|
||||
dp_rank: int, t: PassTimings, total: float, other: float, residual: float
|
||||
) -> None:
|
||||
if not pass_span_decision(t.seq):
|
||||
return
|
||||
loop = getattr(_LOOP_TLS, "current", None)
|
||||
@@ -709,6 +746,12 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) ->
|
||||
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
|
||||
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
|
||||
"sglang.other_ms": other * 1e3,
|
||||
"sglang.pending_abort_ms": t.pending_abort * 1e3,
|
||||
"sglang.chunk_ms": t.chunk * 1e3,
|
||||
"sglang.new_batch_ms": t.new_batch * 1e3,
|
||||
"sglang.set_sched_ms": t.set_sched * 1e3,
|
||||
"sglang.get_next_residual_ms": residual * 1e3,
|
||||
"sglang.chunk_count": int(t.chunk_count),
|
||||
"sglang.candidates": int(t.candidates),
|
||||
"sglang.skipped_prefetch": int(t.skipped_prefetch),
|
||||
"sglang.admitted": int(t.admitted),
|
||||
@@ -735,6 +778,13 @@ def end_pass(dp_rank: int) -> None:
|
||||
other = total - measured
|
||||
if other < 0.0:
|
||||
other = 0.0
|
||||
# Residual after also charging the get_next sub-phases.
|
||||
measured2 = measured + (
|
||||
t.pending_abort + t.chunk + t.new_batch + t.set_sched
|
||||
)
|
||||
residual = total - measured2
|
||||
if residual < 0.0:
|
||||
residual = 0.0
|
||||
hist, counters = _ensure_stage_metrics()
|
||||
child_key = (t.phase, dp_rank)
|
||||
keyed = _STAGE_CHILDREN.get(child_key)
|
||||
@@ -768,7 +818,7 @@ def end_pass(dp_rank: int) -> None:
|
||||
elif t.new_chunk:
|
||||
keyed["new_chunk"].inc(1)
|
||||
note_pass_snapshot(t, total)
|
||||
_emit_pass_span(dp_rank, t, total, other)
|
||||
_emit_pass_span(dp_rank, t, total, other, residual)
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop
|
||||
_STAGE_TLS.current = None
|
||||
|
||||
Reference in New Issue
Block a user