From a5fbe834e80cf4c66450550c76790b59f32b7355 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sun, 8 Nov 2026 00:20:55 +0800 Subject: [PATCH] observability: split get_next residual into its four phases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the Phase A instrumentation from the async-backup handoff (ยง7.1): * PassTimings: pending_abort / chunk / new_batch / set_sched (+ chunk_count) * prefill.get_next_disagg_prefill_batch_to_run: time process_pending_chunked_abort, process_prefill_chunk, get_new_batch_prefill and set_schedule_time_batch * scheduler.prefill.pass gains sglang.{pending_abort,chunk,new_batch,set_sched}_ms, sglang.get_next_residual_ms (= other - these four) and sglang.chunk_count; the existing sglang.other_ms is unchanged for dashboard compatibility. ~5 monotonic calls per pass; enabled by SGLANG_BATCH_STAGE_TIMING (already on). --- python/sglang/srt/disaggregation/prefill.py | 15 ++++++ .../srt/observability/batch_telemetry.py | 54 ++++++++++++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index bdfd812db6..c2846102e5 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -697,8 +697,11 @@ class SchedulerDisaggregationPrefillMixin: # the last chunk of a resumed request clears self.chunked_req. batch_telemetry.note_had_chunked(self.chunked_req is not None) + if bt_pass is not None: + _bt_t = time.monotonic() self.process_pending_chunked_abort() if bt_pass is not None: + batch_telemetry.add_pending_abort(time.monotonic() - _bt_t) _bt_t = time.monotonic() self._process_hicache_events( should_retry_storage_prefetch=should_retry_storage_prefetch @@ -717,16 +720,25 @@ class SchedulerDisaggregationPrefillMixin: batch_telemetry.add_bootstrap(time.monotonic() - _bt_t) # Skip repeated chunk processing while polling a pending batch. + _chunk_t0 = time.monotonic() if bt_pass is not None else None + _had_chunk = self.chunked_req is not None if last_batch is None or not last_batch.prefill_chunk_processed: self.process_prefill_chunk( last_batch=last_batch, running_batch=running_batch ) if last_batch is not None: last_batch.prefill_chunk_processed = True + if _chunk_t0 is not None: + batch_telemetry.add_chunk( + time.monotonic() - _chunk_t0, 1 if _had_chunk else 0 + ) + _nb_t0 = time.monotonic() if bt_pass is not None else None prefill_plan = self.get_new_batch_prefill(running_batch) batch = prefill_plan.batch_to_run running_batch = prefill_plan.running_batch + if _nb_t0 is not None: + batch_telemetry.add_new_batch(time.monotonic() - _nb_t0) if bt_pass is not None: # Drop any gather value left by an earlier pass; only a gather that # actually runs inside the call below may be attributed to this one. @@ -737,7 +749,10 @@ class SchedulerDisaggregationPrefillMixin: batch_telemetry.add_mlp_sync(time.monotonic() - _bt_t) if batch: + _ss_t0 = time.monotonic() if bt_pass is not None else None set_schedule_time_batch(batch) + if _ss_t0 is not None: + batch_telemetry.add_set_sched(time.monotonic() - _ss_t0) if bt_pass is not None: batch_telemetry.end_pass(dp_rank=get_parallel().attn_dp_rank) diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index e3a4466924..742d17d815 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -535,6 +535,11 @@ class PassTimings: "hicache", "bootstrap", "mlp_sync", + "pending_abort", + "chunk", + "new_batch", + "set_sched", + "chunk_count", "dp_gather", "candidates", "skipped_prefetch", @@ -554,6 +559,11 @@ class PassTimings: self.hicache = 0.0 self.bootstrap = 0.0 self.mlp_sync = 0.0 + self.pending_abort = 0.0 + self.chunk = 0.0 + self.new_batch = 0.0 + self.set_sched = 0.0 + self.chunk_count = 0 self.dp_gather = None self.candidates = 0 self.skipped_prefetch = 0 @@ -604,6 +614,31 @@ def add_bootstrap(seconds: float) -> None: t.bootstrap += seconds +def add_pending_abort(seconds: float) -> None: + t = _current() + if t is not None: + t.pending_abort += seconds + + +def add_chunk(seconds: float, count: int = 0) -> None: + t = _current() + if t is not None: + t.chunk += seconds + t.chunk_count += int(count) + + +def add_new_batch(seconds: float) -> None: + t = _current() + if t is not None: + t.new_batch += seconds + + +def add_set_sched(seconds: float) -> None: + t = _current() + if t is not None: + t.set_sched += seconds + + def mark_candidate(n: int = 1) -> None: t = _current() if t is not None: @@ -678,7 +713,9 @@ def add_mlp_sync(seconds: float) -> None: t.seq = _DP_GATHER_SEQ -def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None: +def _emit_pass_span( + dp_rank: int, t: PassTimings, total: float, other: float, residual: float +) -> None: if not pass_span_decision(t.seq): return loop = getattr(_LOOP_TLS, "current", None) @@ -709,6 +746,12 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> "sglang.mlp_sync_ms": t.mlp_sync * 1e3, "sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3, "sglang.other_ms": other * 1e3, + "sglang.pending_abort_ms": t.pending_abort * 1e3, + "sglang.chunk_ms": t.chunk * 1e3, + "sglang.new_batch_ms": t.new_batch * 1e3, + "sglang.set_sched_ms": t.set_sched * 1e3, + "sglang.get_next_residual_ms": residual * 1e3, + "sglang.chunk_count": int(t.chunk_count), "sglang.candidates": int(t.candidates), "sglang.skipped_prefetch": int(t.skipped_prefetch), "sglang.admitted": int(t.admitted), @@ -735,6 +778,13 @@ def end_pass(dp_rank: int) -> None: other = total - measured if other < 0.0: other = 0.0 + # Residual after also charging the get_next sub-phases. + measured2 = measured + ( + t.pending_abort + t.chunk + t.new_batch + t.set_sched + ) + residual = total - measured2 + if residual < 0.0: + residual = 0.0 hist, counters = _ensure_stage_metrics() child_key = (t.phase, dp_rank) keyed = _STAGE_CHILDREN.get(child_key) @@ -768,7 +818,7 @@ def end_pass(dp_rank: int) -> None: elif t.new_chunk: keyed["new_chunk"].inc(1) note_pass_snapshot(t, total) - _emit_pass_span(dp_rank, t, total, other) + _emit_pass_span(dp_rank, t, total, other, residual) except Exception: # Telemetry must never break the scheduler loop _STAGE_TLS.current = None