observability: fix scheduler no-batch accounting
This commit is contained in:
@@ -617,11 +617,15 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self.attn_tp_cpu_group,
|
||||
)
|
||||
failed = set()
|
||||
bootstrap_candidates = len(candidates)
|
||||
pending_bootstrap = 0
|
||||
for req, poll in zip(candidates, polls):
|
||||
if poll == KVPoll.Failed:
|
||||
self.handle_bootstrap_failure(req)
|
||||
failed.add(req)
|
||||
elif (
|
||||
bootstrap_candidates -= 1
|
||||
continue
|
||||
if (
|
||||
poll == KVPoll.WaitingForInput
|
||||
and req.pending_bootstrap
|
||||
and not should_force_retry(req)
|
||||
@@ -630,6 +634,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# finalize cannot fail here; if it ever does, the request stays
|
||||
# pending and the post-forward check resolves it.
|
||||
self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req)
|
||||
pending_bootstrap += int(req.pending_bootstrap)
|
||||
batch_telemetry.set_bootstrap_pending(
|
||||
pending_bootstrap, bootstrap_candidates
|
||||
)
|
||||
if failed:
|
||||
self.waiting_queue = [
|
||||
req for req in self.waiting_queue if req not in failed
|
||||
@@ -807,6 +815,8 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self.last_batch is not None
|
||||
or len(self.result_queue) > 0
|
||||
or len(self.waiting_queue) > 0
|
||||
or len(self._deferred_input_requests) > 0
|
||||
or self._engine_paused
|
||||
or (
|
||||
self.disagg_prefill_inflight_queue is not None
|
||||
and len(self.disagg_prefill_inflight_queue) > 0
|
||||
@@ -823,10 +833,6 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
inflight = self.disagg_prefill_inflight_queue
|
||||
return {
|
||||
"waiting": len(self.waiting_queue),
|
||||
"waiting_all_bootstrap": bool(self.waiting_queue)
|
||||
and all(
|
||||
getattr(r, "pending_bootstrap", False) for r in self.waiting_queue
|
||||
),
|
||||
"result_queue": len(self.result_queue),
|
||||
"result_limit": 2,
|
||||
"inflight": len(inflight) if inflight is not None else 0,
|
||||
@@ -889,6 +895,14 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if not waiting_for_batch_result or received_inputs or bootstrapped_reqs:
|
||||
skip_batch_creation = False
|
||||
|
||||
# Preserve the launch decision before processing a result mutates the
|
||||
# queues; end-of-loop depths cannot explain why this loop did not launch.
|
||||
if _lt is not None:
|
||||
if len(self.result_queue) >= 2:
|
||||
batch_telemetry.note_launch_blocked(_lt, "result_backpressure")
|
||||
elif self._deferred_input_requests:
|
||||
batch_telemetry.note_launch_blocked(_lt, "deferred_input")
|
||||
|
||||
# Keep at most two batches in flight.
|
||||
if (
|
||||
len(self.result_queue) < 2
|
||||
@@ -917,9 +931,13 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_rb_t0 = time.time_ns()
|
||||
_rb_t0 = (
|
||||
time.time_ns()
|
||||
if _lt is not None and _lt.span is not None
|
||||
else 0
|
||||
)
|
||||
batch_result = self.run_batch(batch)
|
||||
if _lt is not None:
|
||||
if _rb_t0:
|
||||
try:
|
||||
batch_telemetry.mark_run_batch(
|
||||
_rb_t0,
|
||||
@@ -951,13 +969,20 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
):
|
||||
waiting_for_batch_result = True
|
||||
self._yield_gil_if_needed()
|
||||
_end_loop(_lt, False, None, waiting_for_result=True)
|
||||
_end_loop(
|
||||
_lt,
|
||||
loop_batch is not None,
|
||||
loop_batch,
|
||||
waiting_for_result=True,
|
||||
)
|
||||
continue
|
||||
# Process the oldest batch result.
|
||||
oldest_batch, oldest_batch_result = self.result_queue.popleft()
|
||||
_pr_t0 = time.time_ns()
|
||||
_pr_t0 = (
|
||||
time.time_ns() if _lt is not None and _lt.span is not None else 0
|
||||
)
|
||||
self.process_batch_result(oldest_batch, oldest_batch_result)
|
||||
if _lt is not None:
|
||||
if _pr_t0:
|
||||
try:
|
||||
batch_telemetry.mark_process(
|
||||
_pr_t0,
|
||||
|
||||
@@ -529,6 +529,8 @@ class PassTimings:
|
||||
"candidates",
|
||||
"skipped_prefetch",
|
||||
"admitted",
|
||||
"bootstrap_candidates",
|
||||
"pending_bootstrap",
|
||||
"had_chunked",
|
||||
"new_chunk",
|
||||
"seq",
|
||||
@@ -546,6 +548,8 @@ class PassTimings:
|
||||
self.candidates = 0
|
||||
self.skipped_prefetch = 0
|
||||
self.admitted = 0
|
||||
self.bootstrap_candidates = 0
|
||||
self.pending_bootstrap = 0
|
||||
self.had_chunked = False
|
||||
self.new_chunk = False
|
||||
self.seq = 0
|
||||
@@ -608,6 +612,14 @@ def set_admitted(n: int) -> None:
|
||||
t.admitted = int(n)
|
||||
|
||||
|
||||
def set_bootstrap_pending(pending: int, candidates: int) -> None:
|
||||
"""Reuse the bootstrap resolver's existing pass; do not rescan the queue."""
|
||||
t = _current()
|
||||
if t is not None:
|
||||
t.pending_bootstrap = int(pending)
|
||||
t.bootstrap_candidates = int(candidates)
|
||||
|
||||
|
||||
def note_had_chunked(flag: bool) -> None:
|
||||
"""Record whether a chunked request was already in flight when the pass began.
|
||||
|
||||
@@ -659,12 +671,16 @@ def add_mlp_sync(seconds: float) -> None:
|
||||
def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) -> None:
|
||||
if not pass_span_decision(t.seq):
|
||||
return
|
||||
loop = getattr(_LOOP_TLS, "current", None)
|
||||
loop_ctx = loop.span_ctx if loop is not None else None
|
||||
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
|
||||
# An unsampled loop owns this pass; do not leak an independent pass trace.
|
||||
# If work is discovered later, end_loop emits the snapshot as its child.
|
||||
return
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.pass")
|
||||
loop = getattr(_LOOP_TLS, "current", None)
|
||||
loop_ctx = loop.span_ctx if loop is not None else None
|
||||
end_ns = time.time_ns()
|
||||
start_ns = end_ns - int(total * 1e9)
|
||||
tracer.start_span(
|
||||
@@ -845,12 +861,17 @@ def observe_recv(
|
||||
_RECV_SPAN_SAMPLE * 10000
|
||||
):
|
||||
return
|
||||
loop = current_loop()
|
||||
loop_ctx = current_loop_context()
|
||||
if _LOOP_SPAN_ENABLED and loop is not None and loop_ctx is None:
|
||||
# Keep the snapshot only; end_loop emits it if this loop has work.
|
||||
return
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.recv")
|
||||
tracer.start_span(
|
||||
"scheduler.recv_requests",
|
||||
context=current_loop_context(),
|
||||
context=loop_ctx,
|
||||
start_time=start_ns,
|
||||
attributes={
|
||||
"sglang.phase": "recv",
|
||||
@@ -864,6 +885,8 @@ def observe_recv(
|
||||
"sglang.recv_mm_ms": mm * 1e3,
|
||||
},
|
||||
).end(end_time=end_ns)
|
||||
if loop is not None and loop_ctx is not None:
|
||||
loop.recv_span_child = True
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
@@ -970,7 +993,10 @@ def forward_span_enabled() -> bool:
|
||||
|
||||
|
||||
def kv_send_span_enabled() -> bool:
|
||||
return _KV_SEND_SPAN_ENABLED
|
||||
if not _KV_SEND_SPAN_ENABLED:
|
||||
return False
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
return lt is not None and lt.span is not None
|
||||
|
||||
|
||||
def loop_sample_decision(seq: int) -> bool:
|
||||
@@ -1003,6 +1029,8 @@ class LoopTimings:
|
||||
"span",
|
||||
"span_ctx",
|
||||
"pass_span_child",
|
||||
"recv_span_child",
|
||||
"launch_blocked_reason",
|
||||
"seq",
|
||||
"no_batch_reason",
|
||||
)
|
||||
@@ -1034,10 +1062,37 @@ class LoopTimings:
|
||||
self.span = None
|
||||
self.span_ctx = None
|
||||
self.pass_span_child = False
|
||||
self.recv_span_child = False
|
||||
self.launch_blocked_reason = ""
|
||||
self.seq = 0
|
||||
self.no_batch_reason = ""
|
||||
|
||||
|
||||
def ensure_loop_span(lt: Optional[LoopTimings]) -> None:
|
||||
"""Materialize a sampled span once the loop is known to have work."""
|
||||
if (
|
||||
lt is None
|
||||
or lt.span is not None
|
||||
or not loop_sample_decision(lt.predicted_seq)
|
||||
):
|
||||
return
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.loop")
|
||||
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
|
||||
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
|
||||
except Exception:
|
||||
lt.span = None
|
||||
lt.span_ctx = None
|
||||
|
||||
|
||||
def note_launch_blocked(lt: Optional[LoopTimings], reason: str) -> None:
|
||||
"""Preserve why this loop could not launch before queues are consumed."""
|
||||
if lt is not None and not lt.launch_blocked_reason:
|
||||
lt.launch_blocked_reason = reason
|
||||
|
||||
|
||||
def begin_loop(
|
||||
phase: str = "prefill",
|
||||
*,
|
||||
@@ -1068,16 +1123,8 @@ def begin_loop(
|
||||
)
|
||||
_LAST_MLP_OUTCOME = ("none", "none")
|
||||
_LOOP_TLS.current = lt
|
||||
if _LOOP_SPAN_ENABLED and lt.has_work0 and loop_sample_decision(lt.predicted_seq):
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.loop")
|
||||
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
|
||||
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
|
||||
except Exception:
|
||||
lt.span = None
|
||||
lt.span_ctx = None
|
||||
if lt.has_work0:
|
||||
ensure_loop_span(lt)
|
||||
return lt
|
||||
|
||||
|
||||
@@ -1121,6 +1168,8 @@ def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
|
||||
"sglang.candidates": int(t.candidates),
|
||||
"sglang.skipped_prefetch": int(t.skipped_prefetch),
|
||||
"sglang.admitted": int(t.admitted),
|
||||
"sglang.bootstrap_candidates": int(t.bootstrap_candidates),
|
||||
"sglang.pending_bootstrap": int(t.pending_bootstrap),
|
||||
"sglang.resumed_chunked": int(bool(t.had_chunked)),
|
||||
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
|
||||
"sglang.mlp_sync_seq": int(t.seq),
|
||||
@@ -1154,8 +1203,14 @@ def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
|
||||
|
||||
def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None:
|
||||
"""Set by dp_attn.prepare_mlp_sync_batch_raw once per pass."""
|
||||
if not _LOOP_ENABLED:
|
||||
return
|
||||
global _LAST_MLP_OUTCOME
|
||||
_LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome))
|
||||
if local_kind != "none" or global_outcome != "none":
|
||||
# DP idle work is only known after the sync. Activate the span before
|
||||
# end_pass so the get_next child can share the loop trace.
|
||||
ensure_loop_span(getattr(_LOOP_TLS, "current", None))
|
||||
|
||||
|
||||
def note_forward_device(t: float, category: str = "", **kwargs) -> None:
|
||||
@@ -1227,20 +1282,27 @@ def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str:
|
||||
if state.get("deferred", 0) > 0 or state.get("paused"):
|
||||
return "deferred_input"
|
||||
waiting = int(state.get("waiting", 0) or 0)
|
||||
if waiting > 0:
|
||||
if state.get("waiting_all_bootstrap"):
|
||||
if waiting > 0 and pass_snapshot is not None:
|
||||
bootstrap_candidates = int(
|
||||
pass_snapshot.get("sglang.bootstrap_candidates", 0) or 0
|
||||
)
|
||||
pending_bootstrap = int(
|
||||
pass_snapshot.get("sglang.pending_bootstrap", 0) or 0
|
||||
)
|
||||
if bootstrap_candidates > 0 and pending_bootstrap == bootstrap_candidates:
|
||||
return "waiting_bootstrap"
|
||||
if pass_snapshot is not None:
|
||||
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
|
||||
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
|
||||
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
|
||||
if candidates > 0 and skipped == candidates:
|
||||
return "waiting_prefetch"
|
||||
if admitted == 0:
|
||||
return "admission_blocked"
|
||||
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
|
||||
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
|
||||
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
|
||||
if candidates > 0 and skipped == candidates:
|
||||
return "waiting_prefetch"
|
||||
if admitted == 0:
|
||||
return "admission_blocked"
|
||||
inflight = int(state.get("inflight", 0) or 0)
|
||||
bootstrap = int(state.get("bootstrap", 0) or 0)
|
||||
if waiting == 0 and (inflight or bootstrap):
|
||||
if waiting == 0 and bootstrap:
|
||||
return "waiting_bootstrap"
|
||||
if waiting == 0 and inflight:
|
||||
return "waiting_pd"
|
||||
if (
|
||||
waiting == 0
|
||||
@@ -1317,34 +1379,50 @@ def end_loop(
|
||||
)
|
||||
local_kind, global_outcome = _LAST_MLP_OUTCOME
|
||||
pass_snapshot = lt.get_next[2] if lt.get_next is not None else None
|
||||
pending = bool(
|
||||
state.get("waiting", 0)
|
||||
or state.get("result_queue", 0)
|
||||
or state.get("inflight", 0)
|
||||
or state.get("bootstrap", 0)
|
||||
or state.get("deferred", 0)
|
||||
or state.get("paused")
|
||||
or state.get("waiting_for_result")
|
||||
)
|
||||
reason = ""
|
||||
|
||||
if not has_batch:
|
||||
try:
|
||||
try:
|
||||
if local_kind == "none" and global_outcome == "idle":
|
||||
# The DP sync replaces local None with a real idle batch, so this
|
||||
# accounting must not depend on has_batch being False.
|
||||
_ensure_loop_metrics()["lockstep_seconds"].labels(
|
||||
dp_rank=str(dp_rank), forward_overlap=overlap
|
||||
).inc(loop_s)
|
||||
elif not has_batch:
|
||||
reason = lt.launch_blocked_reason or _classify_no_batch(
|
||||
state, pass_snapshot
|
||||
)
|
||||
global _LAST_NO_BATCH_REASON
|
||||
_LAST_NO_BATCH_REASON = reason
|
||||
metrics = _ensure_loop_metrics()
|
||||
if local_kind == "none" and global_outcome == "idle":
|
||||
metrics["lockstep_seconds"].labels(
|
||||
dp_rank=str(dp_rank), forward_overlap=overlap
|
||||
).inc(loop_s)
|
||||
else:
|
||||
reason = _classify_no_batch(state, pass_snapshot)
|
||||
global _LAST_NO_BATCH_REASON
|
||||
_LAST_NO_BATCH_REASON = reason
|
||||
metrics["no_batch_seconds"].labels(
|
||||
dp_rank=str(dp_rank),
|
||||
reason=reason,
|
||||
forward_overlap=overlap,
|
||||
).inc(loop_s)
|
||||
metrics["no_batch_total"].labels(
|
||||
dp_rank=str(dp_rank),
|
||||
reason=reason,
|
||||
forward_overlap=overlap,
|
||||
).inc()
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
metrics["no_batch_seconds"].labels(
|
||||
dp_rank=str(dp_rank),
|
||||
reason=reason,
|
||||
forward_overlap=overlap,
|
||||
).inc(loop_s)
|
||||
metrics["no_batch_total"].labels(
|
||||
dp_rank=str(dp_rank),
|
||||
reason=reason,
|
||||
forward_overlap=overlap,
|
||||
).inc()
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
lt.no_batch_reason = reason
|
||||
|
||||
if lt.span is None and (has_batch or pending):
|
||||
# Covers work discovered after begin_loop, including first input and
|
||||
# deferred-only loops. DP outcomes activate earlier for pass parenting.
|
||||
ensure_loop_span(lt)
|
||||
if lt.span is None:
|
||||
return
|
||||
|
||||
@@ -1352,13 +1430,6 @@ def end_loop(
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.loop")
|
||||
pending = bool(
|
||||
state.get("waiting", 0)
|
||||
or state.get("result_queue", 0)
|
||||
or state.get("inflight", 0)
|
||||
or state.get("bootstrap", 0)
|
||||
or state.get("waiting_for_result")
|
||||
)
|
||||
attrs = {
|
||||
"sglang.phase": lt.phase,
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
@@ -1387,7 +1458,7 @@ def end_loop(
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if lt.recv is not None:
|
||||
if lt.recv is not None and not lt.recv_span_child:
|
||||
start, end, parts = lt.recv
|
||||
child = {"sglang.dp_rank": int(dp_rank)}
|
||||
child.update(parts)
|
||||
|
||||
Reference in New Issue
Block a user