observability: add scheduler loop span and no-batch accounting
Adds an opt-in scheduler.loop span (SGLANG_TRACE_LOOP_SPAN) with recv /
get_next / run_batch / process_result / kv_send children. The span is only
created for sampled loops that have pending work; fully idle loops stay in
the counters so the 714 pass/s idle spin never floods Jaeger.
No-batch accounting is always on with SGLANG_BATCH_STAGE_TIMING:
- sglang:scheduler_no_batch_{seconds,total}{dp_rank,reason,forward_overlap}
- sglang:scheduler_dp_lockstep_idle_seconds_total{dp_rank,forward_overlap}
with the design's reason priority (result_backpressure, deferred_input,
waiting_bootstrap, waiting_prefetch, admission_blocked, waiting_pd,
fully_idle, no_ready_other) and local_batch_kind/global_outcome from the DP
sync. DP lockstep idle is counted separately from pure no-batch wait.
loop gpu_ms reuses the existing DeviceTimer intervals (no new CUDA events);
the forward.gpu lane is optional and approximate. Pass/recv spans become
children of the loop span when it is active.
Tests: test_loop_span_accounting (5) and the existing stage-timing suite (8)
pass.
This commit is contained in:
@@ -790,10 +790,90 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
batch = None
|
||||
batch_result = None
|
||||
|
||||
def _forward_active() -> bool:
|
||||
reporter = getattr(self, "metrics_reporter", None)
|
||||
timer = (
|
||||
getattr(reporter, "forward_pass_device_timer", None)
|
||||
if reporter is not None
|
||||
else None
|
||||
)
|
||||
return timer.is_active() if timer is not None else False
|
||||
|
||||
def _pre_has_work() -> bool:
|
||||
bootstrap_queue = getattr(
|
||||
self.disagg_prefill_bootstrap_queue, "queue", None
|
||||
)
|
||||
return bool(
|
||||
self.last_batch is not None
|
||||
or len(self.result_queue) > 0
|
||||
or len(self.waiting_queue) > 0
|
||||
or (
|
||||
self.disagg_prefill_inflight_queue is not None
|
||||
and len(self.disagg_prefill_inflight_queue) > 0
|
||||
)
|
||||
or (bootstrap_queue is not None and len(bootstrap_queue) > 0)
|
||||
)
|
||||
|
||||
def _loop_state(
|
||||
waiting_for_result: bool = False, paused: bool = False
|
||||
) -> dict:
|
||||
bootstrap_queue = getattr(
|
||||
self.disagg_prefill_bootstrap_queue, "queue", None
|
||||
)
|
||||
inflight = self.disagg_prefill_inflight_queue
|
||||
return {
|
||||
"waiting": len(self.waiting_queue),
|
||||
"waiting_all_bootstrap": bool(self.waiting_queue)
|
||||
and all(
|
||||
getattr(r, "pending_bootstrap", False) for r in self.waiting_queue
|
||||
),
|
||||
"result_queue": len(self.result_queue),
|
||||
"result_limit": 2,
|
||||
"inflight": len(inflight) if inflight is not None else 0,
|
||||
"bootstrap": len(bootstrap_queue) if bootstrap_queue is not None else 0,
|
||||
"deferred": len(self._deferred_input_requests),
|
||||
"paused": bool(paused or self._engine_paused),
|
||||
"waiting_for_result": bool(waiting_for_result),
|
||||
"skip_batch_creation": bool(skip_batch_creation),
|
||||
"forward_active": _forward_active(),
|
||||
}
|
||||
|
||||
def _end_loop(
|
||||
lt,
|
||||
has_batch: bool,
|
||||
loop_batch,
|
||||
waiting_for_result: bool = False,
|
||||
paused: bool = False,
|
||||
) -> None:
|
||||
if lt is None:
|
||||
return
|
||||
try:
|
||||
batch_telemetry.end_loop(
|
||||
lt,
|
||||
dp_rank=get_parallel().attn_dp_rank,
|
||||
has_batch=has_batch,
|
||||
batch=loop_batch,
|
||||
state=_loop_state(
|
||||
waiting_for_result=waiting_for_result, paused=paused
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
|
||||
while True:
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
_lt = None
|
||||
if batch_telemetry.loop_accounting_enabled():
|
||||
_lt = batch_telemetry.begin_loop(
|
||||
"prefill",
|
||||
pre_has_work=_pre_has_work(),
|
||||
forward_active=_forward_active,
|
||||
)
|
||||
loop_batch = None
|
||||
|
||||
# Stop dispatching at pause requests while waiting for a batch result.
|
||||
received_inputs = self.ingest_requests(
|
||||
stop_at_pause=waiting_for_batch_result
|
||||
@@ -801,6 +881,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if self._engine_paused:
|
||||
self._record_scheduler_state_for_paused_engine()
|
||||
self._yield_gil_if_needed()
|
||||
_end_loop(_lt, False, None, paused=True)
|
||||
continue
|
||||
bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
self.waiting_queue.extend(bootstrapped_reqs)
|
||||
@@ -827,6 +908,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
batch = self.ngram_embedding_manager.prepare_for_forward(
|
||||
batch, chunked_req=self.chunked_req
|
||||
)
|
||||
loop_batch = batch
|
||||
self.cur_batch_for_debug = batch
|
||||
# Skip batch creation next round to avoid repeated preparation failures.
|
||||
skip_batch_creation = batch is None
|
||||
@@ -835,7 +917,23 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_rb_t0 = time.time_ns()
|
||||
batch_result = self.run_batch(batch)
|
||||
if _lt is not None:
|
||||
try:
|
||||
batch_telemetry.mark_run_batch(
|
||||
_rb_t0,
|
||||
time.time_ns(),
|
||||
{
|
||||
"sglang.forward_mode": str(batch.forward_mode),
|
||||
"sglang.bs": batch.batch_size(),
|
||||
"sglang.num_tokens": int(
|
||||
getattr(batch, "extend_num_tokens", 0) or 0
|
||||
),
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
@@ -853,10 +951,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
):
|
||||
waiting_for_batch_result = True
|
||||
self._yield_gil_if_needed()
|
||||
_end_loop(_lt, False, None, waiting_for_result=True)
|
||||
continue
|
||||
# Process the oldest batch result.
|
||||
oldest_batch, oldest_batch_result = self.result_queue.popleft()
|
||||
_pr_t0 = time.time_ns()
|
||||
self.process_batch_result(oldest_batch, oldest_batch_result)
|
||||
if _lt is not None:
|
||||
try:
|
||||
batch_telemetry.mark_process(
|
||||
_pr_t0,
|
||||
time.time_ns(),
|
||||
{"sglang.n_reqs": len(oldest_batch.reqs)},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
elif batch is None:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
@@ -871,6 +980,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self._sched_idled = batch is None
|
||||
self.last_batch = batch
|
||||
waiting_for_batch_result = False
|
||||
_end_loop(_lt, loop_batch is not None, loop_batch)
|
||||
|
||||
def is_disagg_prefill_batch_result_ready(
|
||||
self: Scheduler,
|
||||
@@ -1511,7 +1621,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
state_indices = state_indices_for_request(self, req, end_idx)
|
||||
value = computer.compute(page_indices_gpu, state_indices)
|
||||
self.disagg_metadata_buffers.set_kv_checksum(req, value)
|
||||
_kv_t0 = time.time_ns() if batch_telemetry.kv_send_span_enabled() else 0
|
||||
self._send_kv_chunk(req, last_chunk=last_chunk, end_idx=end_idx)
|
||||
if _kv_t0:
|
||||
batch_telemetry.add_kv_send(_kv_t0, time.time_ns())
|
||||
|
||||
def _send_kv_chunk(
|
||||
self: Scheduler,
|
||||
|
||||
@@ -1417,6 +1417,8 @@ class Scheduler(
|
||||
metrics_collector_context=self.metrics_collector_context,
|
||||
metrics_collector=self.metrics_collector,
|
||||
)
|
||||
# Read the already-running DeviceTimer; adds no CUDA events.
|
||||
batch_telemetry.install_forward_reporter(self)
|
||||
|
||||
def init_schedule_policy(self):
|
||||
# Init schedule policy and new token estimation
|
||||
|
||||
@@ -675,6 +675,15 @@ def prepare_mlp_sync_batch_raw(
|
||||
else:
|
||||
need_idle_batch = max(mlp_sync_info.global_num_tokens) > 0
|
||||
|
||||
# Capture the pre-sync local kind before the idle replacement below: the
|
||||
# loop span's local_batch_kind describes what this rank brought in, not the
|
||||
# idle batch it may be asked to run for DP lockstep.
|
||||
_loop_local_kind = (
|
||||
"none"
|
||||
if local_batch is None
|
||||
else ("idle" if local_batch.forward_mode.is_idle() else "extend")
|
||||
)
|
||||
|
||||
batch_to_gather = local_batch
|
||||
if need_idle_batch:
|
||||
if local_batch is None:
|
||||
@@ -683,6 +692,13 @@ def prepare_mlp_sync_batch_raw(
|
||||
# NOTE: for prebuilt batch, we add an inner idle batch to run MLP sync
|
||||
batch_to_gather = local_batch.inner_idle_batch = get_idle_batch()
|
||||
|
||||
_loop_global_outcome = (
|
||||
"idle"
|
||||
if (_loop_local_kind == "none" and need_idle_batch)
|
||||
else ("none" if _loop_local_kind == "none" else _loop_local_kind)
|
||||
)
|
||||
batch_telemetry.note_mlp_sync_outcome(_loop_local_kind, _loop_global_outcome)
|
||||
|
||||
if batch_to_gather is not None:
|
||||
_update_gather_batch(
|
||||
batch_to_gather,
|
||||
|
||||
@@ -663,10 +663,13 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) ->
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.pass")
|
||||
loop = getattr(_LOOP_TLS, "current", None)
|
||||
loop_ctx = loop.span_ctx if loop is not None else None
|
||||
end_ns = time.time_ns()
|
||||
start_ns = end_ns - int(total * 1e9)
|
||||
tracer.start_span(
|
||||
"scheduler.prefill.pass",
|
||||
context=loop_ctx,
|
||||
start_time=start_ns,
|
||||
attributes={
|
||||
"sglang.phase": t.phase,
|
||||
@@ -687,6 +690,8 @@ def _emit_pass_span(dp_rank: int, t: PassTimings, total: float, other: float) ->
|
||||
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
|
||||
},
|
||||
).end(end_time=end_ns)
|
||||
if loop is not None and loop_ctx is not None:
|
||||
loop.pass_span_child = True
|
||||
except Exception:
|
||||
# Tracing must never break the scheduler loop.
|
||||
pass
|
||||
@@ -736,6 +741,7 @@ def end_pass(dp_rank: int) -> None:
|
||||
keyed["resumed_chunk"].inc(1)
|
||||
elif t.new_chunk:
|
||||
keyed["new_chunk"].inc(1)
|
||||
note_pass_snapshot(t, total)
|
||||
_emit_pass_span(dp_rank, t, total, other)
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop
|
||||
@@ -817,6 +823,22 @@ def observe_recv(
|
||||
counters["msgs"].labels(dp_rank=str(dp_rank)).inc(n_msgs)
|
||||
if n_tokens:
|
||||
counters["prompt_tokens"].labels(dp_rank=str(dp_rank)).inc(n_tokens)
|
||||
end_ns = time.time_ns()
|
||||
start_ns = end_ns - int(total * 1e9)
|
||||
note_recv_phase(
|
||||
start_wall_ns=start_ns,
|
||||
end_wall_ns=end_ns,
|
||||
parts={
|
||||
"sglang.recv_total_ms": total * 1e3,
|
||||
"sglang.recv_pull_ms": pull * 1e3,
|
||||
"sglang.recv_broadcast_ms": broadcast * 1e3,
|
||||
"sglang.recv_unwrap_ms": unwrap * 1e3,
|
||||
"sglang.recv_mm_ms": mm * 1e3,
|
||||
"sglang.recv_msgs": int(n_msgs),
|
||||
"sglang.recv_prompt_tokens": int(n_tokens),
|
||||
},
|
||||
)
|
||||
|
||||
if not _RECV_SPAN_ENABLED:
|
||||
return
|
||||
if _RECV_SPAN_SAMPLE < 1.0 and (_RECV_SEQ * 2654435761) % 10000 >= int(
|
||||
@@ -826,10 +848,10 @@ def observe_recv(
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.recv")
|
||||
end_ns = time.time_ns()
|
||||
tracer.start_span(
|
||||
"scheduler.recv_requests",
|
||||
start_time=end_ns - int(total * 1e9),
|
||||
context=current_loop_context(),
|
||||
start_time=start_ns,
|
||||
attributes={
|
||||
"sglang.phase": "recv",
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
@@ -878,3 +900,530 @@ def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler loop span + no-batch accounting (opt-in, same stage-timing gate).
|
||||
#
|
||||
# One ``scheduler.loop`` span per scheduler iteration (sampled), with phase
|
||||
# children (recv / get_next / run_batch / process_result / kv_send). The
|
||||
# no-batch counters are incremented on *every* loop (sampling only controls
|
||||
# spans), split by a low-cardinality reason and by whether a forward was in
|
||||
# flight (forward_overlap). Fully idle loops never emit a span.
|
||||
#
|
||||
# Enable with ``SGLANG_TRACE_LOOP_SPAN=1`` (requires SGLANG_BATCH_STAGE_TIMING).
|
||||
# Knobs:
|
||||
# SGLANG_TRACE_LOOP_SAMPLE=0.05 head-sample ratio (deterministic per
|
||||
# mlp_sync_seq, identical on all ranks)
|
||||
# SGLANG_TRACE_FORWARD_SPAN=0 emit approximate forward.gpu lanes
|
||||
# SGLANG_TRACE_KV_SEND_SPAN=1 emit the kv_send child (default on
|
||||
# when the loop span is on)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from collections import deque as _deque
|
||||
|
||||
_LOOP_ENABLED = _STAGE_TIMING
|
||||
_LOOP_SPAN_ENABLED = _LOOP_ENABLED and os.getenv(
|
||||
"SGLANG_TRACE_LOOP_SPAN", "0"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
_LOOP_SPAN_SAMPLE = float(os.getenv("SGLANG_TRACE_LOOP_SAMPLE", "0.05"))
|
||||
_FORWARD_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
|
||||
"SGLANG_TRACE_FORWARD_SPAN", "0"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
_KV_SEND_SPAN_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
|
||||
"SGLANG_TRACE_KV_SEND_SPAN", "1"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
|
||||
_NO_BATCH_REASONS = (
|
||||
"fully_idle",
|
||||
"waiting_prefetch",
|
||||
"waiting_bootstrap",
|
||||
"result_backpressure",
|
||||
"deferred_input",
|
||||
"admission_blocked",
|
||||
"waiting_pd",
|
||||
"no_ready_other",
|
||||
)
|
||||
|
||||
_LOOP_METRICS = None
|
||||
_LOOP_TLS = threading.local()
|
||||
|
||||
# Monotonic total of completed forward device time (from the DeviceTimer),
|
||||
# used to derive one loop's gpu_ms without adding a second timer.
|
||||
_FORWARD_DEVICE_NS_TOTAL = 0
|
||||
_FORWARD_EVENTS = _deque(maxlen=128) # (completion_wall_ns, elapsed_ns, category)
|
||||
|
||||
# Set by dp_attn.prepare_mlp_sync_batch_raw() once per pass.
|
||||
_LAST_MLP_OUTCOME = ("none", "none")
|
||||
_LAST_NO_BATCH_REASON = "no_ready_other"
|
||||
|
||||
|
||||
def loop_accounting_enabled() -> bool:
|
||||
return _LOOP_ENABLED
|
||||
|
||||
|
||||
def loop_span_enabled() -> bool:
|
||||
return _LOOP_SPAN_ENABLED
|
||||
|
||||
|
||||
def forward_span_enabled() -> bool:
|
||||
return _FORWARD_SPAN_ENABLED
|
||||
|
||||
|
||||
def kv_send_span_enabled() -> bool:
|
||||
return _KV_SEND_SPAN_ENABLED
|
||||
|
||||
|
||||
def loop_sample_decision(seq: int) -> bool:
|
||||
"""Deterministic, rank-identical loop-span sampling decision."""
|
||||
if not _LOOP_SPAN_ENABLED or seq <= 0:
|
||||
return False
|
||||
if _LOOP_SPAN_SAMPLE >= 1.0:
|
||||
return True
|
||||
if _LOOP_SPAN_SAMPLE <= 0.0:
|
||||
return False
|
||||
return (seq * 2654435761) % 10000 < int(_LOOP_SPAN_SAMPLE * 10000)
|
||||
|
||||
|
||||
class LoopTimings:
|
||||
__slots__ = (
|
||||
"phase",
|
||||
"t0_ns",
|
||||
"t0_wall_ns",
|
||||
"predicted_seq",
|
||||
"active0",
|
||||
"has_work0",
|
||||
"forward_total0",
|
||||
"recv",
|
||||
"get_next",
|
||||
"run_batch",
|
||||
"process",
|
||||
"kv_send_start",
|
||||
"kv_send_end",
|
||||
"kv_chunks",
|
||||
"span",
|
||||
"span_ctx",
|
||||
"pass_span_child",
|
||||
"seq",
|
||||
"no_batch_reason",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
phase: str,
|
||||
t0_ns: int,
|
||||
t0_wall_ns: int,
|
||||
predicted_seq: int,
|
||||
active0: bool,
|
||||
has_work0: bool,
|
||||
forward_total0: int,
|
||||
):
|
||||
self.phase = phase
|
||||
self.t0_ns = t0_ns
|
||||
self.t0_wall_ns = t0_wall_ns
|
||||
self.predicted_seq = predicted_seq
|
||||
self.active0 = bool(active0)
|
||||
self.has_work0 = bool(has_work0)
|
||||
self.forward_total0 = forward_total0
|
||||
self.recv = None
|
||||
self.get_next = None
|
||||
self.run_batch = None
|
||||
self.process = None
|
||||
self.kv_send_start = None
|
||||
self.kv_send_end = None
|
||||
self.kv_chunks = 0
|
||||
self.span = None
|
||||
self.span_ctx = None
|
||||
self.pass_span_child = False
|
||||
self.seq = 0
|
||||
self.no_batch_reason = ""
|
||||
|
||||
|
||||
def begin_loop(
|
||||
phase: str = "prefill",
|
||||
*,
|
||||
pre_has_work: bool = True,
|
||||
forward_active=None,
|
||||
):
|
||||
"""Start one scheduler-loop accounting record.
|
||||
|
||||
``pre_has_work`` must be cheap to compute (queue lengths / last_batch):
|
||||
fully idle loops are still counted, but never create a span.
|
||||
``forward_active`` is a callable (usually DeviceTimer.is_active) or a bool.
|
||||
"""
|
||||
global _LAST_MLP_OUTCOME
|
||||
if not _LOOP_ENABLED:
|
||||
return None
|
||||
try:
|
||||
active0 = bool(forward_active() if callable(forward_active) else forward_active)
|
||||
except Exception:
|
||||
active0 = False
|
||||
lt = LoopTimings(
|
||||
phase=phase,
|
||||
t0_ns=time.monotonic_ns(),
|
||||
t0_wall_ns=time.time_ns(),
|
||||
predicted_seq=_DP_GATHER_SEQ + 1,
|
||||
active0=active0,
|
||||
has_work0=bool(pre_has_work),
|
||||
forward_total0=_FORWARD_DEVICE_NS_TOTAL,
|
||||
)
|
||||
_LAST_MLP_OUTCOME = ("none", "none")
|
||||
_LOOP_TLS.current = lt
|
||||
if _LOOP_SPAN_ENABLED and lt.has_work0 and loop_sample_decision(lt.predicted_seq):
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.loop")
|
||||
lt.span = tracer.start_span("scheduler.loop", start_time=lt.t0_wall_ns)
|
||||
lt.span_ctx = otel_trace.set_span_in_context(lt.span)
|
||||
except Exception:
|
||||
lt.span = None
|
||||
lt.span_ctx = None
|
||||
return lt
|
||||
|
||||
|
||||
def current_loop() -> Optional[LoopTimings]:
|
||||
return getattr(_LOOP_TLS, "current", None)
|
||||
|
||||
|
||||
def current_loop_context():
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
return lt.span_ctx if lt is not None else None
|
||||
|
||||
|
||||
def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None:
|
||||
"""Record the recv_requests phase inside the current loop (if any)."""
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is not None:
|
||||
lt.recv = (int(start_wall_ns), int(end_wall_ns), dict(parts))
|
||||
|
||||
|
||||
def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
|
||||
"""Record get_next timings for the current loop (called from end_pass)."""
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is None:
|
||||
return
|
||||
end_wall_ns = time.time_ns()
|
||||
start_wall_ns = end_wall_ns - int(total_s * 1e9)
|
||||
measured = t.match + t.adder + t.hicache + t.bootstrap + t.mlp_sync
|
||||
other = max(0.0, total_s - measured)
|
||||
lt.get_next = (
|
||||
start_wall_ns,
|
||||
end_wall_ns,
|
||||
{
|
||||
"sglang.get_next_ms": total_s * 1e3,
|
||||
"sglang.match_ms": t.match * 1e3,
|
||||
"sglang.adder_ms": t.adder * 1e3,
|
||||
"sglang.hicache_ms": t.hicache * 1e3,
|
||||
"sglang.bootstrap_ms": t.bootstrap * 1e3,
|
||||
"sglang.mlp_sync_ms": t.mlp_sync * 1e3,
|
||||
"sglang.dp_gather_ms": (t.dp_gather or 0.0) * 1e3,
|
||||
"sglang.other_ms": other * 1e3,
|
||||
"sglang.candidates": int(t.candidates),
|
||||
"sglang.skipped_prefetch": int(t.skipped_prefetch),
|
||||
"sglang.admitted": int(t.admitted),
|
||||
"sglang.resumed_chunked": int(bool(t.had_chunked)),
|
||||
"sglang.new_chunked": int(bool(t.new_chunk) and not t.had_chunked),
|
||||
"sglang.mlp_sync_seq": int(t.seq),
|
||||
},
|
||||
)
|
||||
lt.seq = int(t.seq)
|
||||
|
||||
|
||||
def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is not None:
|
||||
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
|
||||
|
||||
|
||||
def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is not None:
|
||||
lt.process = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
|
||||
|
||||
|
||||
def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is None:
|
||||
return
|
||||
lt.kv_chunks += 1
|
||||
if lt.kv_send_start is None or start_wall_ns < lt.kv_send_start:
|
||||
lt.kv_send_start = int(start_wall_ns)
|
||||
if lt.kv_send_end is None or end_wall_ns > lt.kv_send_end:
|
||||
lt.kv_send_end = int(end_wall_ns)
|
||||
|
||||
|
||||
def note_mlp_sync_outcome(local_kind: str, global_outcome: str) -> None:
|
||||
"""Set by dp_attn.prepare_mlp_sync_batch_raw once per pass."""
|
||||
global _LAST_MLP_OUTCOME
|
||||
_LAST_MLP_OUTCOME = (str(local_kind), str(global_outcome))
|
||||
|
||||
|
||||
def note_forward_device(t: float, category: str = "", **kwargs) -> None:
|
||||
"""DeviceTimer reporter: accumulate completed forward device time."""
|
||||
global _FORWARD_DEVICE_NS_TOTAL
|
||||
try:
|
||||
ns = int(float(t) * 1e9)
|
||||
except Exception:
|
||||
return
|
||||
_FORWARD_DEVICE_NS_TOTAL += ns
|
||||
if _FORWARD_SPAN_ENABLED:
|
||||
_FORWARD_EVENTS.append((time.time_ns(), ns, str(category)))
|
||||
|
||||
|
||||
def install_forward_reporter(scheduler) -> None:
|
||||
"""Attach the forward device-time reporter to the existing DeviceTimer.
|
||||
|
||||
No CUDA event is added here: SGLANG_ENABLE_METRICS_DEVICE_TIMER already
|
||||
runs the timer; this only reads its completed intervals.
|
||||
"""
|
||||
if not _LOOP_ENABLED:
|
||||
return
|
||||
try:
|
||||
reporter = getattr(scheduler, "metrics_reporter", None)
|
||||
timer = getattr(reporter, "forward_pass_device_timer", None)
|
||||
if timer is not None:
|
||||
timer.add_reporter(note_forward_device)
|
||||
except Exception:
|
||||
# Telemetry must never break scheduler init.
|
||||
pass
|
||||
|
||||
|
||||
def _ensure_loop_metrics():
|
||||
global _LOOP_METRICS
|
||||
if _LOOP_METRICS is None:
|
||||
from prometheus_client import Counter
|
||||
|
||||
_LOOP_METRICS = {
|
||||
"no_batch_seconds": Counter(
|
||||
"sglang:scheduler_no_batch_seconds_total",
|
||||
"Wall time of scheduler loops that ended without a runnable batch, "
|
||||
"by reason and forward_overlap",
|
||||
["dp_rank", "reason", "forward_overlap"],
|
||||
),
|
||||
"no_batch_total": Counter(
|
||||
"sglang:scheduler_no_batch_total",
|
||||
"Count of scheduler loops that ended without a runnable batch, "
|
||||
"by reason and forward_overlap",
|
||||
["dp_rank", "reason", "forward_overlap"],
|
||||
),
|
||||
"lockstep_seconds": Counter(
|
||||
"sglang:scheduler_dp_lockstep_idle_seconds_total",
|
||||
"Wall time of DP-lockstep idle-forward loops (this rank had no "
|
||||
"local batch but the DP group required an idle forward)",
|
||||
["dp_rank", "forward_overlap"],
|
||||
),
|
||||
}
|
||||
return _LOOP_METRICS
|
||||
|
||||
|
||||
def _classify_no_batch(state: dict, pass_snapshot: Optional[dict]) -> str:
|
||||
"""Low-cardinality no-batch reason, in the design's priority order.
|
||||
|
||||
``pass_snapshot`` is the get_next attribute dict (candidates / skipped /
|
||||
admitted). It is None when the loop skipped batch preparation entirely.
|
||||
"""
|
||||
if state.get("result_queue", 0) >= state.get("result_limit", 2):
|
||||
return "result_backpressure"
|
||||
if state.get("deferred", 0) > 0 or state.get("paused"):
|
||||
return "deferred_input"
|
||||
waiting = int(state.get("waiting", 0) or 0)
|
||||
if waiting > 0:
|
||||
if state.get("waiting_all_bootstrap"):
|
||||
return "waiting_bootstrap"
|
||||
if pass_snapshot is not None:
|
||||
candidates = int(pass_snapshot.get("sglang.candidates", 0) or 0)
|
||||
skipped = int(pass_snapshot.get("sglang.skipped_prefetch", 0) or 0)
|
||||
admitted = int(pass_snapshot.get("sglang.admitted", 0) or 0)
|
||||
if candidates > 0 and skipped == candidates:
|
||||
return "waiting_prefetch"
|
||||
if admitted == 0:
|
||||
return "admission_blocked"
|
||||
inflight = int(state.get("inflight", 0) or 0)
|
||||
bootstrap = int(state.get("bootstrap", 0) or 0)
|
||||
if waiting == 0 and (inflight or bootstrap):
|
||||
return "waiting_pd"
|
||||
if (
|
||||
waiting == 0
|
||||
and inflight == 0
|
||||
and bootstrap == 0
|
||||
and int(state.get("result_queue", 0) or 0) == 0
|
||||
and int(state.get("deferred", 0) or 0) == 0
|
||||
and not state.get("waiting_for_result")
|
||||
):
|
||||
return "fully_idle"
|
||||
if state.get("skip_batch_creation") and pass_snapshot is None:
|
||||
# A failed preparation defers retries; keep the last classified reason.
|
||||
return _LAST_NO_BATCH_REASON
|
||||
return "no_ready_other"
|
||||
|
||||
|
||||
def _batch_attrs(batch) -> dict:
|
||||
if batch is None:
|
||||
return {
|
||||
"sglang.n_reqs": 0,
|
||||
"sglang.n_new": 0,
|
||||
"sglang.n_cached": 0,
|
||||
"sglang.chunked": False,
|
||||
}
|
||||
try:
|
||||
n_reqs = int(batch.batch_size())
|
||||
except Exception:
|
||||
n_reqs = len(getattr(batch, "reqs", ()) or ())
|
||||
try:
|
||||
n_new = int(getattr(batch, "extend_num_tokens", 0) or 0)
|
||||
except Exception:
|
||||
n_new = 0
|
||||
try:
|
||||
n_cached = int(sum(getattr(batch, "prefix_lens", ()) or ()))
|
||||
except Exception:
|
||||
n_cached = 0
|
||||
try:
|
||||
chunked = getattr(batch, "chunked_req", None) is not None
|
||||
except Exception:
|
||||
chunked = False
|
||||
return {
|
||||
"sglang.n_reqs": n_reqs,
|
||||
"sglang.n_new": n_new,
|
||||
"sglang.n_cached": n_cached,
|
||||
"sglang.chunked": bool(chunked),
|
||||
}
|
||||
|
||||
|
||||
def end_loop(
|
||||
lt: Optional[LoopTimings],
|
||||
*,
|
||||
dp_rank: int,
|
||||
has_batch: bool,
|
||||
batch=None,
|
||||
state: Optional[dict] = None,
|
||||
) -> None:
|
||||
"""Close one loop: counters (always), loop span + phases (if sampled)."""
|
||||
if lt is None:
|
||||
return
|
||||
_LOOP_TLS.current = None
|
||||
state = state or {}
|
||||
try:
|
||||
now_ns = time.monotonic_ns()
|
||||
now_wall_ns = time.time_ns()
|
||||
loop_s = max(0.0, (now_ns - lt.t0_ns) / 1e9)
|
||||
gpu_ns = max(0, _FORWARD_DEVICE_NS_TOTAL - lt.forward_total0)
|
||||
active1 = bool(state.get("forward_active", False))
|
||||
overlap = (
|
||||
"full"
|
||||
if lt.active0 and active1
|
||||
else "partial"
|
||||
if (lt.active0 or active1)
|
||||
else "none"
|
||||
)
|
||||
local_kind, global_outcome = _LAST_MLP_OUTCOME
|
||||
pass_snapshot = lt.get_next[2] if lt.get_next is not None else None
|
||||
reason = ""
|
||||
|
||||
if not has_batch:
|
||||
try:
|
||||
metrics = _ensure_loop_metrics()
|
||||
if local_kind == "none" and global_outcome == "idle":
|
||||
metrics["lockstep_seconds"].labels(
|
||||
dp_rank=str(dp_rank), forward_overlap=overlap
|
||||
).inc(loop_s)
|
||||
else:
|
||||
reason = _classify_no_batch(state, pass_snapshot)
|
||||
global _LAST_NO_BATCH_REASON
|
||||
_LAST_NO_BATCH_REASON = reason
|
||||
metrics["no_batch_seconds"].labels(
|
||||
dp_rank=str(dp_rank),
|
||||
reason=reason,
|
||||
forward_overlap=overlap,
|
||||
).inc(loop_s)
|
||||
metrics["no_batch_total"].labels(
|
||||
dp_rank=str(dp_rank),
|
||||
reason=reason,
|
||||
forward_overlap=overlap,
|
||||
).inc()
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
lt.no_batch_reason = reason
|
||||
|
||||
if lt.span is None:
|
||||
return
|
||||
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.prefill.loop")
|
||||
pending = bool(
|
||||
state.get("waiting", 0)
|
||||
or state.get("result_queue", 0)
|
||||
or state.get("inflight", 0)
|
||||
or state.get("bootstrap", 0)
|
||||
or state.get("waiting_for_result")
|
||||
)
|
||||
attrs = {
|
||||
"sglang.phase": lt.phase,
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.loop_seq": int(lt.seq or lt.predicted_seq),
|
||||
"sglang.loop_ms": loop_s * 1e3,
|
||||
"sglang.gpu_ms": gpu_ns / 1e6,
|
||||
"sglang.gpu_idle_ms": max(0.0, loop_s * 1e3 - gpu_ns / 1e6),
|
||||
"sglang.has_batch": bool(has_batch),
|
||||
"sglang.engine_has_pending_work": pending,
|
||||
"sglang.local_batch_kind": local_kind,
|
||||
"sglang.global_outcome": global_outcome,
|
||||
"sglang.no_batch_reason": reason,
|
||||
"sglang.forward_overlap": overlap,
|
||||
}
|
||||
attrs.update(_batch_attrs(batch))
|
||||
lt.span.set_attributes(attrs)
|
||||
|
||||
def _child(name, start_ns, end_ns, child_attrs):
|
||||
try:
|
||||
tracer.start_span(
|
||||
name,
|
||||
context=lt.span_ctx,
|
||||
start_time=int(start_ns),
|
||||
attributes=child_attrs,
|
||||
).end(end_time=int(end_ns))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if lt.recv is not None:
|
||||
start, end, parts = lt.recv
|
||||
child = {"sglang.dp_rank": int(dp_rank)}
|
||||
child.update(parts)
|
||||
_child("scheduler.loop.recv", start, end, child)
|
||||
if lt.get_next is not None and not lt.pass_span_child:
|
||||
start, end, ng_attrs = lt.get_next
|
||||
_child("scheduler.loop.get_next", start, end, ng_attrs)
|
||||
if lt.run_batch is not None:
|
||||
start, end, rb_attrs = lt.run_batch
|
||||
_child("scheduler.loop.run_batch", start, end, rb_attrs)
|
||||
if lt.process is not None:
|
||||
start, end, pr_attrs = lt.process
|
||||
_child("scheduler.loop.process_result", start, end, pr_attrs)
|
||||
if _KV_SEND_SPAN_ENABLED and lt.kv_send_start is not None:
|
||||
_child(
|
||||
"scheduler.loop.kv_send",
|
||||
lt.kv_send_start,
|
||||
lt.kv_send_end,
|
||||
{"sglang.n_chunks": int(lt.kv_chunks)},
|
||||
)
|
||||
if _FORWARD_SPAN_ENABLED:
|
||||
for completion_wall, elapsed_ns, category in list(_FORWARD_EVENTS):
|
||||
if completion_wall < lt.t0_wall_ns or completion_wall > now_wall_ns:
|
||||
continue
|
||||
_child(
|
||||
"scheduler.loop.forward.gpu",
|
||||
completion_wall - elapsed_ns,
|
||||
completion_wall,
|
||||
{
|
||||
"sglang.gpu_ms": elapsed_ns / 1e6,
|
||||
"sglang.category": category,
|
||||
},
|
||||
)
|
||||
lt.span.end(end_time=now_wall_ns)
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
finally:
|
||||
_LOOP_TLS.current = None
|
||||
|
||||
Reference in New Issue
Block a user