observability: split prefill run_batch/process_result into host phases (M1)

Implements M1 of handoff/prefill-cpu-kv-overlap-design-20261106.md:
per-loop detail phases behind SGLANG_TRACE_LOOP_DETAIL=1 (requires
SGLANG_BATCH_STAGE_TIMING=1 + SGLANG_TRACE_LOOP_SPAN=1, only sampled loops).

- run_batch: early_send_host, resolve_inputs, model_forward_host,
  copy_result_submit (+ run_residual_ms)
- process_result: copy_done_wait, auxiliary_and_logprob, per_request_result,
  kv_state_payload/page_indices/sender_send by send_kind, report_stats
  (+ process_residual_ms)
- war_barrier is measured as its own loop phase (outside run_batch's span)
- repeated/per-request phases are summed attributes; single phases are child
  spans, so span count never grows with n_reqs/n_segments
- DP-skew join fields: pass_end_wall_ns, arrival_est_wall_ns = pass_end -
  mlp_sync, previous-loop phase summary (sglang.prev_*, published only for
  consecutive loop_seq; prev_detail_missing marks a gap)

No CUDA syncs added; when disabled every hook is a module-level bool check.
Unit tests: test_loop_detail_phases.py (10 cases).
This commit is contained in:
ymk
2026-11-06 20:01:56 +08:00
parent 23053730fc
commit 72e556ec81
4 changed files with 1252 additions and 21 deletions
+120 -11
View File
@@ -952,7 +952,10 @@ class SchedulerDisaggregationPrefillMixin:
)
except Exception:
pass
self._apply_war_barrier()
# WAR barrier is outside run_batch's timing boundary, so it is
# recorded as its own loop phase (design §4.3).
with batch_telemetry.detail_phase("war_barrier"):
self._apply_war_barrier()
self.result_queue.append((batch.copy(), batch_result))
else:
batch_result = None
@@ -981,13 +984,32 @@ class SchedulerDisaggregationPrefillMixin:
_pr_t0 = (
time.time_ns() if _lt is not None and _lt.span is not None else 0
)
# Scope KV phases to this call: sends issued later in the loop
# (inflight-queue pending bootstrap) must not be charged here.
_proc_kv_phases = (
"kv_state_payload",
"kv_page_indices",
"kv_sender_send",
)
_proc_detail = batch_telemetry.loop_detail_enabled()
_proc_kv_before = (
batch_telemetry.detail_phase_total_ns(_proc_kv_phases)
if _proc_detail
else 0
)
self.process_batch_result(oldest_batch, oldest_batch_result)
_pr_attrs = {"sglang.n_reqs": len(oldest_batch.reqs)}
if _proc_detail:
_pr_attrs["sglang.process_kv_ms"] = (
batch_telemetry.detail_phase_total_ns(_proc_kv_phases)
- _proc_kv_before
) / 1e6
if _pr_t0:
try:
batch_telemetry.mark_process(
_pr_t0,
time.time_ns(),
{"sglang.n_reqs": len(oldest_batch.reqs)},
_pr_attrs,
)
except Exception:
pass
@@ -1057,7 +1079,11 @@ class SchedulerDisaggregationPrefillMixin:
)
if copy_done is not None:
copy_done.synchronize()
with batch_telemetry.detail_phase("copy_done_wait"):
copy_done.synchronize()
_aux_t0 = (
time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None
)
auxiliary_output_starts = (
self.batch_result_processor.snapshot_auxiliary_output_starts(batch, result)
)
@@ -1103,10 +1129,29 @@ class SchedulerDisaggregationPrefillMixin:
if extend_logprob_start_len < extend_input_len:
logprob_pt += extend_input_len - extend_logprob_start_len
# Auxiliary finalize / logits-to-CPU work is a contiguous host phase.
if _aux_t0 is not None:
batch_telemetry.detail_add(
"auxiliary_and_logprob", _aux_t0, time.monotonic_ns()
)
# Per-request result handling is everything in this loop except the KV
# phases; the delta below keeps `continue` paths accounted for without
# measuring each branch.
_pr_loop_t0 = (
time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None
)
_kv_phases = ("kv_state_payload", "kv_page_indices", "kv_sender_send")
_kv_before = (
batch_telemetry.detail_phase_total_ns(_kv_phases)
if _pr_loop_t0 is not None
else 0
)
for i, (req, next_token_id) in enumerate(
zip(batch.reqs, next_token_ids, strict=True)
):
if req.inflight_middle_chunks <= 0:
batch_telemetry.detail_count("n_last_reqs")
req.time_stats.set_prefill_finished_time()
# Its KV never landed: abort instead of sending it to decode.
@@ -1210,6 +1255,7 @@ class SchedulerDisaggregationPrefillMixin:
else:
# being chunked reqs' prefill is not finished
batch_telemetry.detail_count("n_middle_reqs")
req.inflight_middle_chunks -= 1
# Still chunking iff its next chunk was launched: either it is
@@ -1260,12 +1306,27 @@ class SchedulerDisaggregationPrefillMixin:
self.send_kv_chunk(req, last_chunk=False, end_idx=req.tmp_end_idx)
req.time_stats.set_last_chunked_prefill_finish_time()
if _pr_loop_t0 is not None:
_kv_ns = batch_telemetry.detail_phase_total_ns(_kv_phases) - _kv_before
batch_telemetry.detail_add(
"per_request_result",
_pr_loop_t0 + max(0, _kv_ns),
time.monotonic_ns(),
)
if auxiliary_output is not None:
_aux2_t0 = (
time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None
)
self.batch_result_processor.consume_auxiliary_output(
batch,
auxiliary_output,
auxiliary_output_starts,
)
if _aux2_t0 is not None:
batch_telemetry.detail_add(
"auxiliary_and_logprob", _aux2_t0, time.monotonic_ns()
)
if aborted_reqs:
self.output_streamer.stream_output(
@@ -1274,12 +1335,13 @@ class SchedulerDisaggregationPrefillMixin:
)
can_run_cuda_graph = result.can_run_cuda_graph
self.metrics_reporter.report_prefill_stats(
batch=batch,
prefill_stats=batch.prefill_stats,
can_run_cuda_graph=can_run_cuda_graph,
dp_cooperation_info=batch.dp_cooperation_info,
)
with batch_telemetry.detail_phase("report_stats"):
self.metrics_reporter.report_prefill_stats(
batch=batch,
prefill_stats=batch.prefill_stats,
can_run_cuda_graph=can_run_cuda_graph,
dp_cooperation_info=batch.dp_cooperation_info,
)
self.maybe_send_health_check_signal()
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
@@ -1627,14 +1689,21 @@ class SchedulerDisaggregationPrefillMixin:
ev = torch.cuda.Event()
ev.record(self.forward_stream)
req.disagg_kv_sender._early_send_wait_event = ev
self.send_kv_chunk(req, last_chunk=False, end_idx=cached_end)
if batch_telemetry.loop_detail_enabled():
batch_telemetry.detail_count("early_send_reqs")
self.send_kv_chunk(
req, last_chunk=False, end_idx=cached_end, send_kind="cached_prefix"
)
def send_kv_chunk(
self,
req: Req,
last_chunk: bool = False,
end_idx: Optional[int] = None,
send_kind: Optional[str] = None,
) -> None:
if send_kind is None:
send_kind = "last" if last_chunk else "middle"
computer: Optional[KvChecksumComputer] = self.kv_checksum_computer
if last_chunk and computer is not None:
if is_health_check_req(req):
@@ -1647,7 +1716,11 @@ class SchedulerDisaggregationPrefillMixin:
value = computer.compute(page_indices_gpu, state_indices)
self.disagg_metadata_buffers.set_kv_checksum(req, value)
_kv_t0 = time.time_ns() if batch_telemetry.kv_send_span_enabled() else 0
self._send_kv_chunk(req, last_chunk=last_chunk, end_idx=end_idx)
if batch_telemetry.loop_detail_enabled():
batch_telemetry.detail_count("n_kv_calls")
self._send_kv_chunk(
req, last_chunk=last_chunk, end_idx=end_idx, send_kind=send_kind
)
if _kv_t0:
batch_telemetry.add_kv_send(_kv_t0, time.time_ns())
@@ -1656,6 +1729,7 @@ class SchedulerDisaggregationPrefillMixin:
req: Req,
last_chunk: bool = False,
end_idx: Optional[int] = None,
send_kind: str = "last",
) -> None:
"""
Send a prefilled chunk to the decode server
@@ -1690,6 +1764,11 @@ class SchedulerDisaggregationPrefillMixin:
return
state_indices: Optional[List] = None
_state_tok = (
batch_telemetry.detail_begin("kv_state_payload." + send_kind)
if last_chunk
else None
)
if last_chunk:
self.disagg_metadata_buffers.set_buf(req)
@@ -1828,6 +1907,12 @@ class SchedulerDisaggregationPrefillMixin:
state_indices = [
payloads[st]() if st in payloads else None for st in state_types
]
if _state_tok is not None:
batch_telemetry.detail_attr(
"state_types",
",".join(sorted(str(getattr(st, "name", st)) for st in state_types)),
)
batch_telemetry.detail_end("kv_state_payload." + send_kind, _state_tok)
transfer_chunk_tokens = req.disagg_kv_sender.get_max_transfer_tokens()
if self.enable_staging:
@@ -1848,7 +1933,16 @@ class SchedulerDisaggregationPrefillMixin:
else:
segments = [(start_idx, end_idx)]
# Classify the chunk for detail accounting (cached_prefix/middle/last).
# Set on the sender so backend send() signatures stay unchanged.
req.disagg_kv_sender._send_kind = send_kind
for seg_start, seg_end in segments:
_seg_t0 = (
time.monotonic_ns()
if batch_telemetry.loop_detail_enabled()
else None
)
is_final_segment = seg_end == end_idx
raw_kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, seg_start:seg_end
@@ -1880,10 +1974,20 @@ class SchedulerDisaggregationPrefillMixin:
indices_event.record(torch.cuda.current_stream())
else:
page_indices = kv_to_page_indices(kv_indices, page_size)
if _seg_t0 is not None:
batch_telemetry.detail_add(
"kv_page_indices." + send_kind, _seg_t0, time.monotonic_ns()
)
batch_telemetry.detail_count("n_pages", len(page_indices))
segment_is_last = last_chunk and is_final_segment
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
continue
send_state_indices = state_indices if segment_is_last else None
_send_t0 = (
time.monotonic_ns()
if batch_telemetry.loop_detail_enabled()
else None
)
if indices_event is not None:
sender.send(
page_indices,
@@ -1897,6 +2001,11 @@ class SchedulerDisaggregationPrefillMixin:
send_state_indices,
num_kv_tokens=seg_end - seg_start,
)
if _send_t0 is not None:
batch_telemetry.detail_add(
"kv_sender_send." + send_kind, _send_t0, time.monotonic_ns()
)
batch_telemetry.detail_count("n_segments")
req.start_send_idx = end_idx
# A last chunk needs no entry: every `last_chunk=True` call site has
# already put the request on `disagg_prefill_inflight_queue`.
+13 -7
View File
@@ -4645,8 +4645,9 @@ class Scheduler(
# PD prefill: early-send cached prefix KV, overlapping the suffix forward.
if self.disaggregation_mode == DisaggregationMode.PREFILL:
for req in batch.reqs:
self.maybe_send_cached_prefix_chunk(req)
with batch_telemetry.detail_phase("early_send_host"):
for req in batch.reqs:
self.maybe_send_cached_prefix_chunk(req)
# Run forward
if self.is_generation:
@@ -4673,7 +4674,8 @@ class Scheduler(
# mix_running_indices). Run OUTSIDE isolation so the
# snapshot captures the post-consume state — restoring
# post-forward must not un-consume staging.
resolve_forward_inputs(batch, self.future_map)
with batch_telemetry.detail_phase("resolve_inputs"):
resolve_forward_inputs(batch, self.future_map)
with self._forward_isolation(batch, overlap=True):
future_indices = batch.req_pool_indices
@@ -4696,9 +4698,10 @@ class Scheduler(
)
# FIXME: pp is not compatible with overlap
batch_result = self.model_worker.forward_batch_generation(
batch, **fwd_kwargs
)
with batch_telemetry.detail_phase("model_forward_host"):
batch_result = self.model_worker.forward_batch_generation(
batch, **fwd_kwargs
)
if batch.spec_algorithm.is_none():
self.future_map.publish(future_indices, batch.seq_lens + 1)
# Park any refs the worker wants kept alive 2 iters
@@ -4736,7 +4739,10 @@ class Scheduler(
return_hidden_states=batch.return_hidden_states,
)
else:
self._copy_result_to_cpu(batch_result, batch)
with batch_telemetry.detail_phase(
"copy_result_submit"
):
self._copy_result_to_cpu(batch_result, batch)
else:
batch_result.future_indices = future_indices
@@ -20,6 +20,13 @@ Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0)
SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span
SGLANG_TRACE_LOOP_DETAIL=1 split run_batch/process_result into host
phases (requires SGLANG_BATCH_STAGE_TIMING=1
and SGLANG_TRACE_LOOP_SPAN=1; only sampled
loops are recorded)
SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown
(queue/source/indices/transfer/join)
Cost model (measured on our 4xDP4 / 4-GPU instances):
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
* span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s.
@@ -1033,6 +1040,9 @@ class LoopTimings:
"launch_blocked_reason",
"seq",
"no_batch_reason",
"pass_end_wall_ns",
"arrival_est_wall_ns",
"detail",
)
def __init__(
@@ -1066,6 +1076,11 @@ class LoopTimings:
self.launch_blocked_reason = ""
self.seq = 0
self.no_batch_reason = ""
self.pass_end_wall_ns = 0
self.arrival_est_wall_ns = 0
# Only populated when SGLANG_TRACE_LOOP_DETAIL=1 and this loop is
# sampled; see the "Loop detail phases" section below.
self.detail = None
def ensure_loop_span(lt: Optional[LoopTimings]) -> None:
@@ -1137,6 +1152,561 @@ def current_loop_context():
return lt.span_ctx if lt is not None else None
# ---------------------------------------------------------------------------
# Loop detail phases (SGLANG_TRACE_LOOP_DETAIL=1).
#
# Splits run_batch / process_result into host phases. Gated twice: the module
# switch must be on AND the current loop must be sampled (same deterministic
# decision as the loop span), so detail data never covers an unsampled loop.
#
# Accounting rules (see handoff/prefill-cpu-kv-overlap-design-20261106.md §3.2):
# * single phases -> one child span (``scheduler.loop.<name>``)
# * repeated phases -> summed attributes only (``sglang.<name>_ms`` + count)
# * durations use time.monotonic_ns(); OTLP start/end use time.time_ns()
# * residuals are clamped >= 0 and reported, never folded into a phase
# ---------------------------------------------------------------------------
_DETAIL_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
"SGLANG_TRACE_LOOP_DETAIL", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_DETAIL_TLS = threading.local()
# Chunk-level (Mooncake transfer worker) detail tracing. Independent switch:
# the worker runs on other threads and has no scheduler loop TLS.
_MOONCAKE_DETAIL_ENABLED = os.getenv(
"SGLANG_TRACE_MOONCAKE_DETAIL", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
# Previous loop summary per scheduler thread, used for DP-skew correlation
# (previous loop's phases vs this loop's arrival skew).
_LAST_LOOP_SUMMARY: dict = {}
# Phases that are inherently per-request / per-segment: always attributes,
# even when a loop happens to have exactly one occurrence, so span count never
# grows with n_reqs / n_segments (design §4.2).
_DETAIL_ATTRIBUTE_ONLY_PREFIXES = (
"per_request_result",
"kv_state_payload",
"kv_page_indices",
"kv_sender_send",
)
_DETAIL_PREV_KEYS = (
"run_batch_ms",
"process_result_ms",
"copy_done_wait_ms",
"kv_state_payload_ms",
"kv_page_indices_ms",
"kv_sender_send_ms",
)
def loop_detail_enabled() -> bool:
"""True when detail is on and the *current* loop is sampled."""
if not _DETAIL_ENABLED:
return False
lt = getattr(_LOOP_TLS, "current", None)
return lt is not None and loop_sample_decision(lt.predicted_seq)
def mooncake_detail_enabled() -> bool:
return _MOONCAKE_DETAIL_ENABLED
def _detail_state():
"""Detail state of the current loop, only when that loop is sampled."""
if not _DETAIL_ENABLED:
return None
lt = getattr(_LOOP_TLS, "current", None)
if lt is None or not loop_sample_decision(lt.predicted_seq):
return None
if lt.detail is None:
lt.detail = {"phases": {}, "counts": {}}
return lt.detail
def detail_begin(name: str) -> Optional[int]:
"""Start a single-occurrence phase. Returns a monotonic-ns token."""
if not _DETAIL_ENABLED:
return None
state = _detail_state()
if state is None:
return None
now_ns = time.monotonic_ns()
opens = getattr(_DETAIL_TLS, "open", None)
if opens is None:
opens = {}
_DETAIL_TLS.open = opens
opens[name] = (now_ns, time.time_ns())
return now_ns
def detail_end(name: str, token: Optional[int] = None) -> None:
"""Close a phase opened by ``detail_begin`` (or started at ``token``)."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
now_ns = time.monotonic_ns()
opens = getattr(_DETAIL_TLS, "open", None) or {}
entry = opens.pop(name, None)
start_wall_ns = entry[1] if entry is not None else None
if token is None:
if entry is None:
return
token = entry[0]
if start_wall_ns is None:
start_wall_ns = time.time_ns() - max(0, now_ns - int(token))
_detail_record(state, name, int(token), now_ns, start_wall_ns, time.time_ns())
def detail_add(name: str, start_ns: int, end_ns: int) -> None:
"""Accumulate one occurrence of a repeated phase (no span per occurrence)."""
if not _DETAIL_ENABLED or start_ns is None or end_ns is None:
return
state = _detail_state()
if state is None:
return
start_ns = int(start_ns)
end_ns = int(end_ns)
if end_ns < start_ns:
end_ns = start_ns
now_wall_ns = time.time_ns()
_detail_record(
state, name, start_ns, end_ns, now_wall_ns - (end_ns - start_ns), now_wall_ns
)
def _detail_record(state, name, start_ns, end_ns, start_wall_ns, end_wall_ns) -> None:
rec = state["phases"].get(name)
if rec is None:
state["phases"][name] = [
end_ns - start_ns,
1,
int(start_wall_ns),
int(end_wall_ns),
]
return
rec[0] += end_ns - start_ns
rec[1] += 1
rec[3] = int(end_wall_ns)
def detail_count(name: str, n: int = 1) -> None:
"""Accumulate a low-cardinality counter for this loop."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
state["counts"][name] = int(state["counts"].get(name, 0)) + int(n)
def detail_attr(name: str, value) -> None:
"""Set a low-cardinality loop attribute (last write wins)."""
if not _DETAIL_ENABLED:
return
state = _detail_state()
if state is None:
return
state.setdefault("attrs", {})[name] = value
def detail_phase_total_ns(names) -> int:
"""Sum of accumulated ns for the given phases in the current loop.
``names`` are *base* names: a phase recorded as ``kv_sender_send.last``
(per send kind) is included when ``kv_sender_send`` is requested. Phase
names are only generated by this module, so prefix matching cannot pick up
an unrelated phase that merely shares a leading string.
"""
state = _detail_state()
if state is None:
return 0
phases = state.get("phases") or {}
total = 0
for name, rec in phases.items():
if rec is None:
continue
for want in names:
if name == want or name.startswith(want + "."):
total += int(rec[0])
break
return total
class detail_phase:
"""Context manager for one single-occurrence phase (cheap when disabled)."""
__slots__ = ("name", "token")
def __init__(self, name: str):
self.name = name
self.token = None
def __enter__(self):
self.token = detail_begin(self.name)
return self
def __exit__(self, exc_type, exc, tb):
detail_end(self.name, self.token)
return False
def current_loop_seq() -> int:
"""Loop sequence for cross-record correlation (0 when unavailable)."""
lt = getattr(_LOOP_TLS, "current", None)
if lt is None:
return 0
return int(lt.seq or lt.predicted_seq or 0)
def current_loop_context_for_spans():
"""OTel context of the current loop span (None if unsampled)."""
lt = getattr(_LOOP_TLS, "current", None)
return lt.span_ctx if lt is not None else None
def _chunk_parent_context(chunk):
trace_ctx = getattr(chunk, "trace_ctx", None)
return getattr(trace_ctx, "root_span_context", None)
def split_queue_wait(
queue_wait_accum_ns: int, last_enqueue_mono_ns: int, dequeue_mono_ns: int
) -> tuple:
"""Return ``(attempt_queue_wait_ns, total_queue_wait_ns)``.
The attempt value matches the last_enqueue->dequeue wall window used by the
``mooncake.queue_wait`` child span; the total value additionally includes
the queue waits of earlier deferred attempts (design §5.3 metric).
Keeping them separate prevents a span whose wall length disagrees with its
``wait_ms`` attribute (review finding).
"""
attempt = max(0, int(dequeue_mono_ns) - int(last_enqueue_mono_ns))
return attempt, max(0, int(queue_wait_accum_ns)) + attempt
class RoomDetailState:
"""Thread-safe per-room detail bookkeeping (scheduler vs worker threads).
The scheduler aborts/retracts a room while the transfer worker may still be
finishing an attempt. A reset therefore records a cancellation mark, and
the worker's join update re-checks under the same lock before writing, so a
late worker can never resurrect stats after abort (review finding).
"""
# Bounded: a marker only needs to survive until the in-flight worker
# attempt finishes (milliseconds). The cap prevents unbounded growth if an
# aborted room never gets a worker teardown.
_MAX_CANCELLED = 8192
def __init__(self):
self._lock = threading.Lock()
self._join: dict = {}
self._seq: dict = {}
self._first: dict = {}
self._cancelled: dict = {}
def _mark_cancelled(self, room: int) -> None:
self._cancelled[int(room)] = time.monotonic_ns()
while len(self._cancelled) > self._MAX_CANCELLED:
self._cancelled.pop(next(iter(self._cancelled)))
def begin_chunk(self, room: int, now_mono_ns: int, now_wall_ns: int):
"""Stamp a real enqueue. Returns ``(chunk_seq, first_enqueue_wall_ns)``.
A new chunk clears a previous cancellation mark: it is the start of a
fresh flow (a retry may land on the same room id).
"""
with self._lock:
self._cancelled.pop(int(room), None)
seq = int(self._seq.get(int(room), 0))
self._seq[int(room)] = seq + 1
first = self._first.get(int(room))
if first is None:
first = (int(now_mono_ns), int(now_wall_ns))
self._first[int(room)] = first
return seq, int(first[1])
def record_join(
self,
room: int,
*,
status: str,
room_failed: bool,
now_wall_ns: int,
queue_wait_total_ns: int,
worker_busy_ns: int,
enqueue_wall_ns: int,
is_last_chunk: bool,
chunk_seq: int,
) -> bool:
"""Update the per-room join summary; False when the write was refused."""
if status != "ok":
return False
with self._lock:
if room_failed or int(room) in self._cancelled:
return False
stats = self._join.get(int(room))
if stats is None:
stats = {"chunk_count": 0}
self._join[int(room)] = stats
stats["chunk_count"] = int(stats.get("chunk_count", 0)) + 1
first = self._first.get(int(room))
if first is not None:
stats["first_enqueue_wall_ns"] = int(first[1])
if is_last_chunk:
stats["last_chunk_enqueue_wall_ns"] = int(enqueue_wall_ns)
stats["last_chunk_complete_wall_ns"] = int(now_wall_ns)
stats["last_chunk_queue_wait_ms"] = queue_wait_total_ns / 1e6
stats["last_chunk_worker_ms"] = worker_busy_ns / 1e6
stats["last_chunk_chunk_seq"] = int(chunk_seq)
return True
def reset(self, room: int) -> None:
with self._lock:
self._join.pop(int(room), None)
self._seq.pop(int(room), None)
self._first.pop(int(room), None)
self._mark_cancelled(room)
def teardown(self, room: int, keep_join: bool) -> None:
with self._lock:
if not keep_join:
self._join.pop(int(room), None)
self._seq.pop(int(room), None)
self._first.pop(int(room), None)
self._cancelled.pop(int(room), None)
def pop_join(self, room: int):
with self._lock:
return self._join.pop(int(room), None)
def note_chunk_requeue(chunk, now_mono_ns: int, now_wall_ns: int) -> None:
"""Accumulate only the *queue* part of a finished attempt before re-enqueue.
The previous implementation added ``requeue - last_enqueue``, which also
contains the worker's own execution and the staging watermark wait, thus
inflating queue wait on every defer (review finding). Only the interval
from the last enqueue to this attempt's dequeue is queue wait.
"""
if not _MOONCAKE_DETAIL_ENABLED or not getattr(chunk, "observe", False):
return
last_enqueue = int(getattr(chunk, "last_enqueue_mono_ns", 0) or 0)
dequeue = int(getattr(chunk, "dequeue_mono_ns", 0) or 0)
if last_enqueue and dequeue:
chunk.queue_wait_accum_ns = int(
getattr(chunk, "queue_wait_accum_ns", 0) or 0
) + max(0, dequeue - last_enqueue)
chunk.last_enqueue_mono_ns = int(now_mono_ns)
chunk.last_enqueue_wall_ns = int(now_wall_ns)
chunk.dequeue_mono_ns = 0
chunk.requeue_count = int(getattr(chunk, "requeue_count", 0) or 0) + 1
def emit_mooncake_chunk(
*,
chunk,
status: str,
queue_wait_ns: int,
queue_wait_total_ns: int,
source_wait_ns: int,
indices_wait_ns: int,
kv_transfer_ns: int,
state_transfer_ns: int,
aux_transfer_ns: int,
attempt_wall_ns: int,
dequeue_wall_ns: int,
end_wall_ns: int,
n_destinations: int,
source_wait_wall=None,
indices_wait_wall=None,
dp_rank: int = -1,
tp_rank: int = -1,
) -> None:
"""One span per transfer-worker attempt, attributed by room/chunk_seq.
``worker_busy_ms`` follows the design definition: attempt wall time minus
the source/indices event waits (those are blocking waits, not worker work).
``attempt_wall_ms`` is the raw dequeue->end time.
Wait child spans use the real event timestamps captured by the worker.
A wait without a recorded window is attributes-only; no synthetic
``end - duration`` placement is drawn (review finding).
"""
if not _MOONCAKE_DETAIL_ENABLED:
return
try:
from opentelemetry import trace as otel_trace
tracer = otel_trace.get_tracer("sglang.mooncake.chunk")
# Attempt-scoped start: the last enqueue of this attempt, not the
# first enqueue of the chunk, so deferred retries do not overlap.
start_wall_ns = int(
getattr(chunk, "last_enqueue_wall_ns", 0)
or chunk.enqueue_wall_ns
or dequeue_wall_ns
) or end_wall_ns
worker_busy_ns = max(
0, int(attempt_wall_ns) - int(source_wait_ns) - int(indices_wait_ns)
)
residual_ns = max(
0,
worker_busy_ns
- int(kv_transfer_ns)
- int(state_transfer_ns)
- int(aux_transfer_ns),
)
span = tracer.start_span(
"mooncake.kv_chunk",
context=_chunk_parent_context(chunk),
start_time=start_wall_ns,
attributes={
"sglang.bootstrap_room": int(chunk.room),
"sglang.chunk_seq": int(chunk.chunk_seq),
"sglang.loop_seq": int(chunk.loop_seq),
"sglang.send_kind": str(chunk.send_kind or "unknown"),
"sglang.is_last_chunk": bool(chunk.is_last_chunk),
"sglang.queue_id": int(chunk.queue_id),
"sglang.requeue_count": int(chunk.requeue_count),
"sglang.num_kv_tokens": int(chunk.num_kv_tokens or 0),
"sglang.num_pages": int(chunk.num_pages),
"sglang.approx_bytes": int(chunk.approx_bytes),
"sglang.n_destinations": int(n_destinations),
"sglang.chunk_status": str(status),
"sglang.dp_rank": int(dp_rank),
"sglang.tp_rank": int(tp_rank),
# Attempt window (matches the child span) vs cumulative wait
# across deferred attempts (design §5.3 metric).
"sglang.queue_wait_ms": queue_wait_ns / 1e6,
"sglang.queue_wait_total_ms": queue_wait_total_ns / 1e6,
"sglang.source_wait_ms": source_wait_ns / 1e6,
"sglang.indices_wait_ms": indices_wait_ns / 1e6,
"sglang.kv_transfer_ms": kv_transfer_ns / 1e6,
"sglang.state_transfer_ms": state_transfer_ns / 1e6,
"sglang.aux_transfer_ms": aux_transfer_ns / 1e6,
"sglang.attempt_wall_ms": attempt_wall_ns / 1e6,
"sglang.worker_busy_ms": worker_busy_ns / 1e6,
"sglang.worker_residual_ms": residual_ns / 1e6,
},
)
span_ctx = otel_trace.set_span_in_context(span)
def _child(name, start_ns, stop_ns, wait_ns):
start_ns = int(start_ns or 0)
stop_ns = int(stop_ns or 0)
if start_ns <= 0 or stop_ns <= start_ns:
return
try:
tracer.start_span(
name,
context=span_ctx,
start_time=start_ns,
attributes={"sglang.wait_ms": max(0, int(wait_ns)) / 1e6},
).end(end_time=stop_ns)
except Exception:
pass
# Real event windows only; queue wait ends at dequeue. Zero-wait
# attempts (immediate dequeue) get no span.
if queue_wait_ns > 0:
_child(
"mooncake.queue_wait",
getattr(chunk, "last_enqueue_wall_ns", 0),
dequeue_wall_ns,
queue_wait_ns,
)
if source_wait_wall is not None:
_child(
"mooncake.wait_source",
source_wait_wall[0],
source_wait_wall[1],
source_wait_ns,
)
if indices_wait_wall is not None:
_child(
"mooncake.wait_indices",
indices_wait_wall[0],
indices_wait_wall[1],
indices_wait_ns,
)
span.end(end_time=int(end_wall_ns))
except Exception:
# Telemetry must never break the transfer worker.
pass
def emit_request_kv_join(
*,
req,
stats: dict,
dp_rank: int = -1,
completion_wall_ns: Optional[int] = None,
) -> None:
"""Request-level join summary: last-chunk exposure + completion poll."""
if not _MOONCAKE_DETAIL_ENABLED or not stats:
return
try:
from opentelemetry import trace as otel_trace
from opentelemetry.trace import Link
now_wall_ns = int(completion_wall_ns or time.time_ns())
enqueue_wall_ns = int(stats.get("last_chunk_enqueue_wall_ns", 0) or 0)
complete_wall_ns = int(stats.get("last_chunk_complete_wall_ns", 0) or 0)
final_exposed_ns = (
max(0, complete_wall_ns - enqueue_wall_ns)
if enqueue_wall_ns and complete_wall_ns
else 0
)
completion_poll_ns = (
max(0, now_wall_ns - complete_wall_ns) if complete_wall_ns else 0
)
# First enqueue comes from the manager's real queue.put() event
# (wall clock), not from pre-send preparation (review finding).
first_enqueue_wall_ns = int(stats.get("first_enqueue_wall_ns", 0) or 0)
first_enqueue_to_completion_ms = None
if first_enqueue_wall_ns:
first_enqueue_to_completion_ms = max(
0.0, (now_wall_ns - first_enqueue_wall_ns) / 1e6
)
req_ctx = _request_span_context(req)
attrs = {
"sglang.bootstrap_room": int(getattr(req, "bootstrap_room", 0) or 0),
"sglang.rid": str(getattr(req, "rid", "")),
"sglang.dp_rank": int(dp_rank),
"sglang.chunk_count": int(stats.get("chunk_count", 0)),
"sglang.last_chunk_queue_wait_ms": float(
stats.get("last_chunk_queue_wait_ms", 0.0)
),
"sglang.last_chunk_worker_ms": float(
stats.get("last_chunk_worker_ms", 0.0)
),
"sglang.final_exposed_ms": final_exposed_ns / 1e6,
"sglang.completion_poll_ms": completion_poll_ns / 1e6,
# Per-rank request join. rank_join_skew_ms (max-min over TP/DP
# ranks) is computed offline by joining this span by room.
"sglang.request_join_ms": (final_exposed_ns + completion_poll_ns) / 1e6,
}
if first_enqueue_to_completion_ms is not None:
attrs["sglang.first_enqueue_to_completion_ms"] = (
first_enqueue_to_completion_ms
)
tracer = otel_trace.get_tracer("sglang.kv.request")
tracer.start_span(
"kv.request_join",
# Cover the whole request transfer when first enqueue is known.
start_time=first_enqueue_wall_ns or enqueue_wall_ns or now_wall_ns,
links=[Link(req_ctx)] if req_ctx is not None else [],
attributes=attrs,
).end(end_time=now_wall_ns)
except Exception:
pass
def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None:
"""Record the recv_requests phase inside the current loop (if any)."""
lt = getattr(_LOOP_TLS, "current", None)
@@ -1176,18 +1746,31 @@ def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
},
)
lt.seq = int(t.seq)
lt.pass_end_wall_ns = end_wall_ns
# Estimated DP-sync arrival: the pass ended, then mlp_sync waited.
# See design §1.2: arrival_est = pass_end - mlp_sync.
lt.arrival_est_wall_ns = end_wall_ns - int(t.mlp_sync * 1e9)
def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
attrs = dict(attrs or {})
attrs.setdefault(
"sglang.run_batch_ms", max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6
)
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), attrs)
def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
lt = getattr(_LOOP_TLS, "current", None)
if lt is not None:
lt.process = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
attrs = dict(attrs or {})
attrs.setdefault(
"sglang.process_result_ms",
max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6,
)
lt.process = (int(start_wall_ns), int(end_wall_ns), attrs)
def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
@@ -1351,6 +1934,143 @@ def _batch_attrs(batch) -> dict:
}
_LAST_LOOP_SEQ = 0
_LAST_LOOP_HAS_DETAIL = False
def _total_ms(ms: dict, base: str) -> float:
return sum(
value for name, value in ms.items() if name == base or name.startswith(base + ".")
)
def _update_prev_state(lt: LoopTimings, loop_seq: int):
"""Capture the previous loop summary and store this loop for the next one.
Returns ``(prev_seq, prev_has_detail, prev_summary, consecutive)``. Only a
loop that actually recorded detail becomes a valid predecessor, so a
sampled loop after an unsampled one is marked non-consecutive instead of
silently publishing a stale window (design §4.5).
"""
global _LAST_LOOP_SEQ, _LAST_LOOP_HAS_DETAIL
prev_seq = int(_LAST_LOOP_SEQ)
prev_has = bool(_LAST_LOOP_HAS_DETAIL)
prev_summary = dict(_LAST_LOOP_SUMMARY)
consecutive = prev_has and prev_seq == loop_seq - 1
ms = {
name: rec[0] / 1e6
for name, rec in ((lt.detail or {}).get("phases") or {}).items()
if rec
}
_LAST_LOOP_SUMMARY.clear()
if lt.detail is not None:
_LAST_LOOP_SUMMARY.update(
{
"run_batch_ms": float(
lt.run_batch[2].get("sglang.run_batch_ms", 0.0)
)
if lt.run_batch is not None
else 0.0,
"process_result_ms": float(
lt.process[2].get("sglang.process_result_ms", 0.0)
)
if lt.process is not None
else 0.0,
"copy_done_wait_ms": _total_ms(ms, "copy_done_wait"),
"kv_state_payload_ms": _total_ms(ms, "kv_state_payload"),
"kv_page_indices_ms": _total_ms(ms, "kv_page_indices"),
"kv_sender_send_ms": _total_ms(ms, "kv_sender_send"),
}
)
_LAST_LOOP_SEQ = int(loop_seq)
_LAST_LOOP_HAS_DETAIL = lt.detail is not None
return prev_seq, prev_has, prev_summary, consecutive
def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None:
"""Expose detail phases on the loop span and its child spans.
Single-occurrence phases become children (``scheduler.loop.<name>``);
repeated phases only contribute summed attributes, so span count never
grows with n_reqs / n_segments.
"""
detail = lt.detail or {}
phases = detail.get("phases") or {}
ms = {}
for name, rec in phases.items():
try:
total_ns, count, start_wall, end_wall = rec
except Exception:
continue
key = "sglang." + name.replace(".", "_") + "_ms"
ms[name] = total_ns / 1e6
attrs[key] = ms[name]
if count > 1:
attrs["sglang." + name.replace(".", "_") + "_count"] = int(count)
continue
if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES:
continue
child(
"scheduler.loop." + name.replace(".", "_"),
start_wall,
end_wall,
{
"sglang.dp_rank": int(dp_rank),
"sglang.phase_ms": total_ns / 1e6,
},
)
for name, value in (detail.get("counts") or {}).items():
attrs["sglang." + name.replace(".", "_") + "_count"] = int(value)
for name, value in (detail.get("attrs") or {}).items():
attrs["sglang." + name.replace(".", "_")] = value
# Accounting residuals (design §3.2): never negative, never folded in.
def _total(*bases):
total = 0.0
for base in bases:
for name, value in ms.items():
if name == base or name.startswith(base + "."):
total += value
return total
# Roll per-kind variants (kv_page_indices.last, ...) up to one base
# attribute so analysis can use the total without knowing the kinds.
bases = {name.split(".", 1)[0] for name in ms}
for base in bases:
if base not in ms:
attrs["sglang." + base + "_ms"] = _total(base)
if lt.run_batch is not None:
run_ms = float(lt.run_batch[2].get("sglang.run_batch_ms", 0.0))
parts = _total(
"early_send_host",
"resolve_inputs",
"model_forward_host",
"copy_result_submit",
)
attrs["sglang.run_residual_ms"] = max(0.0, run_ms - parts)
if lt.process is not None:
proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0))
parts = _total(
"copy_done_wait",
"auxiliary_and_logprob",
"per_request_result",
"report_stats",
)
if "sglang.process_kv_ms" in lt.process[2]:
# KV phases charged to this call only (sends issued later in the
# loop, e.g. pending-bootstrap, are not part of process_result).
parts += float(lt.process[2]["sglang.process_kv_ms"])
else:
parts += _total(
"kv_state_payload", "kv_page_indices", "kv_sender_send"
)
attrs["sglang.process_residual_ms"] = max(0.0, proc_ms - parts)
def end_loop(
lt: Optional[LoopTimings],
*,
@@ -1419,6 +2139,14 @@ def end_loop(
pass
lt.no_batch_reason = reason
# Update the prev-loop chain on every loop (sampled or not) so a
# sampled loop after an unsampled one is reported as non-consecutive
# instead of silently carrying a stale window (review finding).
loop_seq_cur = int(lt.seq or lt.predicted_seq)
prev_state = None
if _DETAIL_ENABLED:
prev_state = _update_prev_state(lt, loop_seq_cur)
if lt.span is None and (has_batch or pending):
# Covers work discovered after begin_loop, including first input and
# deferred-only loops. DP outcomes activate earlier for pass parenting.
@@ -1445,7 +2173,6 @@ def end_loop(
"sglang.forward_overlap": overlap,
}
attrs.update(_batch_attrs(batch))
lt.span.set_attributes(attrs)
def _child(name, start_ns, end_ns, child_attrs):
try:
@@ -1458,6 +2185,26 @@ def end_loop(
except Exception:
pass
if lt.detail is not None:
_emit_loop_detail(lt, attrs, _child, int(dp_rank))
if _DETAIL_ENABLED:
if lt.pass_end_wall_ns:
# DP-skew join fields (design §4.5); independent of phases.
attrs["sglang.pass_end_wall_ns"] = int(lt.pass_end_wall_ns)
attrs["sglang.arrival_est_wall_ns"] = int(lt.arrival_est_wall_ns)
if prev_state is not None:
prev_seq, prev_has, prev_summary, consecutive = prev_state
attrs["sglang.prev_loop_seq"] = int(prev_seq)
if consecutive:
for key in _DETAIL_PREV_KEYS:
if key in prev_summary:
attrs["sglang.prev_" + key] = prev_summary[key]
elif prev_seq:
# The previous loop was unsampled or had no detail;
# its phases must not be treated as seq-1 values.
attrs["sglang.prev_detail_missing"] = True
lt.span.set_attributes(attrs)
if lt.recv is not None and not lt.recv_span_child:
start, end, parts = lt.recv
child = {"sglang.dp_rank": int(dp_rank)}