observability: split prefill run_batch/process_result into host phases (M1)
Implements M1 of handoff/prefill-cpu-kv-overlap-design-20261106.md: per-loop detail phases behind SGLANG_TRACE_LOOP_DETAIL=1 (requires SGLANG_BATCH_STAGE_TIMING=1 + SGLANG_TRACE_LOOP_SPAN=1, only sampled loops). - run_batch: early_send_host, resolve_inputs, model_forward_host, copy_result_submit (+ run_residual_ms) - process_result: copy_done_wait, auxiliary_and_logprob, per_request_result, kv_state_payload/page_indices/sender_send by send_kind, report_stats (+ process_residual_ms) - war_barrier is measured as its own loop phase (outside run_batch's span) - repeated/per-request phases are summed attributes; single phases are child spans, so span count never grows with n_reqs/n_segments - DP-skew join fields: pass_end_wall_ns, arrival_est_wall_ns = pass_end - mlp_sync, previous-loop phase summary (sglang.prev_*, published only for consecutive loop_seq; prev_detail_missing marks a gap) No CUDA syncs added; when disabled every hook is a module-level bool check. Unit tests: test_loop_detail_phases.py (10 cases).
This commit is contained in:
@@ -952,7 +952,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self._apply_war_barrier()
|
||||
# WAR barrier is outside run_batch's timing boundary, so it is
|
||||
# recorded as its own loop phase (design §4.3).
|
||||
with batch_telemetry.detail_phase("war_barrier"):
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
@@ -981,13 +984,32 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
_pr_t0 = (
|
||||
time.time_ns() if _lt is not None and _lt.span is not None else 0
|
||||
)
|
||||
# Scope KV phases to this call: sends issued later in the loop
|
||||
# (inflight-queue pending bootstrap) must not be charged here.
|
||||
_proc_kv_phases = (
|
||||
"kv_state_payload",
|
||||
"kv_page_indices",
|
||||
"kv_sender_send",
|
||||
)
|
||||
_proc_detail = batch_telemetry.loop_detail_enabled()
|
||||
_proc_kv_before = (
|
||||
batch_telemetry.detail_phase_total_ns(_proc_kv_phases)
|
||||
if _proc_detail
|
||||
else 0
|
||||
)
|
||||
self.process_batch_result(oldest_batch, oldest_batch_result)
|
||||
_pr_attrs = {"sglang.n_reqs": len(oldest_batch.reqs)}
|
||||
if _proc_detail:
|
||||
_pr_attrs["sglang.process_kv_ms"] = (
|
||||
batch_telemetry.detail_phase_total_ns(_proc_kv_phases)
|
||||
- _proc_kv_before
|
||||
) / 1e6
|
||||
if _pr_t0:
|
||||
try:
|
||||
batch_telemetry.mark_process(
|
||||
_pr_t0,
|
||||
time.time_ns(),
|
||||
{"sglang.n_reqs": len(oldest_batch.reqs)},
|
||||
_pr_attrs,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -1057,7 +1079,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
|
||||
if copy_done is not None:
|
||||
copy_done.synchronize()
|
||||
with batch_telemetry.detail_phase("copy_done_wait"):
|
||||
copy_done.synchronize()
|
||||
_aux_t0 = (
|
||||
time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None
|
||||
)
|
||||
auxiliary_output_starts = (
|
||||
self.batch_result_processor.snapshot_auxiliary_output_starts(batch, result)
|
||||
)
|
||||
@@ -1103,10 +1129,29 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if extend_logprob_start_len < extend_input_len:
|
||||
logprob_pt += extend_input_len - extend_logprob_start_len
|
||||
|
||||
# Auxiliary finalize / logits-to-CPU work is a contiguous host phase.
|
||||
if _aux_t0 is not None:
|
||||
batch_telemetry.detail_add(
|
||||
"auxiliary_and_logprob", _aux_t0, time.monotonic_ns()
|
||||
)
|
||||
# Per-request result handling is everything in this loop except the KV
|
||||
# phases; the delta below keeps `continue` paths accounted for without
|
||||
# measuring each branch.
|
||||
_pr_loop_t0 = (
|
||||
time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None
|
||||
)
|
||||
_kv_phases = ("kv_state_payload", "kv_page_indices", "kv_sender_send")
|
||||
_kv_before = (
|
||||
batch_telemetry.detail_phase_total_ns(_kv_phases)
|
||||
if _pr_loop_t0 is not None
|
||||
else 0
|
||||
)
|
||||
|
||||
for i, (req, next_token_id) in enumerate(
|
||||
zip(batch.reqs, next_token_ids, strict=True)
|
||||
):
|
||||
if req.inflight_middle_chunks <= 0:
|
||||
batch_telemetry.detail_count("n_last_reqs")
|
||||
req.time_stats.set_prefill_finished_time()
|
||||
|
||||
# Its KV never landed: abort instead of sending it to decode.
|
||||
@@ -1210,6 +1255,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
else:
|
||||
# being chunked reqs' prefill is not finished
|
||||
batch_telemetry.detail_count("n_middle_reqs")
|
||||
req.inflight_middle_chunks -= 1
|
||||
|
||||
# Still chunking iff its next chunk was launched: either it is
|
||||
@@ -1260,12 +1306,27 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self.send_kv_chunk(req, last_chunk=False, end_idx=req.tmp_end_idx)
|
||||
req.time_stats.set_last_chunked_prefill_finish_time()
|
||||
|
||||
if _pr_loop_t0 is not None:
|
||||
_kv_ns = batch_telemetry.detail_phase_total_ns(_kv_phases) - _kv_before
|
||||
batch_telemetry.detail_add(
|
||||
"per_request_result",
|
||||
_pr_loop_t0 + max(0, _kv_ns),
|
||||
time.monotonic_ns(),
|
||||
)
|
||||
|
||||
if auxiliary_output is not None:
|
||||
_aux2_t0 = (
|
||||
time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None
|
||||
)
|
||||
self.batch_result_processor.consume_auxiliary_output(
|
||||
batch,
|
||||
auxiliary_output,
|
||||
auxiliary_output_starts,
|
||||
)
|
||||
if _aux2_t0 is not None:
|
||||
batch_telemetry.detail_add(
|
||||
"auxiliary_and_logprob", _aux2_t0, time.monotonic_ns()
|
||||
)
|
||||
|
||||
if aborted_reqs:
|
||||
self.output_streamer.stream_output(
|
||||
@@ -1274,12 +1335,13 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
|
||||
can_run_cuda_graph = result.can_run_cuda_graph
|
||||
self.metrics_reporter.report_prefill_stats(
|
||||
batch=batch,
|
||||
prefill_stats=batch.prefill_stats,
|
||||
can_run_cuda_graph=can_run_cuda_graph,
|
||||
dp_cooperation_info=batch.dp_cooperation_info,
|
||||
)
|
||||
with batch_telemetry.detail_phase("report_stats"):
|
||||
self.metrics_reporter.report_prefill_stats(
|
||||
batch=batch,
|
||||
prefill_stats=batch.prefill_stats,
|
||||
can_run_cuda_graph=can_run_cuda_graph,
|
||||
dp_cooperation_info=batch.dp_cooperation_info,
|
||||
)
|
||||
self.maybe_send_health_check_signal()
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
|
||||
@@ -1627,14 +1689,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
ev = torch.cuda.Event()
|
||||
ev.record(self.forward_stream)
|
||||
req.disagg_kv_sender._early_send_wait_event = ev
|
||||
self.send_kv_chunk(req, last_chunk=False, end_idx=cached_end)
|
||||
if batch_telemetry.loop_detail_enabled():
|
||||
batch_telemetry.detail_count("early_send_reqs")
|
||||
self.send_kv_chunk(
|
||||
req, last_chunk=False, end_idx=cached_end, send_kind="cached_prefix"
|
||||
)
|
||||
|
||||
def send_kv_chunk(
|
||||
self,
|
||||
req: Req,
|
||||
last_chunk: bool = False,
|
||||
end_idx: Optional[int] = None,
|
||||
send_kind: Optional[str] = None,
|
||||
) -> None:
|
||||
if send_kind is None:
|
||||
send_kind = "last" if last_chunk else "middle"
|
||||
computer: Optional[KvChecksumComputer] = self.kv_checksum_computer
|
||||
if last_chunk and computer is not None:
|
||||
if is_health_check_req(req):
|
||||
@@ -1647,7 +1716,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
value = computer.compute(page_indices_gpu, state_indices)
|
||||
self.disagg_metadata_buffers.set_kv_checksum(req, value)
|
||||
_kv_t0 = time.time_ns() if batch_telemetry.kv_send_span_enabled() else 0
|
||||
self._send_kv_chunk(req, last_chunk=last_chunk, end_idx=end_idx)
|
||||
if batch_telemetry.loop_detail_enabled():
|
||||
batch_telemetry.detail_count("n_kv_calls")
|
||||
self._send_kv_chunk(
|
||||
req, last_chunk=last_chunk, end_idx=end_idx, send_kind=send_kind
|
||||
)
|
||||
if _kv_t0:
|
||||
batch_telemetry.add_kv_send(_kv_t0, time.time_ns())
|
||||
|
||||
@@ -1656,6 +1729,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
req: Req,
|
||||
last_chunk: bool = False,
|
||||
end_idx: Optional[int] = None,
|
||||
send_kind: str = "last",
|
||||
) -> None:
|
||||
"""
|
||||
Send a prefilled chunk to the decode server
|
||||
@@ -1690,6 +1764,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
return
|
||||
|
||||
state_indices: Optional[List] = None
|
||||
_state_tok = (
|
||||
batch_telemetry.detail_begin("kv_state_payload." + send_kind)
|
||||
if last_chunk
|
||||
else None
|
||||
)
|
||||
if last_chunk:
|
||||
self.disagg_metadata_buffers.set_buf(req)
|
||||
|
||||
@@ -1828,6 +1907,12 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
state_indices = [
|
||||
payloads[st]() if st in payloads else None for st in state_types
|
||||
]
|
||||
if _state_tok is not None:
|
||||
batch_telemetry.detail_attr(
|
||||
"state_types",
|
||||
",".join(sorted(str(getattr(st, "name", st)) for st in state_types)),
|
||||
)
|
||||
batch_telemetry.detail_end("kv_state_payload." + send_kind, _state_tok)
|
||||
|
||||
transfer_chunk_tokens = req.disagg_kv_sender.get_max_transfer_tokens()
|
||||
if self.enable_staging:
|
||||
@@ -1848,7 +1933,16 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
else:
|
||||
segments = [(start_idx, end_idx)]
|
||||
|
||||
# Classify the chunk for detail accounting (cached_prefix/middle/last).
|
||||
# Set on the sender so backend send() signatures stay unchanged.
|
||||
req.disagg_kv_sender._send_kind = send_kind
|
||||
|
||||
for seg_start, seg_end in segments:
|
||||
_seg_t0 = (
|
||||
time.monotonic_ns()
|
||||
if batch_telemetry.loop_detail_enabled()
|
||||
else None
|
||||
)
|
||||
is_final_segment = seg_end == end_idx
|
||||
raw_kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, seg_start:seg_end
|
||||
@@ -1880,10 +1974,20 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
indices_event.record(torch.cuda.current_stream())
|
||||
else:
|
||||
page_indices = kv_to_page_indices(kv_indices, page_size)
|
||||
if _seg_t0 is not None:
|
||||
batch_telemetry.detail_add(
|
||||
"kv_page_indices." + send_kind, _seg_t0, time.monotonic_ns()
|
||||
)
|
||||
batch_telemetry.detail_count("n_pages", len(page_indices))
|
||||
segment_is_last = last_chunk and is_final_segment
|
||||
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
|
||||
continue
|
||||
send_state_indices = state_indices if segment_is_last else None
|
||||
_send_t0 = (
|
||||
time.monotonic_ns()
|
||||
if batch_telemetry.loop_detail_enabled()
|
||||
else None
|
||||
)
|
||||
if indices_event is not None:
|
||||
sender.send(
|
||||
page_indices,
|
||||
@@ -1897,6 +2001,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
send_state_indices,
|
||||
num_kv_tokens=seg_end - seg_start,
|
||||
)
|
||||
if _send_t0 is not None:
|
||||
batch_telemetry.detail_add(
|
||||
"kv_sender_send." + send_kind, _send_t0, time.monotonic_ns()
|
||||
)
|
||||
batch_telemetry.detail_count("n_segments")
|
||||
req.start_send_idx = end_idx
|
||||
# A last chunk needs no entry: every `last_chunk=True` call site has
|
||||
# already put the request on `disagg_prefill_inflight_queue`.
|
||||
|
||||
@@ -4645,8 +4645,9 @@ class Scheduler(
|
||||
|
||||
# PD prefill: early-send cached prefix KV, overlapping the suffix forward.
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
for req in batch.reqs:
|
||||
self.maybe_send_cached_prefix_chunk(req)
|
||||
with batch_telemetry.detail_phase("early_send_host"):
|
||||
for req in batch.reqs:
|
||||
self.maybe_send_cached_prefix_chunk(req)
|
||||
|
||||
# Run forward
|
||||
if self.is_generation:
|
||||
@@ -4673,7 +4674,8 @@ class Scheduler(
|
||||
# mix_running_indices). Run OUTSIDE isolation so the
|
||||
# snapshot captures the post-consume state — restoring
|
||||
# post-forward must not un-consume staging.
|
||||
resolve_forward_inputs(batch, self.future_map)
|
||||
with batch_telemetry.detail_phase("resolve_inputs"):
|
||||
resolve_forward_inputs(batch, self.future_map)
|
||||
|
||||
with self._forward_isolation(batch, overlap=True):
|
||||
future_indices = batch.req_pool_indices
|
||||
@@ -4696,9 +4698,10 @@ class Scheduler(
|
||||
)
|
||||
|
||||
# FIXME: pp is not compatible with overlap
|
||||
batch_result = self.model_worker.forward_batch_generation(
|
||||
batch, **fwd_kwargs
|
||||
)
|
||||
with batch_telemetry.detail_phase("model_forward_host"):
|
||||
batch_result = self.model_worker.forward_batch_generation(
|
||||
batch, **fwd_kwargs
|
||||
)
|
||||
if batch.spec_algorithm.is_none():
|
||||
self.future_map.publish(future_indices, batch.seq_lens + 1)
|
||||
# Park any refs the worker wants kept alive 2 iters
|
||||
@@ -4736,7 +4739,10 @@ class Scheduler(
|
||||
return_hidden_states=batch.return_hidden_states,
|
||||
)
|
||||
else:
|
||||
self._copy_result_to_cpu(batch_result, batch)
|
||||
with batch_telemetry.detail_phase(
|
||||
"copy_result_submit"
|
||||
):
|
||||
self._copy_result_to_cpu(batch_result, batch)
|
||||
else:
|
||||
batch_result.future_indices = future_indices
|
||||
|
||||
|
||||
@@ -20,6 +20,13 @@ Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
|
||||
SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0)
|
||||
SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span
|
||||
|
||||
SGLANG_TRACE_LOOP_DETAIL=1 split run_batch/process_result into host
|
||||
phases (requires SGLANG_BATCH_STAGE_TIMING=1
|
||||
and SGLANG_TRACE_LOOP_SPAN=1; only sampled
|
||||
loops are recorded)
|
||||
SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown
|
||||
(queue/source/indices/transfer/join)
|
||||
|
||||
Cost model (measured on our 4xDP4 / 4-GPU instances):
|
||||
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
|
||||
* span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s.
|
||||
@@ -1033,6 +1040,9 @@ class LoopTimings:
|
||||
"launch_blocked_reason",
|
||||
"seq",
|
||||
"no_batch_reason",
|
||||
"pass_end_wall_ns",
|
||||
"arrival_est_wall_ns",
|
||||
"detail",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -1066,6 +1076,11 @@ class LoopTimings:
|
||||
self.launch_blocked_reason = ""
|
||||
self.seq = 0
|
||||
self.no_batch_reason = ""
|
||||
self.pass_end_wall_ns = 0
|
||||
self.arrival_est_wall_ns = 0
|
||||
# Only populated when SGLANG_TRACE_LOOP_DETAIL=1 and this loop is
|
||||
# sampled; see the "Loop detail phases" section below.
|
||||
self.detail = None
|
||||
|
||||
|
||||
def ensure_loop_span(lt: Optional[LoopTimings]) -> None:
|
||||
@@ -1137,6 +1152,561 @@ def current_loop_context():
|
||||
return lt.span_ctx if lt is not None else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Loop detail phases (SGLANG_TRACE_LOOP_DETAIL=1).
|
||||
#
|
||||
# Splits run_batch / process_result into host phases. Gated twice: the module
|
||||
# switch must be on AND the current loop must be sampled (same deterministic
|
||||
# decision as the loop span), so detail data never covers an unsampled loop.
|
||||
#
|
||||
# Accounting rules (see handoff/prefill-cpu-kv-overlap-design-20261106.md §3.2):
|
||||
# * single phases -> one child span (``scheduler.loop.<name>``)
|
||||
# * repeated phases -> summed attributes only (``sglang.<name>_ms`` + count)
|
||||
# * durations use time.monotonic_ns(); OTLP start/end use time.time_ns()
|
||||
# * residuals are clamped >= 0 and reported, never folded into a phase
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DETAIL_ENABLED = _LOOP_SPAN_ENABLED and os.getenv(
|
||||
"SGLANG_TRACE_LOOP_DETAIL", "0"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
|
||||
_DETAIL_TLS = threading.local()
|
||||
|
||||
# Chunk-level (Mooncake transfer worker) detail tracing. Independent switch:
|
||||
# the worker runs on other threads and has no scheduler loop TLS.
|
||||
_MOONCAKE_DETAIL_ENABLED = os.getenv(
|
||||
"SGLANG_TRACE_MOONCAKE_DETAIL", "0"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
|
||||
# Previous loop summary per scheduler thread, used for DP-skew correlation
|
||||
# (previous loop's phases vs this loop's arrival skew).
|
||||
_LAST_LOOP_SUMMARY: dict = {}
|
||||
|
||||
# Phases that are inherently per-request / per-segment: always attributes,
|
||||
# even when a loop happens to have exactly one occurrence, so span count never
|
||||
# grows with n_reqs / n_segments (design §4.2).
|
||||
_DETAIL_ATTRIBUTE_ONLY_PREFIXES = (
|
||||
"per_request_result",
|
||||
"kv_state_payload",
|
||||
"kv_page_indices",
|
||||
"kv_sender_send",
|
||||
)
|
||||
|
||||
_DETAIL_PREV_KEYS = (
|
||||
"run_batch_ms",
|
||||
"process_result_ms",
|
||||
"copy_done_wait_ms",
|
||||
"kv_state_payload_ms",
|
||||
"kv_page_indices_ms",
|
||||
"kv_sender_send_ms",
|
||||
)
|
||||
|
||||
|
||||
def loop_detail_enabled() -> bool:
|
||||
"""True when detail is on and the *current* loop is sampled."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return False
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
return lt is not None and loop_sample_decision(lt.predicted_seq)
|
||||
|
||||
|
||||
def mooncake_detail_enabled() -> bool:
|
||||
return _MOONCAKE_DETAIL_ENABLED
|
||||
|
||||
|
||||
def _detail_state():
|
||||
"""Detail state of the current loop, only when that loop is sampled."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return None
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is None or not loop_sample_decision(lt.predicted_seq):
|
||||
return None
|
||||
if lt.detail is None:
|
||||
lt.detail = {"phases": {}, "counts": {}}
|
||||
return lt.detail
|
||||
|
||||
|
||||
def detail_begin(name: str) -> Optional[int]:
|
||||
"""Start a single-occurrence phase. Returns a monotonic-ns token."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return None
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return None
|
||||
now_ns = time.monotonic_ns()
|
||||
opens = getattr(_DETAIL_TLS, "open", None)
|
||||
if opens is None:
|
||||
opens = {}
|
||||
_DETAIL_TLS.open = opens
|
||||
opens[name] = (now_ns, time.time_ns())
|
||||
return now_ns
|
||||
|
||||
|
||||
def detail_end(name: str, token: Optional[int] = None) -> None:
|
||||
"""Close a phase opened by ``detail_begin`` (or started at ``token``)."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return
|
||||
now_ns = time.monotonic_ns()
|
||||
opens = getattr(_DETAIL_TLS, "open", None) or {}
|
||||
entry = opens.pop(name, None)
|
||||
start_wall_ns = entry[1] if entry is not None else None
|
||||
if token is None:
|
||||
if entry is None:
|
||||
return
|
||||
token = entry[0]
|
||||
if start_wall_ns is None:
|
||||
start_wall_ns = time.time_ns() - max(0, now_ns - int(token))
|
||||
_detail_record(state, name, int(token), now_ns, start_wall_ns, time.time_ns())
|
||||
|
||||
|
||||
def detail_add(name: str, start_ns: int, end_ns: int) -> None:
|
||||
"""Accumulate one occurrence of a repeated phase (no span per occurrence)."""
|
||||
if not _DETAIL_ENABLED or start_ns is None or end_ns is None:
|
||||
return
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return
|
||||
start_ns = int(start_ns)
|
||||
end_ns = int(end_ns)
|
||||
if end_ns < start_ns:
|
||||
end_ns = start_ns
|
||||
now_wall_ns = time.time_ns()
|
||||
_detail_record(
|
||||
state, name, start_ns, end_ns, now_wall_ns - (end_ns - start_ns), now_wall_ns
|
||||
)
|
||||
|
||||
|
||||
def _detail_record(state, name, start_ns, end_ns, start_wall_ns, end_wall_ns) -> None:
|
||||
rec = state["phases"].get(name)
|
||||
if rec is None:
|
||||
state["phases"][name] = [
|
||||
end_ns - start_ns,
|
||||
1,
|
||||
int(start_wall_ns),
|
||||
int(end_wall_ns),
|
||||
]
|
||||
return
|
||||
rec[0] += end_ns - start_ns
|
||||
rec[1] += 1
|
||||
rec[3] = int(end_wall_ns)
|
||||
|
||||
|
||||
def detail_count(name: str, n: int = 1) -> None:
|
||||
"""Accumulate a low-cardinality counter for this loop."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return
|
||||
state["counts"][name] = int(state["counts"].get(name, 0)) + int(n)
|
||||
|
||||
|
||||
def detail_attr(name: str, value) -> None:
|
||||
"""Set a low-cardinality loop attribute (last write wins)."""
|
||||
if not _DETAIL_ENABLED:
|
||||
return
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return
|
||||
state.setdefault("attrs", {})[name] = value
|
||||
|
||||
|
||||
def detail_phase_total_ns(names) -> int:
|
||||
"""Sum of accumulated ns for the given phases in the current loop.
|
||||
|
||||
``names`` are *base* names: a phase recorded as ``kv_sender_send.last``
|
||||
(per send kind) is included when ``kv_sender_send`` is requested. Phase
|
||||
names are only generated by this module, so prefix matching cannot pick up
|
||||
an unrelated phase that merely shares a leading string.
|
||||
"""
|
||||
state = _detail_state()
|
||||
if state is None:
|
||||
return 0
|
||||
phases = state.get("phases") or {}
|
||||
total = 0
|
||||
for name, rec in phases.items():
|
||||
if rec is None:
|
||||
continue
|
||||
for want in names:
|
||||
if name == want or name.startswith(want + "."):
|
||||
total += int(rec[0])
|
||||
break
|
||||
return total
|
||||
|
||||
|
||||
class detail_phase:
|
||||
"""Context manager for one single-occurrence phase (cheap when disabled)."""
|
||||
|
||||
__slots__ = ("name", "token")
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.token = None
|
||||
|
||||
def __enter__(self):
|
||||
self.token = detail_begin(self.name)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
detail_end(self.name, self.token)
|
||||
return False
|
||||
|
||||
|
||||
def current_loop_seq() -> int:
|
||||
"""Loop sequence for cross-record correlation (0 when unavailable)."""
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is None:
|
||||
return 0
|
||||
return int(lt.seq or lt.predicted_seq or 0)
|
||||
|
||||
|
||||
def current_loop_context_for_spans():
|
||||
"""OTel context of the current loop span (None if unsampled)."""
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
return lt.span_ctx if lt is not None else None
|
||||
|
||||
|
||||
def _chunk_parent_context(chunk):
|
||||
trace_ctx = getattr(chunk, "trace_ctx", None)
|
||||
return getattr(trace_ctx, "root_span_context", None)
|
||||
|
||||
|
||||
def split_queue_wait(
|
||||
queue_wait_accum_ns: int, last_enqueue_mono_ns: int, dequeue_mono_ns: int
|
||||
) -> tuple:
|
||||
"""Return ``(attempt_queue_wait_ns, total_queue_wait_ns)``.
|
||||
|
||||
The attempt value matches the last_enqueue->dequeue wall window used by the
|
||||
``mooncake.queue_wait`` child span; the total value additionally includes
|
||||
the queue waits of earlier deferred attempts (design §5.3 metric).
|
||||
Keeping them separate prevents a span whose wall length disagrees with its
|
||||
``wait_ms`` attribute (review finding).
|
||||
"""
|
||||
attempt = max(0, int(dequeue_mono_ns) - int(last_enqueue_mono_ns))
|
||||
return attempt, max(0, int(queue_wait_accum_ns)) + attempt
|
||||
|
||||
|
||||
class RoomDetailState:
|
||||
"""Thread-safe per-room detail bookkeeping (scheduler vs worker threads).
|
||||
|
||||
The scheduler aborts/retracts a room while the transfer worker may still be
|
||||
finishing an attempt. A reset therefore records a cancellation mark, and
|
||||
the worker's join update re-checks under the same lock before writing, so a
|
||||
late worker can never resurrect stats after abort (review finding).
|
||||
"""
|
||||
|
||||
# Bounded: a marker only needs to survive until the in-flight worker
|
||||
# attempt finishes (milliseconds). The cap prevents unbounded growth if an
|
||||
# aborted room never gets a worker teardown.
|
||||
_MAX_CANCELLED = 8192
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._join: dict = {}
|
||||
self._seq: dict = {}
|
||||
self._first: dict = {}
|
||||
self._cancelled: dict = {}
|
||||
|
||||
def _mark_cancelled(self, room: int) -> None:
|
||||
self._cancelled[int(room)] = time.monotonic_ns()
|
||||
while len(self._cancelled) > self._MAX_CANCELLED:
|
||||
self._cancelled.pop(next(iter(self._cancelled)))
|
||||
|
||||
def begin_chunk(self, room: int, now_mono_ns: int, now_wall_ns: int):
|
||||
"""Stamp a real enqueue. Returns ``(chunk_seq, first_enqueue_wall_ns)``.
|
||||
|
||||
A new chunk clears a previous cancellation mark: it is the start of a
|
||||
fresh flow (a retry may land on the same room id).
|
||||
"""
|
||||
with self._lock:
|
||||
self._cancelled.pop(int(room), None)
|
||||
seq = int(self._seq.get(int(room), 0))
|
||||
self._seq[int(room)] = seq + 1
|
||||
first = self._first.get(int(room))
|
||||
if first is None:
|
||||
first = (int(now_mono_ns), int(now_wall_ns))
|
||||
self._first[int(room)] = first
|
||||
return seq, int(first[1])
|
||||
|
||||
def record_join(
|
||||
self,
|
||||
room: int,
|
||||
*,
|
||||
status: str,
|
||||
room_failed: bool,
|
||||
now_wall_ns: int,
|
||||
queue_wait_total_ns: int,
|
||||
worker_busy_ns: int,
|
||||
enqueue_wall_ns: int,
|
||||
is_last_chunk: bool,
|
||||
chunk_seq: int,
|
||||
) -> bool:
|
||||
"""Update the per-room join summary; False when the write was refused."""
|
||||
if status != "ok":
|
||||
return False
|
||||
with self._lock:
|
||||
if room_failed or int(room) in self._cancelled:
|
||||
return False
|
||||
stats = self._join.get(int(room))
|
||||
if stats is None:
|
||||
stats = {"chunk_count": 0}
|
||||
self._join[int(room)] = stats
|
||||
stats["chunk_count"] = int(stats.get("chunk_count", 0)) + 1
|
||||
first = self._first.get(int(room))
|
||||
if first is not None:
|
||||
stats["first_enqueue_wall_ns"] = int(first[1])
|
||||
if is_last_chunk:
|
||||
stats["last_chunk_enqueue_wall_ns"] = int(enqueue_wall_ns)
|
||||
stats["last_chunk_complete_wall_ns"] = int(now_wall_ns)
|
||||
stats["last_chunk_queue_wait_ms"] = queue_wait_total_ns / 1e6
|
||||
stats["last_chunk_worker_ms"] = worker_busy_ns / 1e6
|
||||
stats["last_chunk_chunk_seq"] = int(chunk_seq)
|
||||
return True
|
||||
|
||||
def reset(self, room: int) -> None:
|
||||
with self._lock:
|
||||
self._join.pop(int(room), None)
|
||||
self._seq.pop(int(room), None)
|
||||
self._first.pop(int(room), None)
|
||||
self._mark_cancelled(room)
|
||||
|
||||
def teardown(self, room: int, keep_join: bool) -> None:
|
||||
with self._lock:
|
||||
if not keep_join:
|
||||
self._join.pop(int(room), None)
|
||||
self._seq.pop(int(room), None)
|
||||
self._first.pop(int(room), None)
|
||||
self._cancelled.pop(int(room), None)
|
||||
|
||||
def pop_join(self, room: int):
|
||||
with self._lock:
|
||||
return self._join.pop(int(room), None)
|
||||
|
||||
|
||||
def note_chunk_requeue(chunk, now_mono_ns: int, now_wall_ns: int) -> None:
|
||||
"""Accumulate only the *queue* part of a finished attempt before re-enqueue.
|
||||
|
||||
The previous implementation added ``requeue - last_enqueue``, which also
|
||||
contains the worker's own execution and the staging watermark wait, thus
|
||||
inflating queue wait on every defer (review finding). Only the interval
|
||||
from the last enqueue to this attempt's dequeue is queue wait.
|
||||
"""
|
||||
if not _MOONCAKE_DETAIL_ENABLED or not getattr(chunk, "observe", False):
|
||||
return
|
||||
last_enqueue = int(getattr(chunk, "last_enqueue_mono_ns", 0) or 0)
|
||||
dequeue = int(getattr(chunk, "dequeue_mono_ns", 0) or 0)
|
||||
if last_enqueue and dequeue:
|
||||
chunk.queue_wait_accum_ns = int(
|
||||
getattr(chunk, "queue_wait_accum_ns", 0) or 0
|
||||
) + max(0, dequeue - last_enqueue)
|
||||
chunk.last_enqueue_mono_ns = int(now_mono_ns)
|
||||
chunk.last_enqueue_wall_ns = int(now_wall_ns)
|
||||
chunk.dequeue_mono_ns = 0
|
||||
chunk.requeue_count = int(getattr(chunk, "requeue_count", 0) or 0) + 1
|
||||
|
||||
|
||||
def emit_mooncake_chunk(
|
||||
*,
|
||||
chunk,
|
||||
status: str,
|
||||
queue_wait_ns: int,
|
||||
queue_wait_total_ns: int,
|
||||
source_wait_ns: int,
|
||||
indices_wait_ns: int,
|
||||
kv_transfer_ns: int,
|
||||
state_transfer_ns: int,
|
||||
aux_transfer_ns: int,
|
||||
attempt_wall_ns: int,
|
||||
dequeue_wall_ns: int,
|
||||
end_wall_ns: int,
|
||||
n_destinations: int,
|
||||
source_wait_wall=None,
|
||||
indices_wait_wall=None,
|
||||
dp_rank: int = -1,
|
||||
tp_rank: int = -1,
|
||||
) -> None:
|
||||
"""One span per transfer-worker attempt, attributed by room/chunk_seq.
|
||||
|
||||
``worker_busy_ms`` follows the design definition: attempt wall time minus
|
||||
the source/indices event waits (those are blocking waits, not worker work).
|
||||
``attempt_wall_ms`` is the raw dequeue->end time.
|
||||
|
||||
Wait child spans use the real event timestamps captured by the worker.
|
||||
A wait without a recorded window is attributes-only; no synthetic
|
||||
``end - duration`` placement is drawn (review finding).
|
||||
"""
|
||||
if not _MOONCAKE_DETAIL_ENABLED:
|
||||
return
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
tracer = otel_trace.get_tracer("sglang.mooncake.chunk")
|
||||
# Attempt-scoped start: the last enqueue of this attempt, not the
|
||||
# first enqueue of the chunk, so deferred retries do not overlap.
|
||||
start_wall_ns = int(
|
||||
getattr(chunk, "last_enqueue_wall_ns", 0)
|
||||
or chunk.enqueue_wall_ns
|
||||
or dequeue_wall_ns
|
||||
) or end_wall_ns
|
||||
worker_busy_ns = max(
|
||||
0, int(attempt_wall_ns) - int(source_wait_ns) - int(indices_wait_ns)
|
||||
)
|
||||
residual_ns = max(
|
||||
0,
|
||||
worker_busy_ns
|
||||
- int(kv_transfer_ns)
|
||||
- int(state_transfer_ns)
|
||||
- int(aux_transfer_ns),
|
||||
)
|
||||
span = tracer.start_span(
|
||||
"mooncake.kv_chunk",
|
||||
context=_chunk_parent_context(chunk),
|
||||
start_time=start_wall_ns,
|
||||
attributes={
|
||||
"sglang.bootstrap_room": int(chunk.room),
|
||||
"sglang.chunk_seq": int(chunk.chunk_seq),
|
||||
"sglang.loop_seq": int(chunk.loop_seq),
|
||||
"sglang.send_kind": str(chunk.send_kind or "unknown"),
|
||||
"sglang.is_last_chunk": bool(chunk.is_last_chunk),
|
||||
"sglang.queue_id": int(chunk.queue_id),
|
||||
"sglang.requeue_count": int(chunk.requeue_count),
|
||||
"sglang.num_kv_tokens": int(chunk.num_kv_tokens or 0),
|
||||
"sglang.num_pages": int(chunk.num_pages),
|
||||
"sglang.approx_bytes": int(chunk.approx_bytes),
|
||||
"sglang.n_destinations": int(n_destinations),
|
||||
"sglang.chunk_status": str(status),
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.tp_rank": int(tp_rank),
|
||||
# Attempt window (matches the child span) vs cumulative wait
|
||||
# across deferred attempts (design §5.3 metric).
|
||||
"sglang.queue_wait_ms": queue_wait_ns / 1e6,
|
||||
"sglang.queue_wait_total_ms": queue_wait_total_ns / 1e6,
|
||||
"sglang.source_wait_ms": source_wait_ns / 1e6,
|
||||
"sglang.indices_wait_ms": indices_wait_ns / 1e6,
|
||||
"sglang.kv_transfer_ms": kv_transfer_ns / 1e6,
|
||||
"sglang.state_transfer_ms": state_transfer_ns / 1e6,
|
||||
"sglang.aux_transfer_ms": aux_transfer_ns / 1e6,
|
||||
"sglang.attempt_wall_ms": attempt_wall_ns / 1e6,
|
||||
"sglang.worker_busy_ms": worker_busy_ns / 1e6,
|
||||
"sglang.worker_residual_ms": residual_ns / 1e6,
|
||||
},
|
||||
)
|
||||
span_ctx = otel_trace.set_span_in_context(span)
|
||||
|
||||
def _child(name, start_ns, stop_ns, wait_ns):
|
||||
start_ns = int(start_ns or 0)
|
||||
stop_ns = int(stop_ns or 0)
|
||||
if start_ns <= 0 or stop_ns <= start_ns:
|
||||
return
|
||||
try:
|
||||
tracer.start_span(
|
||||
name,
|
||||
context=span_ctx,
|
||||
start_time=start_ns,
|
||||
attributes={"sglang.wait_ms": max(0, int(wait_ns)) / 1e6},
|
||||
).end(end_time=stop_ns)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Real event windows only; queue wait ends at dequeue. Zero-wait
|
||||
# attempts (immediate dequeue) get no span.
|
||||
if queue_wait_ns > 0:
|
||||
_child(
|
||||
"mooncake.queue_wait",
|
||||
getattr(chunk, "last_enqueue_wall_ns", 0),
|
||||
dequeue_wall_ns,
|
||||
queue_wait_ns,
|
||||
)
|
||||
if source_wait_wall is not None:
|
||||
_child(
|
||||
"mooncake.wait_source",
|
||||
source_wait_wall[0],
|
||||
source_wait_wall[1],
|
||||
source_wait_ns,
|
||||
)
|
||||
if indices_wait_wall is not None:
|
||||
_child(
|
||||
"mooncake.wait_indices",
|
||||
indices_wait_wall[0],
|
||||
indices_wait_wall[1],
|
||||
indices_wait_ns,
|
||||
)
|
||||
span.end(end_time=int(end_wall_ns))
|
||||
except Exception:
|
||||
# Telemetry must never break the transfer worker.
|
||||
pass
|
||||
|
||||
|
||||
def emit_request_kv_join(
|
||||
*,
|
||||
req,
|
||||
stats: dict,
|
||||
dp_rank: int = -1,
|
||||
completion_wall_ns: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Request-level join summary: last-chunk exposure + completion poll."""
|
||||
if not _MOONCAKE_DETAIL_ENABLED or not stats:
|
||||
return
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
from opentelemetry.trace import Link
|
||||
|
||||
now_wall_ns = int(completion_wall_ns or time.time_ns())
|
||||
enqueue_wall_ns = int(stats.get("last_chunk_enqueue_wall_ns", 0) or 0)
|
||||
complete_wall_ns = int(stats.get("last_chunk_complete_wall_ns", 0) or 0)
|
||||
final_exposed_ns = (
|
||||
max(0, complete_wall_ns - enqueue_wall_ns)
|
||||
if enqueue_wall_ns and complete_wall_ns
|
||||
else 0
|
||||
)
|
||||
completion_poll_ns = (
|
||||
max(0, now_wall_ns - complete_wall_ns) if complete_wall_ns else 0
|
||||
)
|
||||
# First enqueue comes from the manager's real queue.put() event
|
||||
# (wall clock), not from pre-send preparation (review finding).
|
||||
first_enqueue_wall_ns = int(stats.get("first_enqueue_wall_ns", 0) or 0)
|
||||
first_enqueue_to_completion_ms = None
|
||||
if first_enqueue_wall_ns:
|
||||
first_enqueue_to_completion_ms = max(
|
||||
0.0, (now_wall_ns - first_enqueue_wall_ns) / 1e6
|
||||
)
|
||||
req_ctx = _request_span_context(req)
|
||||
attrs = {
|
||||
"sglang.bootstrap_room": int(getattr(req, "bootstrap_room", 0) or 0),
|
||||
"sglang.rid": str(getattr(req, "rid", "")),
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.chunk_count": int(stats.get("chunk_count", 0)),
|
||||
"sglang.last_chunk_queue_wait_ms": float(
|
||||
stats.get("last_chunk_queue_wait_ms", 0.0)
|
||||
),
|
||||
"sglang.last_chunk_worker_ms": float(
|
||||
stats.get("last_chunk_worker_ms", 0.0)
|
||||
),
|
||||
"sglang.final_exposed_ms": final_exposed_ns / 1e6,
|
||||
"sglang.completion_poll_ms": completion_poll_ns / 1e6,
|
||||
# Per-rank request join. rank_join_skew_ms (max-min over TP/DP
|
||||
# ranks) is computed offline by joining this span by room.
|
||||
"sglang.request_join_ms": (final_exposed_ns + completion_poll_ns) / 1e6,
|
||||
}
|
||||
if first_enqueue_to_completion_ms is not None:
|
||||
attrs["sglang.first_enqueue_to_completion_ms"] = (
|
||||
first_enqueue_to_completion_ms
|
||||
)
|
||||
tracer = otel_trace.get_tracer("sglang.kv.request")
|
||||
tracer.start_span(
|
||||
"kv.request_join",
|
||||
# Cover the whole request transfer when first enqueue is known.
|
||||
start_time=first_enqueue_wall_ns or enqueue_wall_ns or now_wall_ns,
|
||||
links=[Link(req_ctx)] if req_ctx is not None else [],
|
||||
attributes=attrs,
|
||||
).end(end_time=now_wall_ns)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None:
|
||||
"""Record the recv_requests phase inside the current loop (if any)."""
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
@@ -1176,18 +1746,31 @@ def note_pass_snapshot(t: "PassTimings", total_s: float) -> None:
|
||||
},
|
||||
)
|
||||
lt.seq = int(t.seq)
|
||||
lt.pass_end_wall_ns = end_wall_ns
|
||||
# Estimated DP-sync arrival: the pass ended, then mlp_sync waited.
|
||||
# See design §1.2: arrival_est = pass_end - mlp_sync.
|
||||
lt.arrival_est_wall_ns = end_wall_ns - int(t.mlp_sync * 1e9)
|
||||
|
||||
|
||||
def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is not None:
|
||||
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
|
||||
attrs = dict(attrs or {})
|
||||
attrs.setdefault(
|
||||
"sglang.run_batch_ms", max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6
|
||||
)
|
||||
lt.run_batch = (int(start_wall_ns), int(end_wall_ns), attrs)
|
||||
|
||||
|
||||
def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None):
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
if lt is not None:
|
||||
lt.process = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {}))
|
||||
attrs = dict(attrs or {})
|
||||
attrs.setdefault(
|
||||
"sglang.process_result_ms",
|
||||
max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6,
|
||||
)
|
||||
lt.process = (int(start_wall_ns), int(end_wall_ns), attrs)
|
||||
|
||||
|
||||
def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None:
|
||||
@@ -1351,6 +1934,143 @@ def _batch_attrs(batch) -> dict:
|
||||
}
|
||||
|
||||
|
||||
_LAST_LOOP_SEQ = 0
|
||||
_LAST_LOOP_HAS_DETAIL = False
|
||||
|
||||
|
||||
def _total_ms(ms: dict, base: str) -> float:
|
||||
return sum(
|
||||
value for name, value in ms.items() if name == base or name.startswith(base + ".")
|
||||
)
|
||||
|
||||
|
||||
def _update_prev_state(lt: LoopTimings, loop_seq: int):
|
||||
"""Capture the previous loop summary and store this loop for the next one.
|
||||
|
||||
Returns ``(prev_seq, prev_has_detail, prev_summary, consecutive)``. Only a
|
||||
loop that actually recorded detail becomes a valid predecessor, so a
|
||||
sampled loop after an unsampled one is marked non-consecutive instead of
|
||||
silently publishing a stale window (design §4.5).
|
||||
"""
|
||||
global _LAST_LOOP_SEQ, _LAST_LOOP_HAS_DETAIL
|
||||
prev_seq = int(_LAST_LOOP_SEQ)
|
||||
prev_has = bool(_LAST_LOOP_HAS_DETAIL)
|
||||
prev_summary = dict(_LAST_LOOP_SUMMARY)
|
||||
consecutive = prev_has and prev_seq == loop_seq - 1
|
||||
|
||||
ms = {
|
||||
name: rec[0] / 1e6
|
||||
for name, rec in ((lt.detail or {}).get("phases") or {}).items()
|
||||
if rec
|
||||
}
|
||||
_LAST_LOOP_SUMMARY.clear()
|
||||
if lt.detail is not None:
|
||||
_LAST_LOOP_SUMMARY.update(
|
||||
{
|
||||
"run_batch_ms": float(
|
||||
lt.run_batch[2].get("sglang.run_batch_ms", 0.0)
|
||||
)
|
||||
if lt.run_batch is not None
|
||||
else 0.0,
|
||||
"process_result_ms": float(
|
||||
lt.process[2].get("sglang.process_result_ms", 0.0)
|
||||
)
|
||||
if lt.process is not None
|
||||
else 0.0,
|
||||
"copy_done_wait_ms": _total_ms(ms, "copy_done_wait"),
|
||||
"kv_state_payload_ms": _total_ms(ms, "kv_state_payload"),
|
||||
"kv_page_indices_ms": _total_ms(ms, "kv_page_indices"),
|
||||
"kv_sender_send_ms": _total_ms(ms, "kv_sender_send"),
|
||||
}
|
||||
)
|
||||
_LAST_LOOP_SEQ = int(loop_seq)
|
||||
_LAST_LOOP_HAS_DETAIL = lt.detail is not None
|
||||
return prev_seq, prev_has, prev_summary, consecutive
|
||||
|
||||
|
||||
def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None:
|
||||
"""Expose detail phases on the loop span and its child spans.
|
||||
|
||||
Single-occurrence phases become children (``scheduler.loop.<name>``);
|
||||
repeated phases only contribute summed attributes, so span count never
|
||||
grows with n_reqs / n_segments.
|
||||
"""
|
||||
detail = lt.detail or {}
|
||||
phases = detail.get("phases") or {}
|
||||
ms = {}
|
||||
for name, rec in phases.items():
|
||||
try:
|
||||
total_ns, count, start_wall, end_wall = rec
|
||||
except Exception:
|
||||
continue
|
||||
key = "sglang." + name.replace(".", "_") + "_ms"
|
||||
ms[name] = total_ns / 1e6
|
||||
attrs[key] = ms[name]
|
||||
if count > 1:
|
||||
attrs["sglang." + name.replace(".", "_") + "_count"] = int(count)
|
||||
continue
|
||||
if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES:
|
||||
continue
|
||||
child(
|
||||
"scheduler.loop." + name.replace(".", "_"),
|
||||
start_wall,
|
||||
end_wall,
|
||||
{
|
||||
"sglang.dp_rank": int(dp_rank),
|
||||
"sglang.phase_ms": total_ns / 1e6,
|
||||
},
|
||||
)
|
||||
|
||||
for name, value in (detail.get("counts") or {}).items():
|
||||
attrs["sglang." + name.replace(".", "_") + "_count"] = int(value)
|
||||
for name, value in (detail.get("attrs") or {}).items():
|
||||
attrs["sglang." + name.replace(".", "_")] = value
|
||||
|
||||
# Accounting residuals (design §3.2): never negative, never folded in.
|
||||
def _total(*bases):
|
||||
total = 0.0
|
||||
for base in bases:
|
||||
for name, value in ms.items():
|
||||
if name == base or name.startswith(base + "."):
|
||||
total += value
|
||||
return total
|
||||
|
||||
# Roll per-kind variants (kv_page_indices.last, ...) up to one base
|
||||
# attribute so analysis can use the total without knowing the kinds.
|
||||
bases = {name.split(".", 1)[0] for name in ms}
|
||||
for base in bases:
|
||||
if base not in ms:
|
||||
attrs["sglang." + base + "_ms"] = _total(base)
|
||||
|
||||
if lt.run_batch is not None:
|
||||
run_ms = float(lt.run_batch[2].get("sglang.run_batch_ms", 0.0))
|
||||
parts = _total(
|
||||
"early_send_host",
|
||||
"resolve_inputs",
|
||||
"model_forward_host",
|
||||
"copy_result_submit",
|
||||
)
|
||||
attrs["sglang.run_residual_ms"] = max(0.0, run_ms - parts)
|
||||
if lt.process is not None:
|
||||
proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0))
|
||||
parts = _total(
|
||||
"copy_done_wait",
|
||||
"auxiliary_and_logprob",
|
||||
"per_request_result",
|
||||
"report_stats",
|
||||
)
|
||||
if "sglang.process_kv_ms" in lt.process[2]:
|
||||
# KV phases charged to this call only (sends issued later in the
|
||||
# loop, e.g. pending-bootstrap, are not part of process_result).
|
||||
parts += float(lt.process[2]["sglang.process_kv_ms"])
|
||||
else:
|
||||
parts += _total(
|
||||
"kv_state_payload", "kv_page_indices", "kv_sender_send"
|
||||
)
|
||||
attrs["sglang.process_residual_ms"] = max(0.0, proc_ms - parts)
|
||||
|
||||
|
||||
|
||||
def end_loop(
|
||||
lt: Optional[LoopTimings],
|
||||
*,
|
||||
@@ -1419,6 +2139,14 @@ def end_loop(
|
||||
pass
|
||||
lt.no_batch_reason = reason
|
||||
|
||||
# Update the prev-loop chain on every loop (sampled or not) so a
|
||||
# sampled loop after an unsampled one is reported as non-consecutive
|
||||
# instead of silently carrying a stale window (review finding).
|
||||
loop_seq_cur = int(lt.seq or lt.predicted_seq)
|
||||
prev_state = None
|
||||
if _DETAIL_ENABLED:
|
||||
prev_state = _update_prev_state(lt, loop_seq_cur)
|
||||
|
||||
if lt.span is None and (has_batch or pending):
|
||||
# Covers work discovered after begin_loop, including first input and
|
||||
# deferred-only loops. DP outcomes activate earlier for pass parenting.
|
||||
@@ -1445,7 +2173,6 @@ def end_loop(
|
||||
"sglang.forward_overlap": overlap,
|
||||
}
|
||||
attrs.update(_batch_attrs(batch))
|
||||
lt.span.set_attributes(attrs)
|
||||
|
||||
def _child(name, start_ns, end_ns, child_attrs):
|
||||
try:
|
||||
@@ -1458,6 +2185,26 @@ def end_loop(
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if lt.detail is not None:
|
||||
_emit_loop_detail(lt, attrs, _child, int(dp_rank))
|
||||
if _DETAIL_ENABLED:
|
||||
if lt.pass_end_wall_ns:
|
||||
# DP-skew join fields (design §4.5); independent of phases.
|
||||
attrs["sglang.pass_end_wall_ns"] = int(lt.pass_end_wall_ns)
|
||||
attrs["sglang.arrival_est_wall_ns"] = int(lt.arrival_est_wall_ns)
|
||||
if prev_state is not None:
|
||||
prev_seq, prev_has, prev_summary, consecutive = prev_state
|
||||
attrs["sglang.prev_loop_seq"] = int(prev_seq)
|
||||
if consecutive:
|
||||
for key in _DETAIL_PREV_KEYS:
|
||||
if key in prev_summary:
|
||||
attrs["sglang.prev_" + key] = prev_summary[key]
|
||||
elif prev_seq:
|
||||
# The previous loop was unsampled or had no detail;
|
||||
# its phases must not be treated as seq-1 values.
|
||||
attrs["sglang.prev_detail_missing"] = True
|
||||
lt.span.set_attributes(attrs)
|
||||
|
||||
if lt.recv is not None and not lt.recv_span_child:
|
||||
start, end, parts = lt.recv
|
||||
child = {"sglang.dp_rank": int(dp_rank)}
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
"""Unit tests for SGLANG_TRACE_LOOP_DETAIL / SGLANG_TRACE_MOONCAKE_DETAIL.
|
||||
|
||||
Covers the design's phase-1/phase-2 acceptance points
|
||||
(handoff/prefill-cpu-kv-overlap-design-20261106.md §8.1):
|
||||
|
||||
* detail is off by default and records nothing;
|
||||
* detail only records for sampled loops (same decision as the loop span);
|
||||
* single phases become child spans, repeated phases only attributes;
|
||||
* residuals are clamped at zero and never folded into a phase;
|
||||
* per-kind KV phases roll up to one base attribute;
|
||||
* the previous-loop summary is published for DP-skew correlation;
|
||||
* chunk spans break worker busy time into queue/source/indices/transfer.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def _module_path():
|
||||
p = (
|
||||
pathlib.Path(__file__).resolve().parents[4]
|
||||
/ "python/sglang/srt/observability/batch_telemetry.py"
|
||||
)
|
||||
if p.exists():
|
||||
return p
|
||||
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
|
||||
return pathlib.Path(spec.origin)
|
||||
|
||||
|
||||
def _load(**env):
|
||||
base = {
|
||||
"SGLANG_BATCH_STAGE_TIMING": "1",
|
||||
"SGLANG_TRACE_LOOP_SPAN": "1",
|
||||
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
|
||||
"SGLANG_TRACE_LOOP_DETAIL": "1",
|
||||
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
|
||||
"SGLANG_TRACE_FORWARD_SPAN": "0",
|
||||
}
|
||||
base.update(env)
|
||||
with mock.patch.dict(os.environ, base):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"batch_telemetry_detail_under_test", _module_path()
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
class _FakeCounter:
|
||||
def __init__(self):
|
||||
self.data = {}
|
||||
|
||||
def labels(self, **kw):
|
||||
self._key = tuple(sorted(kw.items()))
|
||||
return self
|
||||
|
||||
def inc(self, value=1.0):
|
||||
self.data[self._key] = self.data.get(self._key, 0.0) + value
|
||||
|
||||
|
||||
def _fake_loop_metrics():
|
||||
return {
|
||||
"no_batch_seconds": _FakeCounter(),
|
||||
"no_batch_total": _FakeCounter(),
|
||||
"lockstep_seconds": _FakeCounter(),
|
||||
}
|
||||
|
||||
|
||||
class _FakeSpan:
|
||||
def __init__(self, name, **kwargs):
|
||||
self.name = name
|
||||
self.kwargs = kwargs
|
||||
self.attributes = dict(kwargs.get("attributes") or {})
|
||||
self.end_time = None
|
||||
|
||||
def set_attributes(self, attrs):
|
||||
self.attributes.update(attrs)
|
||||
|
||||
def get_span_context(self):
|
||||
return types.SimpleNamespace(
|
||||
is_valid=True, trace_id=1, span_id=2
|
||||
)
|
||||
|
||||
def end(self, end_time=None):
|
||||
self.end_time = end_time
|
||||
|
||||
|
||||
class _FakeTracer:
|
||||
def __init__(self):
|
||||
self.spans = []
|
||||
|
||||
def start_span(self, name, **kwargs):
|
||||
span = _FakeSpan(name, **kwargs)
|
||||
self.spans.append(span)
|
||||
return span
|
||||
|
||||
|
||||
def _fake_otel():
|
||||
tracer = _FakeTracer()
|
||||
trace = types.ModuleType("opentelemetry.trace")
|
||||
trace.get_tracer = lambda _name: tracer
|
||||
trace.set_span_in_context = lambda span: ("ctx", span)
|
||||
trace.Link = lambda ctx: ("link", ctx)
|
||||
package = types.ModuleType("opentelemetry")
|
||||
package.trace = trace
|
||||
patcher = mock.patch.dict(
|
||||
sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace}
|
||||
)
|
||||
return tracer, patcher
|
||||
|
||||
|
||||
def _state(**overrides):
|
||||
state = {
|
||||
"waiting": 0,
|
||||
"result_queue": 0,
|
||||
"result_limit": 2,
|
||||
"inflight": 0,
|
||||
"bootstrap": 0,
|
||||
"deferred": 0,
|
||||
"paused": False,
|
||||
"waiting_for_result": False,
|
||||
"skip_batch_creation": False,
|
||||
"forward_active": False,
|
||||
}
|
||||
state.update(overrides)
|
||||
return state
|
||||
|
||||
|
||||
def _span(tracer, name):
|
||||
return next(s for s in tracer.spans if s.name == name)
|
||||
|
||||
|
||||
def _last_loop_attrs(tracer):
|
||||
spans = [s for s in tracer.spans if s.name == "scheduler.loop"]
|
||||
return spans[-1].attributes
|
||||
|
||||
|
||||
def _pass_with_seq(mod, seq):
|
||||
pass_t = mod.PassTimings("prefill")
|
||||
pass_t.seq = seq
|
||||
return pass_t
|
||||
|
||||
|
||||
class LoopDetailTest(unittest.TestCase):
|
||||
def test_disabled_records_nothing(self):
|
||||
mod = _load(SGLANG_TRACE_LOOP_DETAIL="0")
|
||||
self.assertFalse(mod.loop_detail_enabled())
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
self.assertIsNone(mod.detail_begin("model_forward_host"))
|
||||
mod.detail_add("kv_page_indices.last", 0, 10_000)
|
||||
mod.detail_count("n_segments")
|
||||
mod.detail_attr("state_types", "SWA")
|
||||
mod.mark_run_batch(0, 50_000_000, None)
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _span(tracer, "scheduler.loop").attributes
|
||||
self.assertNotIn("sglang.kv_page_indices_ms", attrs)
|
||||
self.assertNotIn("sglang.state_types", attrs)
|
||||
self.assertIsNone(lt.detail)
|
||||
|
||||
def test_unsampled_loop_records_nothing(self):
|
||||
mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0")
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
self.assertFalse(mod.loop_detail_enabled())
|
||||
self.assertIsNone(mod.detail_begin("model_forward_host"))
|
||||
mod.detail_add("kv_page_indices.last", 0, 1_000)
|
||||
self.assertIsNone(lt.detail)
|
||||
|
||||
def test_phases_attributes_and_residuals(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
self.assertTrue(mod.loop_detail_enabled())
|
||||
|
||||
# Single phase -> child span.
|
||||
tok = mod.detail_begin("report_stats")
|
||||
mod.detail_end("report_stats", tok)
|
||||
with mod.detail_phase("copy_done_wait"):
|
||||
pass
|
||||
|
||||
# Repeated phase -> attributes only, split by send_kind.
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("kv_page_indices.cached_prefix", t0, t0 + 2_000_000)
|
||||
mod.detail_add("kv_page_indices.last", t0, t0 + 5_000_000)
|
||||
mod.detail_add("kv_sender_send.last", t0, t0 + 3_000_000)
|
||||
mod.detail_count("n_segments", 3)
|
||||
mod.detail_attr("state_types", "REQUEST_ROW,SWA")
|
||||
|
||||
# run_batch = 100 ms with 20 ms of measured phases.
|
||||
start = time.time_ns()
|
||||
mod.detail_add("early_send_host", t0, t0 + 10_000_000)
|
||||
mod.detail_add("resolve_inputs", t0, t0 + 5_000_000)
|
||||
mod.detail_add("model_forward_host", t0, t0 + 50_000_000)
|
||||
mod.detail_add("copy_result_submit", t0, t0 + 5_000_000)
|
||||
mod.mark_run_batch(start, start + 100_000_000, None)
|
||||
|
||||
# process_result = 100 ms with 62 ms measured -> residual ~38 ms.
|
||||
# copy_done_wait is a real measured span (sub-ms) on top of the
|
||||
# synthetic 20 ms used for the run_batch accounting above.
|
||||
mod.detail_add("copy_done_wait", t0, t0 + 20_000_000)
|
||||
mod.detail_add("per_request_result", t0, t0 + 40_000_000)
|
||||
mod.mark_process(start, start + 100_000_000, None)
|
||||
|
||||
mod.end_loop(lt, dp_rank=2, has_batch=True, state=_state())
|
||||
|
||||
loop = _span(tracer, "scheduler.loop")
|
||||
attrs = loop.attributes
|
||||
self.assertAlmostEqual(attrs["sglang.kv_page_indices_ms"], 7.0)
|
||||
self.assertAlmostEqual(attrs["sglang.kv_page_indices_cached_prefix_ms"], 2.0)
|
||||
self.assertAlmostEqual(attrs["sglang.kv_page_indices_last_ms"], 5.0)
|
||||
self.assertEqual(attrs["sglang.n_segments_count"], 3)
|
||||
self.assertEqual(attrs["sglang.state_types"], "REQUEST_ROW,SWA")
|
||||
# model_forward_host appears once -> child span + attribute.
|
||||
self.assertAlmostEqual(attrs["sglang.model_forward_host_ms"], 50.0)
|
||||
self.assertAlmostEqual(attrs["sglang.run_residual_ms"], 30.0, places=3)
|
||||
# copy_done_wait 20 + per_request 40 + kv_page_indices 7 + sender 3.
|
||||
self.assertAlmostEqual(attrs["sglang.process_residual_ms"], 30.0, delta=2.0)
|
||||
names = [s.name for s in tracer.spans]
|
||||
self.assertIn("scheduler.loop.model_forward_host", names)
|
||||
self.assertIn("scheduler.loop.report_stats", names)
|
||||
# Per-request / per-segment phases never create spans.
|
||||
self.assertNotIn("scheduler.loop.kv_page_indices_last", names)
|
||||
self.assertNotIn("scheduler.loop.per_request_result", names)
|
||||
|
||||
def test_residual_is_clamped_at_zero(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("per_request_result", t0, t0 + 300_000_000)
|
||||
mod.mark_process(time.time_ns(), time.time_ns() + 100_000_000, None)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _span(tracer, "scheduler.loop").attributes
|
||||
self.assertEqual(attrs["sglang.process_residual_ms"], 0.0)
|
||||
|
||||
def test_process_kv_scope_excludes_later_sends(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
# KV send issued inside process_result (charged) and a second one
|
||||
# issued later in the same loop by the inflight queue (not charged).
|
||||
mod.detail_add("kv_sender_send.last", t0, t0 + 10_000_000)
|
||||
start = time.time_ns()
|
||||
mod.mark_process(
|
||||
start, start + 100_000_000, {"sglang.process_kv_ms": 10.0}
|
||||
)
|
||||
mod.detail_add("kv_sender_send.last", t0, t0 + 500_000_000)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _span(tracer, "scheduler.loop").attributes
|
||||
self.assertAlmostEqual(attrs["sglang.process_residual_ms"], 90.0)
|
||||
|
||||
def test_previous_loop_summary_is_published(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("copy_done_wait", t0, t0 + 7_000_000)
|
||||
mod.detail_add("kv_sender_send.last", t0, t0 + 2_000_000)
|
||||
start = time.time_ns()
|
||||
mod.mark_run_batch(start, start + 30_000_000, None)
|
||||
mod.mark_process(start, start + 50_000_000, None)
|
||||
# Consecutive loop_seq values are required for prev_* to be
|
||||
# published (a sampled loop after an unsampled one is marked).
|
||||
mod.note_pass_snapshot(_pass_with_seq(mod, 5), 0.01)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
|
||||
lt2 = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
mod.note_pass_snapshot(_pass_with_seq(mod, 6), 0.01)
|
||||
mod.end_loop(lt2, dp_rank=0, has_batch=True, state=_state())
|
||||
|
||||
attrs = _last_loop_attrs(tracer)
|
||||
self.assertEqual(attrs["sglang.prev_loop_seq"], 5)
|
||||
self.assertAlmostEqual(attrs["sglang.prev_run_batch_ms"], 30.0)
|
||||
self.assertAlmostEqual(attrs["sglang.prev_process_result_ms"], 50.0)
|
||||
self.assertAlmostEqual(attrs["sglang.prev_copy_done_wait_ms"], 7.0)
|
||||
self.assertAlmostEqual(attrs["sglang.prev_kv_sender_send_ms"], 2.0)
|
||||
|
||||
def test_phase_total_supports_per_kind_names(self):
|
||||
"""Regression: recorded names are kv_*.last/middle/cached_prefix."""
|
||||
mod = _load()
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("kv_page_indices.cached_prefix", t0, t0 + 2_000_000)
|
||||
mod.detail_add("kv_page_indices.middle", t0, t0 + 3_000_000)
|
||||
mod.detail_add("kv_sender_send.last", t0, t0 + 10_000_000)
|
||||
self.assertEqual(
|
||||
mod.detail_phase_total_ns(("kv_page_indices",)), 5_000_000
|
||||
)
|
||||
self.assertEqual(mod.detail_phase_total_ns(("kv_sender_send",)), 10_000_000)
|
||||
self.assertEqual(
|
||||
mod.detail_phase_total_ns(
|
||||
("kv_state_payload", "kv_page_indices", "kv_sender_send")
|
||||
),
|
||||
15_000_000,
|
||||
)
|
||||
# A base name must not swallow an unrelated longer base.
|
||||
self.assertEqual(mod.detail_phase_total_ns(("kv_page",)), 0)
|
||||
mod._LOOP_TLS.current = None
|
||||
|
||||
def test_prev_summary_not_published_after_unsampled_loop(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
mod.detail_add("copy_done_wait", time.monotonic_ns(), time.monotonic_ns())
|
||||
mod.note_pass_snapshot(_pass_with_seq(mod, 10), 0.01)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
|
||||
# Loop 11 has no detail (simulated unsampled): the chain must not
|
||||
# pretend loop 12's predecessor is loop 10.
|
||||
lt2 = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
lt2.detail = None
|
||||
mod.note_pass_snapshot(_pass_with_seq(mod, 11), 0.01)
|
||||
mod.end_loop(lt2, dp_rank=0, has_batch=True, state=_state())
|
||||
|
||||
lt3 = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
mod.detail_add("copy_done_wait", time.monotonic_ns(), time.monotonic_ns())
|
||||
mod.note_pass_snapshot(_pass_with_seq(mod, 12), 0.01)
|
||||
mod.end_loop(lt3, dp_rank=0, has_batch=True, state=_state())
|
||||
attrs = _last_loop_attrs(tracer)
|
||||
self.assertEqual(attrs["sglang.prev_loop_seq"], 11)
|
||||
self.assertTrue(attrs["sglang.prev_detail_missing"])
|
||||
self.assertNotIn("sglang.prev_copy_done_wait_ms", attrs)
|
||||
|
||||
def test_arrival_estimate_uses_pass_end_minus_mlp_sync(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
pass_t = mod.PassTimings("prefill")
|
||||
pass_t.seq = lt.predicted_seq
|
||||
pass_t.mlp_sync = 0.05
|
||||
mod.note_pass_snapshot(pass_t, 0.1)
|
||||
mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state())
|
||||
attrs = _span(tracer, "scheduler.loop").attributes
|
||||
self.assertEqual(
|
||||
attrs["sglang.pass_end_wall_ns"] - attrs["sglang.arrival_est_wall_ns"],
|
||||
50_000_000,
|
||||
)
|
||||
|
||||
def test_current_loop_seq_available_to_senders(self):
|
||||
mod = _load()
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
pass_t = mod.PassTimings("prefill")
|
||||
pass_t.seq = 41
|
||||
mod.note_pass_snapshot(pass_t, 0.01)
|
||||
self.assertEqual(mod.current_loop_seq(), 41)
|
||||
mod._LOOP_TLS.current = None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user