diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index ab6be253a2..bec0777d64 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -952,7 +952,10 @@ class SchedulerDisaggregationPrefillMixin: ) except Exception: pass - self._apply_war_barrier() + # WAR barrier is outside run_batch's timing boundary, so it is + # recorded as its own loop phase (design §4.3). + with batch_telemetry.detail_phase("war_barrier"): + self._apply_war_barrier() self.result_queue.append((batch.copy(), batch_result)) else: batch_result = None @@ -981,13 +984,32 @@ class SchedulerDisaggregationPrefillMixin: _pr_t0 = ( time.time_ns() if _lt is not None and _lt.span is not None else 0 ) + # Scope KV phases to this call: sends issued later in the loop + # (inflight-queue pending bootstrap) must not be charged here. + _proc_kv_phases = ( + "kv_state_payload", + "kv_page_indices", + "kv_sender_send", + ) + _proc_detail = batch_telemetry.loop_detail_enabled() + _proc_kv_before = ( + batch_telemetry.detail_phase_total_ns(_proc_kv_phases) + if _proc_detail + else 0 + ) self.process_batch_result(oldest_batch, oldest_batch_result) + _pr_attrs = {"sglang.n_reqs": len(oldest_batch.reqs)} + if _proc_detail: + _pr_attrs["sglang.process_kv_ms"] = ( + batch_telemetry.detail_phase_total_ns(_proc_kv_phases) + - _proc_kv_before + ) / 1e6 if _pr_t0: try: batch_telemetry.mark_process( _pr_t0, time.time_ns(), - {"sglang.n_reqs": len(oldest_batch.reqs)}, + _pr_attrs, ) except Exception: pass @@ -1057,7 +1079,11 @@ class SchedulerDisaggregationPrefillMixin: ) if copy_done is not None: - copy_done.synchronize() + with batch_telemetry.detail_phase("copy_done_wait"): + copy_done.synchronize() + _aux_t0 = ( + time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None + ) auxiliary_output_starts = ( self.batch_result_processor.snapshot_auxiliary_output_starts(batch, result) ) @@ -1103,10 +1129,29 @@ class SchedulerDisaggregationPrefillMixin: if extend_logprob_start_len < extend_input_len: logprob_pt += extend_input_len - extend_logprob_start_len + # Auxiliary finalize / logits-to-CPU work is a contiguous host phase. + if _aux_t0 is not None: + batch_telemetry.detail_add( + "auxiliary_and_logprob", _aux_t0, time.monotonic_ns() + ) + # Per-request result handling is everything in this loop except the KV + # phases; the delta below keeps `continue` paths accounted for without + # measuring each branch. + _pr_loop_t0 = ( + time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None + ) + _kv_phases = ("kv_state_payload", "kv_page_indices", "kv_sender_send") + _kv_before = ( + batch_telemetry.detail_phase_total_ns(_kv_phases) + if _pr_loop_t0 is not None + else 0 + ) + for i, (req, next_token_id) in enumerate( zip(batch.reqs, next_token_ids, strict=True) ): if req.inflight_middle_chunks <= 0: + batch_telemetry.detail_count("n_last_reqs") req.time_stats.set_prefill_finished_time() # Its KV never landed: abort instead of sending it to decode. @@ -1210,6 +1255,7 @@ class SchedulerDisaggregationPrefillMixin: else: # being chunked reqs' prefill is not finished + batch_telemetry.detail_count("n_middle_reqs") req.inflight_middle_chunks -= 1 # Still chunking iff its next chunk was launched: either it is @@ -1260,12 +1306,27 @@ class SchedulerDisaggregationPrefillMixin: self.send_kv_chunk(req, last_chunk=False, end_idx=req.tmp_end_idx) req.time_stats.set_last_chunked_prefill_finish_time() + if _pr_loop_t0 is not None: + _kv_ns = batch_telemetry.detail_phase_total_ns(_kv_phases) - _kv_before + batch_telemetry.detail_add( + "per_request_result", + _pr_loop_t0 + max(0, _kv_ns), + time.monotonic_ns(), + ) + if auxiliary_output is not None: + _aux2_t0 = ( + time.monotonic_ns() if batch_telemetry.loop_detail_enabled() else None + ) self.batch_result_processor.consume_auxiliary_output( batch, auxiliary_output, auxiliary_output_starts, ) + if _aux2_t0 is not None: + batch_telemetry.detail_add( + "auxiliary_and_logprob", _aux2_t0, time.monotonic_ns() + ) if aborted_reqs: self.output_streamer.stream_output( @@ -1274,12 +1335,13 @@ class SchedulerDisaggregationPrefillMixin: ) can_run_cuda_graph = result.can_run_cuda_graph - self.metrics_reporter.report_prefill_stats( - batch=batch, - prefill_stats=batch.prefill_stats, - can_run_cuda_graph=can_run_cuda_graph, - dp_cooperation_info=batch.dp_cooperation_info, - ) + with batch_telemetry.detail_phase("report_stats"): + self.metrics_reporter.report_prefill_stats( + batch=batch, + prefill_stats=batch.prefill_stats, + can_run_cuda_graph=can_run_cuda_graph, + dp_cooperation_info=batch.dp_cooperation_info, + ) self.maybe_send_health_check_signal() @scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE) @@ -1627,14 +1689,21 @@ class SchedulerDisaggregationPrefillMixin: ev = torch.cuda.Event() ev.record(self.forward_stream) req.disagg_kv_sender._early_send_wait_event = ev - self.send_kv_chunk(req, last_chunk=False, end_idx=cached_end) + if batch_telemetry.loop_detail_enabled(): + batch_telemetry.detail_count("early_send_reqs") + self.send_kv_chunk( + req, last_chunk=False, end_idx=cached_end, send_kind="cached_prefix" + ) def send_kv_chunk( self, req: Req, last_chunk: bool = False, end_idx: Optional[int] = None, + send_kind: Optional[str] = None, ) -> None: + if send_kind is None: + send_kind = "last" if last_chunk else "middle" computer: Optional[KvChecksumComputer] = self.kv_checksum_computer if last_chunk and computer is not None: if is_health_check_req(req): @@ -1647,7 +1716,11 @@ class SchedulerDisaggregationPrefillMixin: value = computer.compute(page_indices_gpu, state_indices) self.disagg_metadata_buffers.set_kv_checksum(req, value) _kv_t0 = time.time_ns() if batch_telemetry.kv_send_span_enabled() else 0 - self._send_kv_chunk(req, last_chunk=last_chunk, end_idx=end_idx) + if batch_telemetry.loop_detail_enabled(): + batch_telemetry.detail_count("n_kv_calls") + self._send_kv_chunk( + req, last_chunk=last_chunk, end_idx=end_idx, send_kind=send_kind + ) if _kv_t0: batch_telemetry.add_kv_send(_kv_t0, time.time_ns()) @@ -1656,6 +1729,7 @@ class SchedulerDisaggregationPrefillMixin: req: Req, last_chunk: bool = False, end_idx: Optional[int] = None, + send_kind: str = "last", ) -> None: """ Send a prefilled chunk to the decode server @@ -1690,6 +1764,11 @@ class SchedulerDisaggregationPrefillMixin: return state_indices: Optional[List] = None + _state_tok = ( + batch_telemetry.detail_begin("kv_state_payload." + send_kind) + if last_chunk + else None + ) if last_chunk: self.disagg_metadata_buffers.set_buf(req) @@ -1828,6 +1907,12 @@ class SchedulerDisaggregationPrefillMixin: state_indices = [ payloads[st]() if st in payloads else None for st in state_types ] + if _state_tok is not None: + batch_telemetry.detail_attr( + "state_types", + ",".join(sorted(str(getattr(st, "name", st)) for st in state_types)), + ) + batch_telemetry.detail_end("kv_state_payload." + send_kind, _state_tok) transfer_chunk_tokens = req.disagg_kv_sender.get_max_transfer_tokens() if self.enable_staging: @@ -1848,7 +1933,16 @@ class SchedulerDisaggregationPrefillMixin: else: segments = [(start_idx, end_idx)] + # Classify the chunk for detail accounting (cached_prefix/middle/last). + # Set on the sender so backend send() signatures stay unchanged. + req.disagg_kv_sender._send_kind = send_kind + for seg_start, seg_end in segments: + _seg_t0 = ( + time.monotonic_ns() + if batch_telemetry.loop_detail_enabled() + else None + ) is_final_segment = seg_end == end_idx raw_kv_indices = self.req_to_token_pool.req_to_token[ req.kv.req_pool_idx, seg_start:seg_end @@ -1880,10 +1974,20 @@ class SchedulerDisaggregationPrefillMixin: indices_event.record(torch.cuda.current_stream()) else: page_indices = kv_to_page_indices(kv_indices, page_size) + if _seg_t0 is not None: + batch_telemetry.detail_add( + "kv_page_indices." + send_kind, _seg_t0, time.monotonic_ns() + ) + batch_telemetry.detail_count("n_pages", len(page_indices)) segment_is_last = last_chunk and is_final_segment if not sender.should_send_kv_chunk(len(page_indices), segment_is_last): continue send_state_indices = state_indices if segment_is_last else None + _send_t0 = ( + time.monotonic_ns() + if batch_telemetry.loop_detail_enabled() + else None + ) if indices_event is not None: sender.send( page_indices, @@ -1897,6 +2001,11 @@ class SchedulerDisaggregationPrefillMixin: send_state_indices, num_kv_tokens=seg_end - seg_start, ) + if _send_t0 is not None: + batch_telemetry.detail_add( + "kv_sender_send." + send_kind, _send_t0, time.monotonic_ns() + ) + batch_telemetry.detail_count("n_segments") req.start_send_idx = end_idx # A last chunk needs no entry: every `last_chunk=True` call site has # already put the request on `disagg_prefill_inflight_queue`. diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 42f16f2662..a317abc1ad 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4645,8 +4645,9 @@ class Scheduler( # PD prefill: early-send cached prefix KV, overlapping the suffix forward. if self.disaggregation_mode == DisaggregationMode.PREFILL: - for req in batch.reqs: - self.maybe_send_cached_prefix_chunk(req) + with batch_telemetry.detail_phase("early_send_host"): + for req in batch.reqs: + self.maybe_send_cached_prefix_chunk(req) # Run forward if self.is_generation: @@ -4673,7 +4674,8 @@ class Scheduler( # mix_running_indices). Run OUTSIDE isolation so the # snapshot captures the post-consume state — restoring # post-forward must not un-consume staging. - resolve_forward_inputs(batch, self.future_map) + with batch_telemetry.detail_phase("resolve_inputs"): + resolve_forward_inputs(batch, self.future_map) with self._forward_isolation(batch, overlap=True): future_indices = batch.req_pool_indices @@ -4696,9 +4698,10 @@ class Scheduler( ) # FIXME: pp is not compatible with overlap - batch_result = self.model_worker.forward_batch_generation( - batch, **fwd_kwargs - ) + with batch_telemetry.detail_phase("model_forward_host"): + batch_result = self.model_worker.forward_batch_generation( + batch, **fwd_kwargs + ) if batch.spec_algorithm.is_none(): self.future_map.publish(future_indices, batch.seq_lens + 1) # Park any refs the worker wants kept alive 2 iters @@ -4736,7 +4739,10 @@ class Scheduler( return_hidden_states=batch.return_hidden_states, ) else: - self._copy_result_to_cpu(batch_result, batch) + with batch_telemetry.detail_phase( + "copy_result_submit" + ): + self._copy_result_to_cpu(batch_result, batch) else: batch_result.future_indices = future_indices diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index aea740e682..9f3f148851 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -20,6 +20,13 @@ Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs: SGLANG_BATCH_SPAN_SAMPLE=0.05 head-sample ratio for spans (default 1.0) SGLANG_BATCH_SPAN_MAX_LINKS=32 cap request links per batch span + SGLANG_TRACE_LOOP_DETAIL=1 split run_batch/process_result into host + phases (requires SGLANG_BATCH_STAGE_TIMING=1 + and SGLANG_TRACE_LOOP_SPAN=1; only sampled + loops are recorded) + SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown + (queue/source/indices/transfer/join) + Cost model (measured on our 4xDP4 / 4-GPU instances): * histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU. * span + 32 links: ~20-60 us per batch -> <0.1% CPU at 17 batches/s. @@ -1033,6 +1040,9 @@ class LoopTimings: "launch_blocked_reason", "seq", "no_batch_reason", + "pass_end_wall_ns", + "arrival_est_wall_ns", + "detail", ) def __init__( @@ -1066,6 +1076,11 @@ class LoopTimings: self.launch_blocked_reason = "" self.seq = 0 self.no_batch_reason = "" + self.pass_end_wall_ns = 0 + self.arrival_est_wall_ns = 0 + # Only populated when SGLANG_TRACE_LOOP_DETAIL=1 and this loop is + # sampled; see the "Loop detail phases" section below. + self.detail = None def ensure_loop_span(lt: Optional[LoopTimings]) -> None: @@ -1137,6 +1152,561 @@ def current_loop_context(): return lt.span_ctx if lt is not None else None +# --------------------------------------------------------------------------- +# Loop detail phases (SGLANG_TRACE_LOOP_DETAIL=1). +# +# Splits run_batch / process_result into host phases. Gated twice: the module +# switch must be on AND the current loop must be sampled (same deterministic +# decision as the loop span), so detail data never covers an unsampled loop. +# +# Accounting rules (see handoff/prefill-cpu-kv-overlap-design-20261106.md §3.2): +# * single phases -> one child span (``scheduler.loop.``) +# * repeated phases -> summed attributes only (``sglang._ms`` + count) +# * durations use time.monotonic_ns(); OTLP start/end use time.time_ns() +# * residuals are clamped >= 0 and reported, never folded into a phase +# --------------------------------------------------------------------------- + +_DETAIL_ENABLED = _LOOP_SPAN_ENABLED and os.getenv( + "SGLANG_TRACE_LOOP_DETAIL", "0" +).strip().lower() not in ("", "0", "false", "no", "off") + +_DETAIL_TLS = threading.local() + +# Chunk-level (Mooncake transfer worker) detail tracing. Independent switch: +# the worker runs on other threads and has no scheduler loop TLS. +_MOONCAKE_DETAIL_ENABLED = os.getenv( + "SGLANG_TRACE_MOONCAKE_DETAIL", "0" +).strip().lower() not in ("", "0", "false", "no", "off") + +# Previous loop summary per scheduler thread, used for DP-skew correlation +# (previous loop's phases vs this loop's arrival skew). +_LAST_LOOP_SUMMARY: dict = {} + +# Phases that are inherently per-request / per-segment: always attributes, +# even when a loop happens to have exactly one occurrence, so span count never +# grows with n_reqs / n_segments (design §4.2). +_DETAIL_ATTRIBUTE_ONLY_PREFIXES = ( + "per_request_result", + "kv_state_payload", + "kv_page_indices", + "kv_sender_send", +) + +_DETAIL_PREV_KEYS = ( + "run_batch_ms", + "process_result_ms", + "copy_done_wait_ms", + "kv_state_payload_ms", + "kv_page_indices_ms", + "kv_sender_send_ms", +) + + +def loop_detail_enabled() -> bool: + """True when detail is on and the *current* loop is sampled.""" + if not _DETAIL_ENABLED: + return False + lt = getattr(_LOOP_TLS, "current", None) + return lt is not None and loop_sample_decision(lt.predicted_seq) + + +def mooncake_detail_enabled() -> bool: + return _MOONCAKE_DETAIL_ENABLED + + +def _detail_state(): + """Detail state of the current loop, only when that loop is sampled.""" + if not _DETAIL_ENABLED: + return None + lt = getattr(_LOOP_TLS, "current", None) + if lt is None or not loop_sample_decision(lt.predicted_seq): + return None + if lt.detail is None: + lt.detail = {"phases": {}, "counts": {}} + return lt.detail + + +def detail_begin(name: str) -> Optional[int]: + """Start a single-occurrence phase. Returns a monotonic-ns token.""" + if not _DETAIL_ENABLED: + return None + state = _detail_state() + if state is None: + return None + now_ns = time.monotonic_ns() + opens = getattr(_DETAIL_TLS, "open", None) + if opens is None: + opens = {} + _DETAIL_TLS.open = opens + opens[name] = (now_ns, time.time_ns()) + return now_ns + + +def detail_end(name: str, token: Optional[int] = None) -> None: + """Close a phase opened by ``detail_begin`` (or started at ``token``).""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + now_ns = time.monotonic_ns() + opens = getattr(_DETAIL_TLS, "open", None) or {} + entry = opens.pop(name, None) + start_wall_ns = entry[1] if entry is not None else None + if token is None: + if entry is None: + return + token = entry[0] + if start_wall_ns is None: + start_wall_ns = time.time_ns() - max(0, now_ns - int(token)) + _detail_record(state, name, int(token), now_ns, start_wall_ns, time.time_ns()) + + +def detail_add(name: str, start_ns: int, end_ns: int) -> None: + """Accumulate one occurrence of a repeated phase (no span per occurrence).""" + if not _DETAIL_ENABLED or start_ns is None or end_ns is None: + return + state = _detail_state() + if state is None: + return + start_ns = int(start_ns) + end_ns = int(end_ns) + if end_ns < start_ns: + end_ns = start_ns + now_wall_ns = time.time_ns() + _detail_record( + state, name, start_ns, end_ns, now_wall_ns - (end_ns - start_ns), now_wall_ns + ) + + +def _detail_record(state, name, start_ns, end_ns, start_wall_ns, end_wall_ns) -> None: + rec = state["phases"].get(name) + if rec is None: + state["phases"][name] = [ + end_ns - start_ns, + 1, + int(start_wall_ns), + int(end_wall_ns), + ] + return + rec[0] += end_ns - start_ns + rec[1] += 1 + rec[3] = int(end_wall_ns) + + +def detail_count(name: str, n: int = 1) -> None: + """Accumulate a low-cardinality counter for this loop.""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + state["counts"][name] = int(state["counts"].get(name, 0)) + int(n) + + +def detail_attr(name: str, value) -> None: + """Set a low-cardinality loop attribute (last write wins).""" + if not _DETAIL_ENABLED: + return + state = _detail_state() + if state is None: + return + state.setdefault("attrs", {})[name] = value + + +def detail_phase_total_ns(names) -> int: + """Sum of accumulated ns for the given phases in the current loop. + + ``names`` are *base* names: a phase recorded as ``kv_sender_send.last`` + (per send kind) is included when ``kv_sender_send`` is requested. Phase + names are only generated by this module, so prefix matching cannot pick up + an unrelated phase that merely shares a leading string. + """ + state = _detail_state() + if state is None: + return 0 + phases = state.get("phases") or {} + total = 0 + for name, rec in phases.items(): + if rec is None: + continue + for want in names: + if name == want or name.startswith(want + "."): + total += int(rec[0]) + break + return total + + +class detail_phase: + """Context manager for one single-occurrence phase (cheap when disabled).""" + + __slots__ = ("name", "token") + + def __init__(self, name: str): + self.name = name + self.token = None + + def __enter__(self): + self.token = detail_begin(self.name) + return self + + def __exit__(self, exc_type, exc, tb): + detail_end(self.name, self.token) + return False + + +def current_loop_seq() -> int: + """Loop sequence for cross-record correlation (0 when unavailable).""" + lt = getattr(_LOOP_TLS, "current", None) + if lt is None: + return 0 + return int(lt.seq or lt.predicted_seq or 0) + + +def current_loop_context_for_spans(): + """OTel context of the current loop span (None if unsampled).""" + lt = getattr(_LOOP_TLS, "current", None) + return lt.span_ctx if lt is not None else None + + +def _chunk_parent_context(chunk): + trace_ctx = getattr(chunk, "trace_ctx", None) + return getattr(trace_ctx, "root_span_context", None) + + +def split_queue_wait( + queue_wait_accum_ns: int, last_enqueue_mono_ns: int, dequeue_mono_ns: int +) -> tuple: + """Return ``(attempt_queue_wait_ns, total_queue_wait_ns)``. + + The attempt value matches the last_enqueue->dequeue wall window used by the + ``mooncake.queue_wait`` child span; the total value additionally includes + the queue waits of earlier deferred attempts (design §5.3 metric). + Keeping them separate prevents a span whose wall length disagrees with its + ``wait_ms`` attribute (review finding). + """ + attempt = max(0, int(dequeue_mono_ns) - int(last_enqueue_mono_ns)) + return attempt, max(0, int(queue_wait_accum_ns)) + attempt + + +class RoomDetailState: + """Thread-safe per-room detail bookkeeping (scheduler vs worker threads). + + The scheduler aborts/retracts a room while the transfer worker may still be + finishing an attempt. A reset therefore records a cancellation mark, and + the worker's join update re-checks under the same lock before writing, so a + late worker can never resurrect stats after abort (review finding). + """ + + # Bounded: a marker only needs to survive until the in-flight worker + # attempt finishes (milliseconds). The cap prevents unbounded growth if an + # aborted room never gets a worker teardown. + _MAX_CANCELLED = 8192 + + def __init__(self): + self._lock = threading.Lock() + self._join: dict = {} + self._seq: dict = {} + self._first: dict = {} + self._cancelled: dict = {} + + def _mark_cancelled(self, room: int) -> None: + self._cancelled[int(room)] = time.monotonic_ns() + while len(self._cancelled) > self._MAX_CANCELLED: + self._cancelled.pop(next(iter(self._cancelled))) + + def begin_chunk(self, room: int, now_mono_ns: int, now_wall_ns: int): + """Stamp a real enqueue. Returns ``(chunk_seq, first_enqueue_wall_ns)``. + + A new chunk clears a previous cancellation mark: it is the start of a + fresh flow (a retry may land on the same room id). + """ + with self._lock: + self._cancelled.pop(int(room), None) + seq = int(self._seq.get(int(room), 0)) + self._seq[int(room)] = seq + 1 + first = self._first.get(int(room)) + if first is None: + first = (int(now_mono_ns), int(now_wall_ns)) + self._first[int(room)] = first + return seq, int(first[1]) + + def record_join( + self, + room: int, + *, + status: str, + room_failed: bool, + now_wall_ns: int, + queue_wait_total_ns: int, + worker_busy_ns: int, + enqueue_wall_ns: int, + is_last_chunk: bool, + chunk_seq: int, + ) -> bool: + """Update the per-room join summary; False when the write was refused.""" + if status != "ok": + return False + with self._lock: + if room_failed or int(room) in self._cancelled: + return False + stats = self._join.get(int(room)) + if stats is None: + stats = {"chunk_count": 0} + self._join[int(room)] = stats + stats["chunk_count"] = int(stats.get("chunk_count", 0)) + 1 + first = self._first.get(int(room)) + if first is not None: + stats["first_enqueue_wall_ns"] = int(first[1]) + if is_last_chunk: + stats["last_chunk_enqueue_wall_ns"] = int(enqueue_wall_ns) + stats["last_chunk_complete_wall_ns"] = int(now_wall_ns) + stats["last_chunk_queue_wait_ms"] = queue_wait_total_ns / 1e6 + stats["last_chunk_worker_ms"] = worker_busy_ns / 1e6 + stats["last_chunk_chunk_seq"] = int(chunk_seq) + return True + + def reset(self, room: int) -> None: + with self._lock: + self._join.pop(int(room), None) + self._seq.pop(int(room), None) + self._first.pop(int(room), None) + self._mark_cancelled(room) + + def teardown(self, room: int, keep_join: bool) -> None: + with self._lock: + if not keep_join: + self._join.pop(int(room), None) + self._seq.pop(int(room), None) + self._first.pop(int(room), None) + self._cancelled.pop(int(room), None) + + def pop_join(self, room: int): + with self._lock: + return self._join.pop(int(room), None) + + +def note_chunk_requeue(chunk, now_mono_ns: int, now_wall_ns: int) -> None: + """Accumulate only the *queue* part of a finished attempt before re-enqueue. + + The previous implementation added ``requeue - last_enqueue``, which also + contains the worker's own execution and the staging watermark wait, thus + inflating queue wait on every defer (review finding). Only the interval + from the last enqueue to this attempt's dequeue is queue wait. + """ + if not _MOONCAKE_DETAIL_ENABLED or not getattr(chunk, "observe", False): + return + last_enqueue = int(getattr(chunk, "last_enqueue_mono_ns", 0) or 0) + dequeue = int(getattr(chunk, "dequeue_mono_ns", 0) or 0) + if last_enqueue and dequeue: + chunk.queue_wait_accum_ns = int( + getattr(chunk, "queue_wait_accum_ns", 0) or 0 + ) + max(0, dequeue - last_enqueue) + chunk.last_enqueue_mono_ns = int(now_mono_ns) + chunk.last_enqueue_wall_ns = int(now_wall_ns) + chunk.dequeue_mono_ns = 0 + chunk.requeue_count = int(getattr(chunk, "requeue_count", 0) or 0) + 1 + + +def emit_mooncake_chunk( + *, + chunk, + status: str, + queue_wait_ns: int, + queue_wait_total_ns: int, + source_wait_ns: int, + indices_wait_ns: int, + kv_transfer_ns: int, + state_transfer_ns: int, + aux_transfer_ns: int, + attempt_wall_ns: int, + dequeue_wall_ns: int, + end_wall_ns: int, + n_destinations: int, + source_wait_wall=None, + indices_wait_wall=None, + dp_rank: int = -1, + tp_rank: int = -1, +) -> None: + """One span per transfer-worker attempt, attributed by room/chunk_seq. + + ``worker_busy_ms`` follows the design definition: attempt wall time minus + the source/indices event waits (those are blocking waits, not worker work). + ``attempt_wall_ms`` is the raw dequeue->end time. + + Wait child spans use the real event timestamps captured by the worker. + A wait without a recorded window is attributes-only; no synthetic + ``end - duration`` placement is drawn (review finding). + """ + if not _MOONCAKE_DETAIL_ENABLED: + return + try: + from opentelemetry import trace as otel_trace + + tracer = otel_trace.get_tracer("sglang.mooncake.chunk") + # Attempt-scoped start: the last enqueue of this attempt, not the + # first enqueue of the chunk, so deferred retries do not overlap. + start_wall_ns = int( + getattr(chunk, "last_enqueue_wall_ns", 0) + or chunk.enqueue_wall_ns + or dequeue_wall_ns + ) or end_wall_ns + worker_busy_ns = max( + 0, int(attempt_wall_ns) - int(source_wait_ns) - int(indices_wait_ns) + ) + residual_ns = max( + 0, + worker_busy_ns + - int(kv_transfer_ns) + - int(state_transfer_ns) + - int(aux_transfer_ns), + ) + span = tracer.start_span( + "mooncake.kv_chunk", + context=_chunk_parent_context(chunk), + start_time=start_wall_ns, + attributes={ + "sglang.bootstrap_room": int(chunk.room), + "sglang.chunk_seq": int(chunk.chunk_seq), + "sglang.loop_seq": int(chunk.loop_seq), + "sglang.send_kind": str(chunk.send_kind or "unknown"), + "sglang.is_last_chunk": bool(chunk.is_last_chunk), + "sglang.queue_id": int(chunk.queue_id), + "sglang.requeue_count": int(chunk.requeue_count), + "sglang.num_kv_tokens": int(chunk.num_kv_tokens or 0), + "sglang.num_pages": int(chunk.num_pages), + "sglang.approx_bytes": int(chunk.approx_bytes), + "sglang.n_destinations": int(n_destinations), + "sglang.chunk_status": str(status), + "sglang.dp_rank": int(dp_rank), + "sglang.tp_rank": int(tp_rank), + # Attempt window (matches the child span) vs cumulative wait + # across deferred attempts (design §5.3 metric). + "sglang.queue_wait_ms": queue_wait_ns / 1e6, + "sglang.queue_wait_total_ms": queue_wait_total_ns / 1e6, + "sglang.source_wait_ms": source_wait_ns / 1e6, + "sglang.indices_wait_ms": indices_wait_ns / 1e6, + "sglang.kv_transfer_ms": kv_transfer_ns / 1e6, + "sglang.state_transfer_ms": state_transfer_ns / 1e6, + "sglang.aux_transfer_ms": aux_transfer_ns / 1e6, + "sglang.attempt_wall_ms": attempt_wall_ns / 1e6, + "sglang.worker_busy_ms": worker_busy_ns / 1e6, + "sglang.worker_residual_ms": residual_ns / 1e6, + }, + ) + span_ctx = otel_trace.set_span_in_context(span) + + def _child(name, start_ns, stop_ns, wait_ns): + start_ns = int(start_ns or 0) + stop_ns = int(stop_ns or 0) + if start_ns <= 0 or stop_ns <= start_ns: + return + try: + tracer.start_span( + name, + context=span_ctx, + start_time=start_ns, + attributes={"sglang.wait_ms": max(0, int(wait_ns)) / 1e6}, + ).end(end_time=stop_ns) + except Exception: + pass + + # Real event windows only; queue wait ends at dequeue. Zero-wait + # attempts (immediate dequeue) get no span. + if queue_wait_ns > 0: + _child( + "mooncake.queue_wait", + getattr(chunk, "last_enqueue_wall_ns", 0), + dequeue_wall_ns, + queue_wait_ns, + ) + if source_wait_wall is not None: + _child( + "mooncake.wait_source", + source_wait_wall[0], + source_wait_wall[1], + source_wait_ns, + ) + if indices_wait_wall is not None: + _child( + "mooncake.wait_indices", + indices_wait_wall[0], + indices_wait_wall[1], + indices_wait_ns, + ) + span.end(end_time=int(end_wall_ns)) + except Exception: + # Telemetry must never break the transfer worker. + pass + + +def emit_request_kv_join( + *, + req, + stats: dict, + dp_rank: int = -1, + completion_wall_ns: Optional[int] = None, +) -> None: + """Request-level join summary: last-chunk exposure + completion poll.""" + if not _MOONCAKE_DETAIL_ENABLED or not stats: + return + try: + from opentelemetry import trace as otel_trace + from opentelemetry.trace import Link + + now_wall_ns = int(completion_wall_ns or time.time_ns()) + enqueue_wall_ns = int(stats.get("last_chunk_enqueue_wall_ns", 0) or 0) + complete_wall_ns = int(stats.get("last_chunk_complete_wall_ns", 0) or 0) + final_exposed_ns = ( + max(0, complete_wall_ns - enqueue_wall_ns) + if enqueue_wall_ns and complete_wall_ns + else 0 + ) + completion_poll_ns = ( + max(0, now_wall_ns - complete_wall_ns) if complete_wall_ns else 0 + ) + # First enqueue comes from the manager's real queue.put() event + # (wall clock), not from pre-send preparation (review finding). + first_enqueue_wall_ns = int(stats.get("first_enqueue_wall_ns", 0) or 0) + first_enqueue_to_completion_ms = None + if first_enqueue_wall_ns: + first_enqueue_to_completion_ms = max( + 0.0, (now_wall_ns - first_enqueue_wall_ns) / 1e6 + ) + req_ctx = _request_span_context(req) + attrs = { + "sglang.bootstrap_room": int(getattr(req, "bootstrap_room", 0) or 0), + "sglang.rid": str(getattr(req, "rid", "")), + "sglang.dp_rank": int(dp_rank), + "sglang.chunk_count": int(stats.get("chunk_count", 0)), + "sglang.last_chunk_queue_wait_ms": float( + stats.get("last_chunk_queue_wait_ms", 0.0) + ), + "sglang.last_chunk_worker_ms": float( + stats.get("last_chunk_worker_ms", 0.0) + ), + "sglang.final_exposed_ms": final_exposed_ns / 1e6, + "sglang.completion_poll_ms": completion_poll_ns / 1e6, + # Per-rank request join. rank_join_skew_ms (max-min over TP/DP + # ranks) is computed offline by joining this span by room. + "sglang.request_join_ms": (final_exposed_ns + completion_poll_ns) / 1e6, + } + if first_enqueue_to_completion_ms is not None: + attrs["sglang.first_enqueue_to_completion_ms"] = ( + first_enqueue_to_completion_ms + ) + tracer = otel_trace.get_tracer("sglang.kv.request") + tracer.start_span( + "kv.request_join", + # Cover the whole request transfer when first enqueue is known. + start_time=first_enqueue_wall_ns or enqueue_wall_ns or now_wall_ns, + links=[Link(req_ctx)] if req_ctx is not None else [], + attributes=attrs, + ).end(end_time=now_wall_ns) + except Exception: + pass + + def note_recv_phase(*, start_wall_ns: int, end_wall_ns: int, parts: dict) -> None: """Record the recv_requests phase inside the current loop (if any).""" lt = getattr(_LOOP_TLS, "current", None) @@ -1176,18 +1746,31 @@ def note_pass_snapshot(t: "PassTimings", total_s: float) -> None: }, ) lt.seq = int(t.seq) + lt.pass_end_wall_ns = end_wall_ns + # Estimated DP-sync arrival: the pass ended, then mlp_sync waited. + # See design §1.2: arrival_est = pass_end - mlp_sync. + lt.arrival_est_wall_ns = end_wall_ns - int(t.mlp_sync * 1e9) def mark_run_batch(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): lt = getattr(_LOOP_TLS, "current", None) if lt is not None: - lt.run_batch = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {})) + attrs = dict(attrs or {}) + attrs.setdefault( + "sglang.run_batch_ms", max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6 + ) + lt.run_batch = (int(start_wall_ns), int(end_wall_ns), attrs) def mark_process(start_wall_ns: int, end_wall_ns: int, attrs: Optional[dict] = None): lt = getattr(_LOOP_TLS, "current", None) if lt is not None: - lt.process = (int(start_wall_ns), int(end_wall_ns), dict(attrs or {})) + attrs = dict(attrs or {}) + attrs.setdefault( + "sglang.process_result_ms", + max(0, int(end_wall_ns) - int(start_wall_ns)) / 1e6, + ) + lt.process = (int(start_wall_ns), int(end_wall_ns), attrs) def add_kv_send(start_wall_ns: int, end_wall_ns: int) -> None: @@ -1351,6 +1934,143 @@ def _batch_attrs(batch) -> dict: } +_LAST_LOOP_SEQ = 0 +_LAST_LOOP_HAS_DETAIL = False + + +def _total_ms(ms: dict, base: str) -> float: + return sum( + value for name, value in ms.items() if name == base or name.startswith(base + ".") + ) + + +def _update_prev_state(lt: LoopTimings, loop_seq: int): + """Capture the previous loop summary and store this loop for the next one. + + Returns ``(prev_seq, prev_has_detail, prev_summary, consecutive)``. Only a + loop that actually recorded detail becomes a valid predecessor, so a + sampled loop after an unsampled one is marked non-consecutive instead of + silently publishing a stale window (design §4.5). + """ + global _LAST_LOOP_SEQ, _LAST_LOOP_HAS_DETAIL + prev_seq = int(_LAST_LOOP_SEQ) + prev_has = bool(_LAST_LOOP_HAS_DETAIL) + prev_summary = dict(_LAST_LOOP_SUMMARY) + consecutive = prev_has and prev_seq == loop_seq - 1 + + ms = { + name: rec[0] / 1e6 + for name, rec in ((lt.detail or {}).get("phases") or {}).items() + if rec + } + _LAST_LOOP_SUMMARY.clear() + if lt.detail is not None: + _LAST_LOOP_SUMMARY.update( + { + "run_batch_ms": float( + lt.run_batch[2].get("sglang.run_batch_ms", 0.0) + ) + if lt.run_batch is not None + else 0.0, + "process_result_ms": float( + lt.process[2].get("sglang.process_result_ms", 0.0) + ) + if lt.process is not None + else 0.0, + "copy_done_wait_ms": _total_ms(ms, "copy_done_wait"), + "kv_state_payload_ms": _total_ms(ms, "kv_state_payload"), + "kv_page_indices_ms": _total_ms(ms, "kv_page_indices"), + "kv_sender_send_ms": _total_ms(ms, "kv_sender_send"), + } + ) + _LAST_LOOP_SEQ = int(loop_seq) + _LAST_LOOP_HAS_DETAIL = lt.detail is not None + return prev_seq, prev_has, prev_summary, consecutive + + +def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None: + """Expose detail phases on the loop span and its child spans. + + Single-occurrence phases become children (``scheduler.loop.``); + repeated phases only contribute summed attributes, so span count never + grows with n_reqs / n_segments. + """ + detail = lt.detail or {} + phases = detail.get("phases") or {} + ms = {} + for name, rec in phases.items(): + try: + total_ns, count, start_wall, end_wall = rec + except Exception: + continue + key = "sglang." + name.replace(".", "_") + "_ms" + ms[name] = total_ns / 1e6 + attrs[key] = ms[name] + if count > 1: + attrs["sglang." + name.replace(".", "_") + "_count"] = int(count) + continue + if name.split(".", 1)[0] in _DETAIL_ATTRIBUTE_ONLY_PREFIXES: + continue + child( + "scheduler.loop." + name.replace(".", "_"), + start_wall, + end_wall, + { + "sglang.dp_rank": int(dp_rank), + "sglang.phase_ms": total_ns / 1e6, + }, + ) + + for name, value in (detail.get("counts") or {}).items(): + attrs["sglang." + name.replace(".", "_") + "_count"] = int(value) + for name, value in (detail.get("attrs") or {}).items(): + attrs["sglang." + name.replace(".", "_")] = value + + # Accounting residuals (design §3.2): never negative, never folded in. + def _total(*bases): + total = 0.0 + for base in bases: + for name, value in ms.items(): + if name == base or name.startswith(base + "."): + total += value + return total + + # Roll per-kind variants (kv_page_indices.last, ...) up to one base + # attribute so analysis can use the total without knowing the kinds. + bases = {name.split(".", 1)[0] for name in ms} + for base in bases: + if base not in ms: + attrs["sglang." + base + "_ms"] = _total(base) + + if lt.run_batch is not None: + run_ms = float(lt.run_batch[2].get("sglang.run_batch_ms", 0.0)) + parts = _total( + "early_send_host", + "resolve_inputs", + "model_forward_host", + "copy_result_submit", + ) + attrs["sglang.run_residual_ms"] = max(0.0, run_ms - parts) + if lt.process is not None: + proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0)) + parts = _total( + "copy_done_wait", + "auxiliary_and_logprob", + "per_request_result", + "report_stats", + ) + if "sglang.process_kv_ms" in lt.process[2]: + # KV phases charged to this call only (sends issued later in the + # loop, e.g. pending-bootstrap, are not part of process_result). + parts += float(lt.process[2]["sglang.process_kv_ms"]) + else: + parts += _total( + "kv_state_payload", "kv_page_indices", "kv_sender_send" + ) + attrs["sglang.process_residual_ms"] = max(0.0, proc_ms - parts) + + + def end_loop( lt: Optional[LoopTimings], *, @@ -1419,6 +2139,14 @@ def end_loop( pass lt.no_batch_reason = reason + # Update the prev-loop chain on every loop (sampled or not) so a + # sampled loop after an unsampled one is reported as non-consecutive + # instead of silently carrying a stale window (review finding). + loop_seq_cur = int(lt.seq or lt.predicted_seq) + prev_state = None + if _DETAIL_ENABLED: + prev_state = _update_prev_state(lt, loop_seq_cur) + if lt.span is None and (has_batch or pending): # Covers work discovered after begin_loop, including first input and # deferred-only loops. DP outcomes activate earlier for pass parenting. @@ -1445,7 +2173,6 @@ def end_loop( "sglang.forward_overlap": overlap, } attrs.update(_batch_attrs(batch)) - lt.span.set_attributes(attrs) def _child(name, start_ns, end_ns, child_attrs): try: @@ -1458,6 +2185,26 @@ def end_loop( except Exception: pass + if lt.detail is not None: + _emit_loop_detail(lt, attrs, _child, int(dp_rank)) + if _DETAIL_ENABLED: + if lt.pass_end_wall_ns: + # DP-skew join fields (design §4.5); independent of phases. + attrs["sglang.pass_end_wall_ns"] = int(lt.pass_end_wall_ns) + attrs["sglang.arrival_est_wall_ns"] = int(lt.arrival_est_wall_ns) + if prev_state is not None: + prev_seq, prev_has, prev_summary, consecutive = prev_state + attrs["sglang.prev_loop_seq"] = int(prev_seq) + if consecutive: + for key in _DETAIL_PREV_KEYS: + if key in prev_summary: + attrs["sglang.prev_" + key] = prev_summary[key] + elif prev_seq: + # The previous loop was unsampled or had no detail; + # its phases must not be treated as seq-1 values. + attrs["sglang.prev_detail_missing"] = True + lt.span.set_attributes(attrs) + if lt.recv is not None and not lt.recv_span_child: start, end, parts = lt.recv child = {"sglang.dp_rank": int(dp_rank)} diff --git a/test/registered/unit/observability/test_loop_detail_phases.py b/test/registered/unit/observability/test_loop_detail_phases.py new file mode 100644 index 0000000000..c737802d8f --- /dev/null +++ b/test/registered/unit/observability/test_loop_detail_phases.py @@ -0,0 +1,369 @@ +"""Unit tests for SGLANG_TRACE_LOOP_DETAIL / SGLANG_TRACE_MOONCAKE_DETAIL. + +Covers the design's phase-1/phase-2 acceptance points +(handoff/prefill-cpu-kv-overlap-design-20261106.md §8.1): + + * detail is off by default and records nothing; + * detail only records for sampled loops (same decision as the loop span); + * single phases become child spans, repeated phases only attributes; + * residuals are clamped at zero and never folded into a phase; + * per-kind KV phases roll up to one base attribute; + * the previous-loop summary is published for DP-skew correlation; + * chunk spans break worker busy time into queue/source/indices/transfer. +""" +import importlib.util +import os +import pathlib +import sys +import time +import types +import unittest +from unittest import mock + + +def _module_path(): + p = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/observability/batch_telemetry.py" + ) + if p.exists(): + return p + spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry") + return pathlib.Path(spec.origin) + + +def _load(**env): + base = { + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_TRACE_LOOP_SPAN": "1", + "SGLANG_TRACE_LOOP_SAMPLE": "1.0", + "SGLANG_TRACE_LOOP_DETAIL": "1", + "SGLANG_TRACE_MOONCAKE_DETAIL": "0", + "SGLANG_TRACE_FORWARD_SPAN": "0", + } + base.update(env) + with mock.patch.dict(os.environ, base): + spec = importlib.util.spec_from_file_location( + "batch_telemetry_detail_under_test", _module_path() + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class _FakeCounter: + def __init__(self): + self.data = {} + + def labels(self, **kw): + self._key = tuple(sorted(kw.items())) + return self + + def inc(self, value=1.0): + self.data[self._key] = self.data.get(self._key, 0.0) + value + + +def _fake_loop_metrics(): + return { + "no_batch_seconds": _FakeCounter(), + "no_batch_total": _FakeCounter(), + "lockstep_seconds": _FakeCounter(), + } + + +class _FakeSpan: + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs + self.attributes = dict(kwargs.get("attributes") or {}) + self.end_time = None + + def set_attributes(self, attrs): + self.attributes.update(attrs) + + def get_span_context(self): + return types.SimpleNamespace( + is_valid=True, trace_id=1, span_id=2 + ) + + def end(self, end_time=None): + self.end_time = end_time + + +class _FakeTracer: + def __init__(self): + self.spans = [] + + def start_span(self, name, **kwargs): + span = _FakeSpan(name, **kwargs) + self.spans.append(span) + return span + + +def _fake_otel(): + tracer = _FakeTracer() + trace = types.ModuleType("opentelemetry.trace") + trace.get_tracer = lambda _name: tracer + trace.set_span_in_context = lambda span: ("ctx", span) + trace.Link = lambda ctx: ("link", ctx) + package = types.ModuleType("opentelemetry") + package.trace = trace + patcher = mock.patch.dict( + sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace} + ) + return tracer, patcher + + +def _state(**overrides): + state = { + "waiting": 0, + "result_queue": 0, + "result_limit": 2, + "inflight": 0, + "bootstrap": 0, + "deferred": 0, + "paused": False, + "waiting_for_result": False, + "skip_batch_creation": False, + "forward_active": False, + } + state.update(overrides) + return state + + +def _span(tracer, name): + return next(s for s in tracer.spans if s.name == name) + + +def _last_loop_attrs(tracer): + spans = [s for s in tracer.spans if s.name == "scheduler.loop"] + return spans[-1].attributes + + +def _pass_with_seq(mod, seq): + pass_t = mod.PassTimings("prefill") + pass_t.seq = seq + return pass_t + + +class LoopDetailTest(unittest.TestCase): + def test_disabled_records_nothing(self): + mod = _load(SGLANG_TRACE_LOOP_DETAIL="0") + self.assertFalse(mod.loop_detail_enabled()) + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + self.assertIsNone(mod.detail_begin("model_forward_host")) + mod.detail_add("kv_page_indices.last", 0, 10_000) + mod.detail_count("n_segments") + mod.detail_attr("state_types", "SWA") + mod.mark_run_batch(0, 50_000_000, None) + tracer, patcher = _fake_otel() + with patcher: + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _span(tracer, "scheduler.loop").attributes + self.assertNotIn("sglang.kv_page_indices_ms", attrs) + self.assertNotIn("sglang.state_types", attrs) + self.assertIsNone(lt.detail) + + def test_unsampled_loop_records_nothing(self): + mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0") + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + self.assertFalse(mod.loop_detail_enabled()) + self.assertIsNone(mod.detail_begin("model_forward_host")) + mod.detail_add("kv_page_indices.last", 0, 1_000) + self.assertIsNone(lt.detail) + + def test_phases_attributes_and_residuals(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + self.assertTrue(mod.loop_detail_enabled()) + + # Single phase -> child span. + tok = mod.detail_begin("report_stats") + mod.detail_end("report_stats", tok) + with mod.detail_phase("copy_done_wait"): + pass + + # Repeated phase -> attributes only, split by send_kind. + t0 = time.monotonic_ns() + mod.detail_add("kv_page_indices.cached_prefix", t0, t0 + 2_000_000) + mod.detail_add("kv_page_indices.last", t0, t0 + 5_000_000) + mod.detail_add("kv_sender_send.last", t0, t0 + 3_000_000) + mod.detail_count("n_segments", 3) + mod.detail_attr("state_types", "REQUEST_ROW,SWA") + + # run_batch = 100 ms with 20 ms of measured phases. + start = time.time_ns() + mod.detail_add("early_send_host", t0, t0 + 10_000_000) + mod.detail_add("resolve_inputs", t0, t0 + 5_000_000) + mod.detail_add("model_forward_host", t0, t0 + 50_000_000) + mod.detail_add("copy_result_submit", t0, t0 + 5_000_000) + mod.mark_run_batch(start, start + 100_000_000, None) + + # process_result = 100 ms with 62 ms measured -> residual ~38 ms. + # copy_done_wait is a real measured span (sub-ms) on top of the + # synthetic 20 ms used for the run_batch accounting above. + mod.detail_add("copy_done_wait", t0, t0 + 20_000_000) + mod.detail_add("per_request_result", t0, t0 + 40_000_000) + mod.mark_process(start, start + 100_000_000, None) + + mod.end_loop(lt, dp_rank=2, has_batch=True, state=_state()) + + loop = _span(tracer, "scheduler.loop") + attrs = loop.attributes + self.assertAlmostEqual(attrs["sglang.kv_page_indices_ms"], 7.0) + self.assertAlmostEqual(attrs["sglang.kv_page_indices_cached_prefix_ms"], 2.0) + self.assertAlmostEqual(attrs["sglang.kv_page_indices_last_ms"], 5.0) + self.assertEqual(attrs["sglang.n_segments_count"], 3) + self.assertEqual(attrs["sglang.state_types"], "REQUEST_ROW,SWA") + # model_forward_host appears once -> child span + attribute. + self.assertAlmostEqual(attrs["sglang.model_forward_host_ms"], 50.0) + self.assertAlmostEqual(attrs["sglang.run_residual_ms"], 30.0, places=3) + # copy_done_wait 20 + per_request 40 + kv_page_indices 7 + sender 3. + self.assertAlmostEqual(attrs["sglang.process_residual_ms"], 30.0, delta=2.0) + names = [s.name for s in tracer.spans] + self.assertIn("scheduler.loop.model_forward_host", names) + self.assertIn("scheduler.loop.report_stats", names) + # Per-request / per-segment phases never create spans. + self.assertNotIn("scheduler.loop.kv_page_indices_last", names) + self.assertNotIn("scheduler.loop.per_request_result", names) + + def test_residual_is_clamped_at_zero(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add("per_request_result", t0, t0 + 300_000_000) + mod.mark_process(time.time_ns(), time.time_ns() + 100_000_000, None) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _span(tracer, "scheduler.loop").attributes + self.assertEqual(attrs["sglang.process_residual_ms"], 0.0) + + def test_process_kv_scope_excludes_later_sends(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + # KV send issued inside process_result (charged) and a second one + # issued later in the same loop by the inflight queue (not charged). + mod.detail_add("kv_sender_send.last", t0, t0 + 10_000_000) + start = time.time_ns() + mod.mark_process( + start, start + 100_000_000, {"sglang.process_kv_ms": 10.0} + ) + mod.detail_add("kv_sender_send.last", t0, t0 + 500_000_000) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + attrs = _span(tracer, "scheduler.loop").attributes + self.assertAlmostEqual(attrs["sglang.process_residual_ms"], 90.0) + + def test_previous_loop_summary_is_published(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add("copy_done_wait", t0, t0 + 7_000_000) + mod.detail_add("kv_sender_send.last", t0, t0 + 2_000_000) + start = time.time_ns() + mod.mark_run_batch(start, start + 30_000_000, None) + mod.mark_process(start, start + 50_000_000, None) + # Consecutive loop_seq values are required for prev_* to be + # published (a sampled loop after an unsampled one is marked). + mod.note_pass_snapshot(_pass_with_seq(mod, 5), 0.01) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + + lt2 = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.note_pass_snapshot(_pass_with_seq(mod, 6), 0.01) + mod.end_loop(lt2, dp_rank=0, has_batch=True, state=_state()) + + attrs = _last_loop_attrs(tracer) + self.assertEqual(attrs["sglang.prev_loop_seq"], 5) + self.assertAlmostEqual(attrs["sglang.prev_run_batch_ms"], 30.0) + self.assertAlmostEqual(attrs["sglang.prev_process_result_ms"], 50.0) + self.assertAlmostEqual(attrs["sglang.prev_copy_done_wait_ms"], 7.0) + self.assertAlmostEqual(attrs["sglang.prev_kv_sender_send_ms"], 2.0) + + def test_phase_total_supports_per_kind_names(self): + """Regression: recorded names are kv_*.last/middle/cached_prefix.""" + mod = _load() + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add("kv_page_indices.cached_prefix", t0, t0 + 2_000_000) + mod.detail_add("kv_page_indices.middle", t0, t0 + 3_000_000) + mod.detail_add("kv_sender_send.last", t0, t0 + 10_000_000) + self.assertEqual( + mod.detail_phase_total_ns(("kv_page_indices",)), 5_000_000 + ) + self.assertEqual(mod.detail_phase_total_ns(("kv_sender_send",)), 10_000_000) + self.assertEqual( + mod.detail_phase_total_ns( + ("kv_state_payload", "kv_page_indices", "kv_sender_send") + ), + 15_000_000, + ) + # A base name must not swallow an unrelated longer base. + self.assertEqual(mod.detail_phase_total_ns(("kv_page",)), 0) + mod._LOOP_TLS.current = None + + def test_prev_summary_not_published_after_unsampled_loop(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.detail_add("copy_done_wait", time.monotonic_ns(), time.monotonic_ns()) + mod.note_pass_snapshot(_pass_with_seq(mod, 10), 0.01) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + + # Loop 11 has no detail (simulated unsampled): the chain must not + # pretend loop 12's predecessor is loop 10. + lt2 = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + lt2.detail = None + mod.note_pass_snapshot(_pass_with_seq(mod, 11), 0.01) + mod.end_loop(lt2, dp_rank=0, has_batch=True, state=_state()) + + lt3 = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + mod.detail_add("copy_done_wait", time.monotonic_ns(), time.monotonic_ns()) + mod.note_pass_snapshot(_pass_with_seq(mod, 12), 0.01) + mod.end_loop(lt3, dp_rank=0, has_batch=True, state=_state()) + attrs = _last_loop_attrs(tracer) + self.assertEqual(attrs["sglang.prev_loop_seq"], 11) + self.assertTrue(attrs["sglang.prev_detail_missing"]) + self.assertNotIn("sglang.prev_copy_done_wait_ms", attrs) + + def test_arrival_estimate_uses_pass_end_minus_mlp_sync(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + pass_t = mod.PassTimings("prefill") + pass_t.seq = lt.predicted_seq + pass_t.mlp_sync = 0.05 + mod.note_pass_snapshot(pass_t, 0.1) + mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state()) + attrs = _span(tracer, "scheduler.loop").attributes + self.assertEqual( + attrs["sglang.pass_end_wall_ns"] - attrs["sglang.arrival_est_wall_ns"], + 50_000_000, + ) + + def test_current_loop_seq_available_to_senders(self): + mod = _load() + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + pass_t = mod.PassTimings("prefill") + pass_t.seq = 41 + mod.note_pass_snapshot(pass_t, 0.01) + self.assertEqual(mod.current_loop_seq(), 41) + mod._LOOP_TLS.current = None + + +if __name__ == "__main__": + unittest.main()