observability: trace KV transfer chunks end-to-end (M2)

Implements M2 of handoff/prefill-cpu-kv-overlap-design-20261106.md, behind
SGLANG_TRACE_MOONCAKE_DETAIL=1 (default off; reuses the existing trace stack).

- TransferKVChunk gains optional observation fields: chunk_seq, loop_seq,
  send_kind, enqueue/last_enqueue monotonic+wall stamps, queue_wait_accum_ns,
  queue_id, requeue_count, num_pages, approx_bytes, observe
- chunk_seq and enqueue timestamps are assigned at the real queue.put()
  point (after every early return), so skipped paths cannot leave holes
- Mooncake transfer worker measures queue wait, source-event wait, indices
  wait, KV/state/aux transfer, worker busy and residual; emits one
  mooncake.kv_chunk span per attempt (+ queue/source/indices wait children)
  and updates a per-room join summary
- request completion emits kv.request_join with chunk_count,
  last_chunk_queue_wait/worker, final_exposed_ms, completion_poll_ms,
  request_join_ms and first-enqueue-to-completion. First enqueue is stamped
  by the Mooncake manager at the real queue.put(); no ReqTimeStats field is
  added and the legacy queue_entry semantics are unchanged.
- nixl/mori add_transfer_request accept the observation kwarg (no-op there)

No scheduling/transfer semantics change; queue wait is excluded from worker
busy, and worker_busy excludes source/indices event waits. Unit tests: test_mooncake_chunk_detail.py (15 cases, including the
abort/worker interleaving and attempt-vs-total queue wait windows).
This commit is contained in:
ymk
2026-11-06 20:01:56 +08:00
parent 72e556ec81
commit fe9c6574fe
7 changed files with 884 additions and 0 deletions
@@ -51,6 +51,7 @@ from sglang.srt.disaggregation.utils import (
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.trace import TraceNullContext, TraceReqContext
from sglang.srt.runtime_context import (
get_device,
@@ -2091,6 +2092,10 @@ class CommonKVSender(BaseKVSender):
self._transfer_num_state_indices = 0
self._transfer_start_time: Optional[float] = None
self._early_send_wait_event: Optional[object] = None
# Detail tracing: chunk classification only. Enqueue timestamps,
# chunk_seq and the per-request first-enqueue are assigned by the
# manager at the real queue.put() point, after all early returns.
self._send_kind = ""
self.trace_ctx = TraceNullContext()
if self.kv_mgr.sender_trace_module and get_observability().enable_trace:
self.trace_ctx = TraceReqContext(
@@ -2289,6 +2294,19 @@ class CommonKVSender(BaseKVSender):
):
self._transfer_start_time = time.perf_counter()
obs = None
if batch_telemetry.mooncake_detail_enabled():
# Metadata only; the manager stamps chunk_seq and enqueue times
# when it is certain the chunk is actually put on the queue.
obs = {
"loop_seq": int(batch_telemetry.current_loop_seq()),
"send_kind": str(self._send_kind or ""),
"num_pages": int(len(kv_indices)),
"approx_bytes": int(
len(kv_indices) * getattr(self.kv_mgr, "kv_item_lens_sum", 0)
),
}
# The early-send event belongs to exactly one chunk; the Mooncake and
# Mori workers synchronize on it before reading source KV.
wait_event = self._early_send_wait_event
@@ -2304,9 +2322,27 @@ class CommonKVSender(BaseKVSender):
wait_event=wait_event,
indices_event=indices_event,
trace_ctx=self.trace_ctx.copy_for_thread(),
obs=obs,
)
self._record_transfer_indices(kv_indices, state_indices)
def pop_chunk_join_stats(self) -> Optional[dict]:
"""Take (and free) this room's chunk join summary."""
state = getattr(self.kv_mgr, "_room_detail", None)
if state is None:
return None
return state.pop_join(self.bootstrap_room)
def reset_chunk_detail_state(self) -> None:
"""Drop per-room detail state and mark it cancelled (abort / retract).
The mark is what prevents a worker attempt that already passed its
status check from recreating stats after this cleanup.
"""
state = getattr(self.kv_mgr, "_room_detail", None)
if state is not None:
state.reset(self.bootstrap_room)
def poll(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status)
@@ -2425,6 +2461,9 @@ class CommonKVSender(BaseKVSender):
def abort(self):
if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
return
# Drop per-room detail bookkeeping: an aborted request never reaches
# the poll path that normally consumes it (review finding).
self.reset_chunk_detail_state()
self.kv_mgr.record_failure(
self.bootstrap_room,
"Aborted by AbortReq.",
@@ -36,6 +36,24 @@ class TransferKVChunk:
# Must complete before reading asynchronously copied host indices.
indices_event: Optional[object] = None
# --- Optional observation fields (SGLANG_TRACE_MOONCAKE_DETAIL=1) ------
# Filled at the real enqueue point (add_transfer_request), never at the
# send_kv_chunk entry, so skipped paths cannot create holes in chunk_seq.
chunk_seq: int = 0 # per-sender monotonic enqueue index
loop_seq: int = 0 # scheduler loop that produced this chunk
send_kind: str = "" # cached_prefix / middle / last
enqueue_mono_ns: int = 0 # first enqueue of this chunk, monotonic clock
enqueue_wall_ns: int = 0 # first enqueue of this chunk, wall clock
last_enqueue_mono_ns: int = 0 # updated on every (re-)enqueue
last_enqueue_wall_ns: int = 0 # wall clock of the last (re-)enqueue
dequeue_mono_ns: int = 0 # current attempt dequeue; 0 outside the worker
queue_wait_accum_ns: int = 0 # completed queue waits from earlier attempts
queue_id: int = -1
requeue_count: int = 0
num_pages: int = 0
approx_bytes: int = 0
observe: bool = False # True when detail tracing populated this chunk
def pack_list_of_buffers(buffers: List[bytes]) -> bytes:
if not buffers:
@@ -60,6 +60,7 @@ from sglang.srt.disaggregation.utils import (
)
from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine
from sglang.srt.environ import envs
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.mooncake_trace import (
MooncakeRequestStage,
mooncake_trace_func,
@@ -386,6 +387,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# transfer blocks over zmq when the intra-node NVLink transport is
# active (it cannot reach host memory). Populated on bootstrap.
self._session_endpoint_map = {}
# Detail tracing (SGLANG_TRACE_MOONCAKE_DETAIL=1; empty otherwise).
# Lock-protected per-room state: abort/retract and the worker's join
# update must not interleave into resurrected stats.
self._room_detail = batch_telemetry.RoomDetailState()
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
self.max_transfer_batch_indices = (
envs.SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES.get()
@@ -762,6 +767,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# block other rooms on this single worker thread.
with self._staging_ctx.watermark_cv:
self._staging_ctx.watermark_cv.wait(STAGING_WATERMARK_WAIT_S)
if getattr(kv_chunk, "observe", False):
batch_telemetry.note_chunk_requeue(
kv_chunk, time.monotonic_ns(), time.time_ns()
)
queue.put(kv_chunk)
return (-1, True)
@@ -2716,6 +2725,65 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if self.enable_deferred_decode_kv_release:
self.poison_deferred_ack_room(room)
def _emit_chunk_attempt(
self,
kv_chunk: TransferKVChunk,
*,
status: str,
queue_wait_ns: int,
queue_wait_total_ns: int,
source_wait_ns: int,
indices_wait_ns: int,
kv_transfer_ns: int,
state_transfer_ns: int,
aux_transfer_ns: int,
work_start_ns: int,
dequeue_wall_ns: int,
n_destinations: int,
source_wait_wall=None,
indices_wait_wall=None,
) -> None:
"""Close one worker attempt: chunk span + per-room join summary."""
now_mono_ns = time.monotonic_ns()
now_wall_ns = time.time_ns()
attempt_wall_ns = max(0, now_mono_ns - int(work_start_ns))
worker_busy_ns = max(
0, attempt_wall_ns - int(source_wait_ns) - int(indices_wait_ns)
)
batch_telemetry.emit_mooncake_chunk(
chunk=kv_chunk,
status=status,
queue_wait_ns=queue_wait_ns,
queue_wait_total_ns=queue_wait_total_ns,
source_wait_ns=source_wait_ns,
indices_wait_ns=indices_wait_ns,
kv_transfer_ns=kv_transfer_ns,
state_transfer_ns=state_transfer_ns,
aux_transfer_ns=aux_transfer_ns,
attempt_wall_ns=attempt_wall_ns,
dequeue_wall_ns=dequeue_wall_ns,
end_wall_ns=now_wall_ns,
n_destinations=n_destinations,
source_wait_wall=source_wait_wall,
indices_wait_wall=indices_wait_wall,
dp_rank=self.attn_dp_rank,
tp_rank=self.attn_tp_rank,
)
# The status was checked before this call; the registry re-checks the
# cancellation mark under its lock, which is what makes an abort that
# lands in between unable to resurrect stats (review finding).
self._room_detail.record_join(
kv_chunk.room,
status=status,
room_failed=self.check_status(kv_chunk.room) == KVPoll.Failed,
now_wall_ns=now_wall_ns,
queue_wait_total_ns=queue_wait_total_ns,
worker_busy_ns=worker_busy_ns,
enqueue_wall_ns=kv_chunk.enqueue_wall_ns,
is_last_chunk=kv_chunk.is_last_chunk,
chunk_seq=kv_chunk.chunk_seq,
)
def transfer_worker(
self,
queue: FastQueue,
@@ -2740,6 +2808,34 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# _stopped alone can never wake a parked worker during a role switch.
if kv_chunk is None:
break
_obs = bool(getattr(kv_chunk, "observe", False))
_dequeue_mono_ns = time.monotonic_ns() if _obs else 0
_dequeue_wall_ns = time.time_ns() if _obs else 0
if _obs:
# Needed by note_chunk_requeue(): queue wait ends here and
# must not include worker/watermark time.
kv_chunk.dequeue_mono_ns = _dequeue_mono_ns
_queue_wait_ns = 0
_source_wait_ns = 0
_source_wait_wall = None
_indices_wait_ns = 0
_indices_wait_wall = None
_kv_transfer_ns = 0
_state_transfer_ns = 0
_aux_transfer_ns = 0
_n_dest = 0
if _obs:
# queue wait is not part of worker busy time (design §5.2).
# Keep the attempt window (child span) separate from the
# cumulative value across deferred attempts (review finding).
(
_attempt_queue_wait_ns,
_queue_wait_ns,
) = batch_telemetry.split_queue_wait(
kv_chunk.queue_wait_accum_ns,
kv_chunk.last_enqueue_mono_ns,
_dequeue_mono_ns,
)
if self.enable_trace:
kv_chunk.trace_ctx.rebuild_thread_context()
kv_chunk.trace_ctx.trace_slice_start(
@@ -2765,13 +2861,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
kv_chunk.room in self.request_status
and self.check_status(kv_chunk.room) != KVPoll.Failed
):
_wait_t0 = time.monotonic_ns() if _obs else 0
_wait_wall0 = time.time_ns() if _obs else 0
kv_chunk.wait_event.synchronize()
if _wait_t0:
_source_wait_ns += time.monotonic_ns() - _wait_t0
_wait_wall1 = time.time_ns()
_source_wait_wall = (_wait_wall0, _wait_wall1)
kv_chunk.wait_event = None
_ind_t0 = time.monotonic_ns() if _obs else 0
_ind_wall0 = time.time_ns() if _obs else 0
if kv_chunk.indices_event is not None:
kv_chunk.indices_event.synchronize()
kv_chunk.indices_event = None
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
if _ind_t0:
# synchronize + host indices availability, per design §5.2.
_indices_wait_ns += time.monotonic_ns() - _ind_t0
_indices_wait_wall = (_ind_wall0, time.time_ns())
if (
kv_chunk.room not in self.request_status
@@ -2787,6 +2895,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
thread_finish_flag=True,
)
self._staging_outstanding.pop(kv_chunk.room, None)
if _obs:
self._emit_chunk_attempt(
kv_chunk,
status="skipped",
queue_wait_ns=_attempt_queue_wait_ns,
queue_wait_total_ns=_queue_wait_ns,
source_wait_ns=_source_wait_ns,
indices_wait_ns=_indices_wait_ns,
kv_transfer_ns=_kv_transfer_ns,
state_transfer_ns=_state_transfer_ns,
aux_transfer_ns=_aux_transfer_ns,
work_start_ns=_dequeue_mono_ns,
dequeue_wall_ns=_dequeue_wall_ns,
n_destinations=_n_dest,
source_wait_wall=_source_wait_wall,
indices_wait_wall=_indices_wait_wall,
)
if self.enable_deferred_decode_kv_release:
# Skipped => nothing written for this aborted room; ack.
self._maybe_ack_drained_abort(kv_chunk.room)
@@ -2882,6 +3007,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
skip_kv, skip_state = self._get_dsa_cache_transfer_skip_flags(
target_rank_registration_info
)
_kv_t0 = time.monotonic_ns() if _obs else 0
if (
len(kv_chunk.prefill_kv_indices) == 0
or not self.kv_args.kv_data_ptrs
@@ -2954,6 +3080,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
if deferred:
staging_deferred = True
if _kv_t0:
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
_n_dest += 1
# Chunk re-enqueued; stop processing remaining reqs for this chunk
break
else:
@@ -2968,6 +3097,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
executor,
target_rank_registration_info.dst_kv_layer_ids,
)
if _kv_t0:
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
_n_dest += 1
if ret != 0:
with self.session_lock:
self.session_failures[req.mooncake_session_id] += 1
@@ -2989,12 +3121,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if kv_chunk.is_last_chunk:
if kv_chunk.state_indices and not skip_state:
_state_t0 = time.monotonic_ns() if _obs else 0
state_rc = self.maybe_send_extra(
req,
kv_chunk.state_indices,
executor,
target_rank_registration_info,
)
if _state_t0:
_state_transfer_ns += time.monotonic_ns() - _state_t0
# Before the Failed check: an ABORT during the
# send also marks the room Failed.
if state_rc != 0:
@@ -3025,11 +3160,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
break
# Only the last chunk we need to send the aux data
_aux_t0 = time.monotonic_ns() if _obs else 0
ret = self.send_aux(
req,
kv_chunk.prefill_aux_index,
target_rank_registration_info.dst_aux_ptrs,
)
if _aux_t0:
_aux_transfer_ns += time.monotonic_ns() - _aux_t0
if ret != 0:
# The aux write targets decode's metadata
# slot, which the deferred release also frees.
@@ -3071,8 +3209,45 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
if staging_deferred:
if _obs:
self._emit_chunk_attempt(
kv_chunk,
status="deferred",
queue_wait_ns=_attempt_queue_wait_ns,
queue_wait_total_ns=_queue_wait_ns,
source_wait_ns=_source_wait_ns,
indices_wait_ns=_indices_wait_ns,
kv_transfer_ns=_kv_transfer_ns,
state_transfer_ns=_state_transfer_ns,
aux_transfer_ns=_aux_transfer_ns,
work_start_ns=_dequeue_mono_ns,
dequeue_wall_ns=_dequeue_wall_ns,
n_destinations=_n_dest,
source_wait_wall=_source_wait_wall,
indices_wait_wall=_indices_wait_wall,
)
continue
if _obs:
self._emit_chunk_attempt(
kv_chunk,
status="ok"
if self.check_status(kv_chunk.room) != KVPoll.Failed
else "failed",
queue_wait_ns=_attempt_queue_wait_ns,
queue_wait_total_ns=_queue_wait_ns,
source_wait_ns=_source_wait_ns,
indices_wait_ns=_indices_wait_ns,
kv_transfer_ns=_kv_transfer_ns,
state_transfer_ns=_state_transfer_ns,
aux_transfer_ns=_aux_transfer_ns,
work_start_ns=_dequeue_mono_ns,
dequeue_wall_ns=_dequeue_wall_ns,
n_destinations=_n_dest,
source_wait_wall=_source_wait_wall,
indices_wait_wall=_indices_wait_wall,
)
self._staging_outstanding[kv_chunk.room] -= 1
if self.enable_deferred_decode_kv_release:
# In-flight write finished; if aborted and nothing outstanding,
@@ -3096,6 +3271,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self._session_endpoint_map.pop(sid, None)
self.transfer_infos.pop(kv_chunk.room)
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
# Success: the scheduler poll still needs the join summary.
# Failed/absent: nothing will consume it, drop it here.
self._room_detail.teardown(
kv_chunk.room,
keep_join=(
self.request_status.get(kv_chunk.room)
== KVPoll.Success
),
)
if self.enable_staging:
# Purge prefetch bookkeeping for the finished room.
# Snapshot first: the scheduler thread adds concurrently.
@@ -3332,6 +3516,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
wait_event: Optional[object] = None,
indices_event: Optional[object] = None,
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
obs: Optional[dict] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -3361,6 +3546,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if trace_ctx is None:
trace_ctx = TraceNullContext()
if obs is not None:
# Stamp only now: every earlier return (failed room, dummy rank,
# zero-page skip) must not consume a chunk_seq or a first-enqueue
# timestamp (review finding).
now_mono_ns = time.monotonic_ns()
now_wall_ns = time.time_ns()
chunk_seq, _first_wall_ns = self._room_detail.begin_chunk(
bootstrap_room, now_mono_ns, now_wall_ns
)
obs = dict(obs)
obs.update(
chunk_seq=chunk_seq,
enqueue_mono_ns=now_mono_ns,
enqueue_wall_ns=now_wall_ns,
last_enqueue_wall_ns=now_wall_ns,
)
self.transfer_queues[shard_idx].put(
TransferKVChunk(
room=bootstrap_room,
@@ -3373,6 +3575,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
trace_ctx=trace_ctx,
wait_event=wait_event,
indices_event=indices_event,
**(obs or {}),
queue_id=int(shard_idx),
observe=obs is not None,
last_enqueue_mono_ns=int((obs or {}).get("enqueue_mono_ns", 0)),
)
)
@@ -804,6 +804,7 @@ class MoriKVManager(CommonKVManager):
wait_event: Optional[object] = None,
indices_event: Optional[object] = None,
trace_ctx=None,
obs: Optional[dict] = None,
) -> None:
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -3267,6 +3267,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
wait_event=None,
indices_event=None,
trace_ctx=None,
obs: Optional[dict] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -1426,11 +1426,30 @@ class SchedulerDisaggregationPrefillMixin:
self.tree_cache.finish(
req.cache_request_handle, CacheRequestOutcome.SUCCESS
)
# Consume the join summary before clear() releases the room.
_join_stats = None
if batch_telemetry.mooncake_detail_enabled():
_pop_stats = getattr(
req.disagg_kv_sender, "pop_chunk_join_stats", None
)
_join_stats = _pop_stats() if _pop_stats is not None else None
# FIXME: clean up req's data in transfer engine
req.disagg_kv_sender.clear()
done_reqs.append(req)
req.time_stats.set_prefill_kv_transfer_finish_time()
if batch_telemetry.mooncake_detail_enabled():
batch_telemetry.emit_request_kv_join(
req=req,
stats=_join_stats,
dp_rank=get_parallel().attn_dp_rank,
)
elif poll == KVPoll.Failed:
if batch_telemetry.mooncake_detail_enabled():
_pop_stats = getattr(
req.disagg_kv_sender, "pop_chunk_join_stats", None
)
if _pop_stats is not None:
_pop_stats()
self.handle_inflight_transfer_failure(req)
done_reqs.append(req)
else:
@@ -2016,6 +2035,12 @@ class SchedulerDisaggregationPrefillMixin:
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
"""Release KV cache and requeue an optimistic prefill request."""
if batch_telemetry.mooncake_detail_enabled():
# A retry may reuse the same room; stale chunk stats would
# otherwise be merged into the next attempt's join summary.
_reset = getattr(req.disagg_kv_sender, "reset_chunk_detail_state", None)
if _reset is not None:
_reset()
max_attempts = get_disagg().optimistic_prefill_attempts
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
if not uses_write_through_cache:
@@ -0,0 +1,594 @@
"""Unit tests for Mooncake chunk-level tracing (SGLANG_TRACE_MOONCAKE_DETAIL).
Covers design §5: queue/source/indices/transfer breakdown per worker attempt,
request-level join fields, and the additive fields on TransferKVChunk.
"""
import importlib.util
import os
import pathlib
import sys
import time
import types
import unittest
from unittest import mock
def _module_path():
p = (
pathlib.Path(__file__).resolve().parents[4]
/ "python/sglang/srt/observability/batch_telemetry.py"
)
if p.exists():
return p
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
return pathlib.Path(spec.origin)
def _load(**env):
base = {
"SGLANG_BATCH_STAGE_TIMING": "1",
"SGLANG_TRACE_LOOP_SPAN": "1",
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
"SGLANG_TRACE_LOOP_DETAIL": "1",
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
"SGLANG_TRACE_FORWARD_SPAN": "0",
}
base.update(env)
with mock.patch.dict(os.environ, base):
spec = importlib.util.spec_from_file_location(
"batch_telemetry_detail_under_test", _module_path()
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
class _FakeCounter:
def __init__(self):
self.data = {}
def labels(self, **kw):
self._key = tuple(sorted(kw.items()))
return self
def inc(self, value=1.0):
self.data[self._key] = self.data.get(self._key, 0.0) + value
def _fake_loop_metrics():
return {
"no_batch_seconds": _FakeCounter(),
"no_batch_total": _FakeCounter(),
"lockstep_seconds": _FakeCounter(),
}
class _FakeSpan:
def __init__(self, name, **kwargs):
self.name = name
self.kwargs = kwargs
self.attributes = dict(kwargs.get("attributes") or {})
self.end_time = None
def set_attributes(self, attrs):
self.attributes.update(attrs)
def get_span_context(self):
return types.SimpleNamespace(
is_valid=True, trace_id=1, span_id=2
)
def end(self, end_time=None):
self.end_time = end_time
class _FakeTracer:
def __init__(self):
self.spans = []
def start_span(self, name, **kwargs):
span = _FakeSpan(name, **kwargs)
self.spans.append(span)
return span
def _fake_otel():
tracer = _FakeTracer()
trace = types.ModuleType("opentelemetry.trace")
trace.get_tracer = lambda _name: tracer
trace.set_span_in_context = lambda span: ("ctx", span)
trace.Link = lambda ctx: ("link", ctx)
package = types.ModuleType("opentelemetry")
package.trace = trace
patcher = mock.patch.dict(
sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace}
)
return tracer, patcher
def _state(**overrides):
state = {
"waiting": 0,
"result_queue": 0,
"result_limit": 2,
"inflight": 0,
"bootstrap": 0,
"deferred": 0,
"paused": False,
"waiting_for_result": False,
"skip_batch_creation": False,
"forward_active": False,
}
state.update(overrides)
return state
def _span(tracer, name):
return next(s for s in tracer.spans if s.name == name)
class MooncakeChunkDetailTest(unittest.TestCase):
def _chunk(self, **over):
now_wall = time.time_ns()
chunk = types.SimpleNamespace(
room=17,
chunk_seq=3,
loop_seq=99,
send_kind="last",
is_last_chunk=True,
queue_id=5,
requeue_count=1,
num_kv_tokens=4096,
num_pages=32,
approx_bytes=1 << 20,
enqueue_wall_ns=now_wall - 40_000_000,
# Attempt-scoped start: after a defer this is the re-enqueue time.
last_enqueue_wall_ns=now_wall - 20_000_000,
last_enqueue_mono_ns=0,
dequeue_mono_ns=0,
trace_ctx=None,
)
for k, v in over.items():
setattr(chunk, k, v)
return chunk
def test_disabled_emits_no_span(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="0")
self.assertFalse(mod.mooncake_detail_enabled())
tracer, patcher = _fake_otel()
with patcher:
mod.emit_mooncake_chunk(
chunk=self._chunk(),
status="ok",
queue_wait_ns=1,
queue_wait_total_ns=1,
source_wait_ns=1,
indices_wait_ns=1,
kv_transfer_ns=1,
state_transfer_ns=1,
aux_transfer_ns=1,
attempt_wall_ns=10,
dequeue_wall_ns=time.time_ns(),
end_wall_ns=time.time_ns(),
n_destinations=1,
)
self.assertEqual(tracer.spans, [])
def test_chunk_breakdown_attributes(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
self.assertTrue(mod.mooncake_detail_enabled())
tracer, patcher = _fake_otel()
now_wall = time.time_ns()
dequeue_wall = now_wall - 16_000_000
source_wall = (now_wall - 15_000_000, now_wall - 13_000_000)
indices_wall = (now_wall - 12_000_000, now_wall - 11_000_000)
chunk = self._chunk(
last_enqueue_wall_ns=now_wall - 20_000_000,
queue_wait_accum_ns=5_000_000,
)
with patcher:
mod.emit_mooncake_chunk(
chunk=chunk,
status="ok",
queue_wait_ns=4_000_000,
queue_wait_total_ns=9_000_000,
source_wait_ns=2_000_000,
indices_wait_ns=1_000_000,
kv_transfer_ns=10_000_000,
state_transfer_ns=1_000_000,
aux_transfer_ns=1_000_000,
attempt_wall_ns=20_000_000,
dequeue_wall_ns=dequeue_wall,
end_wall_ns=now_wall,
n_destinations=2,
source_wait_wall=source_wall,
indices_wait_wall=indices_wall,
dp_rank=1,
tp_rank=0,
)
span = _span(tracer, "mooncake.kv_chunk")
attrs = span.attributes
self.assertEqual(attrs["sglang.bootstrap_room"], 17)
self.assertEqual(attrs["sglang.chunk_seq"], 3)
self.assertEqual(attrs["sglang.loop_seq"], 99)
self.assertEqual(attrs["sglang.send_kind"], "last")
self.assertEqual(attrs["sglang.queue_id"], 5)
# Attempt window matches the child span; the cumulative value is
# exposed separately (review finding).
self.assertAlmostEqual(attrs["sglang.queue_wait_ms"], 4.0)
self.assertAlmostEqual(attrs["sglang.queue_wait_total_ms"], 9.0)
self.assertAlmostEqual(attrs["sglang.kv_transfer_ms"], 10.0)
self.assertAlmostEqual(attrs["sglang.attempt_wall_ms"], 20.0)
# worker_busy excludes the source/indices event waits (design §5.3).
self.assertAlmostEqual(attrs["sglang.worker_busy_ms"], 17.0)
# 17 - (10 + 1 + 1) = 5 ms unexplained active worker time.
self.assertAlmostEqual(attrs["sglang.worker_residual_ms"], 5.0)
# Parent starts at the attempt's last enqueue, not the first one.
self.assertEqual(span.kwargs["start_time"], chunk.last_enqueue_wall_ns)
names = [s.name for s in tracer.spans]
self.assertIn("mooncake.queue_wait", names)
self.assertIn("mooncake.wait_source", names)
self.assertIn("mooncake.wait_indices", names)
queue_span = _span(tracer, "mooncake.queue_wait")
self.assertEqual(queue_span.kwargs["start_time"], chunk.last_enqueue_wall_ns)
self.assertEqual(queue_span.end_time, dequeue_wall)
self.assertAlmostEqual(
(queue_span.end_time - queue_span.kwargs["start_time"]) / 1e6, 4.0
)
self.assertAlmostEqual(queue_span.attributes["sglang.wait_ms"], 4.0)
source_span = _span(tracer, "mooncake.wait_source")
self.assertEqual(source_span.kwargs["start_time"], source_wall[0])
self.assertEqual(source_span.end_time, source_wall[1])
indices_span = _span(tracer, "mooncake.wait_indices")
self.assertEqual(indices_span.kwargs["start_time"], indices_wall[0])
self.assertEqual(indices_span.end_time, indices_wall[1])
def test_no_synthetic_wait_span_without_real_window(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
tracer, patcher = _fake_otel()
with patcher:
mod.emit_mooncake_chunk(
chunk=self._chunk(),
status="ok",
queue_wait_ns=0,
queue_wait_total_ns=0,
source_wait_ns=2_000_000,
indices_wait_ns=1_000_000,
kv_transfer_ns=1_000_000,
state_transfer_ns=0,
aux_transfer_ns=0,
attempt_wall_ns=10_000_000,
# queue wait has no valid window: last_enqueue == dequeue.
dequeue_wall_ns=time.time_ns(),
end_wall_ns=time.time_ns(),
n_destinations=1,
)
names = [s.name for s in tracer.spans]
self.assertNotIn("mooncake.queue_wait", names)
self.assertNotIn("mooncake.wait_source", names)
self.assertNotIn("mooncake.wait_indices", names)
attrs = _span(tracer, "mooncake.kv_chunk").attributes
self.assertAlmostEqual(attrs["sglang.source_wait_ms"], 2.0)
def test_queue_wait_split_matches_child_span(self):
"""Two defers then a third attempt: 20 us total + 10 us attempt."""
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
attempt_ns, total_ns = mod.split_queue_wait(20_000, 1_000_000, 1_010_000)
self.assertEqual(attempt_ns, 10_000)
self.assertEqual(total_ns, 30_000)
tracer, patcher = _fake_otel()
now = time.time_ns()
chunk = self._chunk(
last_enqueue_wall_ns=now - 10_000,
queue_wait_accum_ns=20_000,
)
with patcher:
mod.emit_mooncake_chunk(
chunk=chunk,
status="ok",
queue_wait_ns=attempt_ns,
queue_wait_total_ns=total_ns,
source_wait_ns=0,
indices_wait_ns=0,
kv_transfer_ns=1_000,
state_transfer_ns=0,
aux_transfer_ns=0,
attempt_wall_ns=2_000,
dequeue_wall_ns=now,
end_wall_ns=now,
n_destinations=1,
)
child = _span(tracer, "mooncake.queue_wait")
# Wall window == this attempt only; the accumulated value is an
# attribute on the parent, never on the child span.
self.assertEqual(
child.end_time - child.kwargs["start_time"], attempt_ns
)
self.assertAlmostEqual(
child.attributes["sglang.wait_ms"], attempt_ns / 1e6
)
attrs = _span(tracer, "mooncake.kv_chunk").attributes
self.assertAlmostEqual(attrs["sglang.queue_wait_ms"], attempt_ns / 1e6)
self.assertAlmostEqual(
attrs["sglang.queue_wait_total_ms"], total_ns / 1e6
)
def test_requeue_accumulates_only_queue_wait(self):
"""Two consecutive defers must not count worker/watermark time."""
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
chunk = types.SimpleNamespace(
observe=True,
queue_wait_accum_ns=0,
last_enqueue_mono_ns=1_000,
last_enqueue_wall_ns=2_000,
dequeue_mono_ns=11_000, # attempt 1: 10 ms queue wait
requeue_count=0,
)
# Worker + watermark consumed 30 ms; only the 10 ms queue wait counts.
mod.note_chunk_requeue(chunk, 41_000, 42_000)
self.assertEqual(chunk.queue_wait_accum_ns, 10_000)
self.assertEqual(chunk.requeue_count, 1)
self.assertEqual(chunk.last_enqueue_mono_ns, 41_000)
self.assertEqual(chunk.last_enqueue_wall_ns, 42_000)
self.assertEqual(chunk.dequeue_mono_ns, 0)
# Attempt 2: dequeued 10 ms later, then another defer.
chunk.dequeue_mono_ns = 51_000
mod.note_chunk_requeue(chunk, 91_000, 92_000)
self.assertEqual(chunk.queue_wait_accum_ns, 20_000)
self.assertEqual(chunk.requeue_count, 2)
def test_requeue_disabled_is_noop(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="0")
chunk = types.SimpleNamespace(
observe=True,
queue_wait_accum_ns=0,
last_enqueue_mono_ns=1_000,
last_enqueue_wall_ns=2_000,
dequeue_mono_ns=11_000,
requeue_count=0,
)
mod.note_chunk_requeue(chunk, 41_000, 42_000)
self.assertEqual(chunk.queue_wait_accum_ns, 0)
self.assertEqual(chunk.requeue_count, 0)
class RequestJoinSpanTest(unittest.TestCase):
def test_request_join_span(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
tracer, patcher = _fake_otel()
now_wall = time.time_ns()
req = types.SimpleNamespace(rid="req-1", bootstrap_room=42)
stats = {
"chunk_count": 4,
"first_enqueue_wall_ns": now_wall - 600_000_000,
"last_chunk_enqueue_wall_ns": now_wall - 30_000_000,
"last_chunk_complete_wall_ns": now_wall - 10_000_000,
"last_chunk_queue_wait_ms": 2.5,
"last_chunk_worker_ms": 8.0,
}
with patcher:
mod.emit_request_kv_join(
req=req, stats=stats, dp_rank=3, completion_wall_ns=now_wall
)
span = _span(tracer, "kv.request_join")
attrs = span.attributes
self.assertEqual(attrs["sglang.bootstrap_room"], 42)
self.assertEqual(attrs["sglang.chunk_count"], 4)
self.assertAlmostEqual(attrs["sglang.final_exposed_ms"], 20.0)
self.assertAlmostEqual(attrs["sglang.completion_poll_ms"], 10.0)
self.assertAlmostEqual(attrs["sglang.request_join_ms"], 30.0)
self.assertAlmostEqual(
attrs["sglang.first_enqueue_to_completion_ms"], 600.0, delta=1.0
)
self.assertEqual(
span.kwargs["start_time"], stats["first_enqueue_wall_ns"]
)
self.assertEqual(len(span.kwargs.get("links") or []), 0)
class RoomDetailStateTest(unittest.TestCase):
def test_begin_chunk_is_sequential_and_keeps_first_enqueue(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
state = mod.RoomDetailState()
seq0, first0 = state.begin_chunk(7, 1_000, 2_000)
seq1, first1 = state.begin_chunk(7, 5_000, 6_000)
self.assertEqual((seq0, seq1), (0, 1))
self.assertEqual(first0, 2_000)
self.assertEqual(first1, 2_000) # first enqueue of the room
def test_abort_between_status_check_and_join_write_cannot_resurrect(self):
"""Deterministic interleaving of the abort/worker race (review)."""
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
state = mod.RoomDetailState()
state.begin_chunk(7, 1_000, 2_000)
# Worker saw status ok, then the scheduler aborts before the write.
state.reset(7)
written = state.record_join(
7,
status="ok",
room_failed=False, # stale status read from before the abort
now_wall_ns=3_000,
queue_wait_total_ns=10,
worker_busy_ns=20,
enqueue_wall_ns=2_000,
is_last_chunk=True,
chunk_seq=0,
)
self.assertFalse(written)
self.assertIsNone(state.pop_join(7))
def test_join_write_then_abort_is_cleaned(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
state = mod.RoomDetailState()
state.begin_chunk(7, 1_000, 2_000)
self.assertTrue(
state.record_join(
7,
status="ok",
room_failed=False,
now_wall_ns=3_000,
queue_wait_total_ns=10,
worker_busy_ns=20,
enqueue_wall_ns=2_000,
is_last_chunk=True,
chunk_seq=0,
)
)
state.reset(7)
self.assertIsNone(state.pop_join(7))
def test_failed_worker_never_writes_join(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
state = mod.RoomDetailState()
state.begin_chunk(7, 1_000, 2_000)
self.assertFalse(
state.record_join(
7,
status="failed",
room_failed=True,
now_wall_ns=3_000,
queue_wait_total_ns=0,
worker_busy_ns=0,
enqueue_wall_ns=2_000,
is_last_chunk=True,
chunk_seq=0,
)
)
self.assertIsNone(state.pop_join(7))
def test_retry_clears_cancellation_mark(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
state = mod.RoomDetailState()
state.begin_chunk(7, 1_000, 2_000)
state.reset(7)
# A retry that reuses the room starts a fresh flow.
state.begin_chunk(7, 10_000, 11_000)
self.assertTrue(
state.record_join(
7,
status="ok",
room_failed=False,
now_wall_ns=12_000,
queue_wait_total_ns=0,
worker_busy_ns=1,
enqueue_wall_ns=11_000,
is_last_chunk=False,
chunk_seq=0,
)
)
self.assertEqual(state.pop_join(7)["chunk_count"], 1)
def test_cancellation_marks_are_bounded(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
state = mod.RoomDetailState()
for room in range(mod.RoomDetailState._MAX_CANCELLED + 10):
state.reset(room)
self.assertLessEqual(
len(state._cancelled), mod.RoomDetailState._MAX_CANCELLED
)
def test_teardown_keeps_success_join_and_drops_failed(self):
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
state = mod.RoomDetailState()
state.begin_chunk(1, 1_000, 2_000)
state.record_join(
1,
status="ok",
room_failed=False,
now_wall_ns=3_000,
queue_wait_total_ns=0,
worker_busy_ns=1,
enqueue_wall_ns=2_000,
is_last_chunk=False,
chunk_seq=0,
)
state.teardown(1, keep_join=True)
self.assertIsNotNone(state.pop_join(1))
state.begin_chunk(2, 1_000, 2_000)
state.record_join(
2,
status="ok",
room_failed=False,
now_wall_ns=3_000,
queue_wait_total_ns=0,
worker_busy_ns=1,
enqueue_wall_ns=2_000,
is_last_chunk=False,
chunk_seq=0,
)
state.teardown(2, keep_join=False)
self.assertIsNone(state.pop_join(2))
class ChunkFieldDefaultsTest(unittest.TestCase):
"""The observation fields must be additive defaults on TransferKVChunk."""
def test_dataclass_defaults(self):
import dataclasses
path = (
pathlib.Path(__file__).resolve().parents[4]
/ "python/sglang/srt/disaggregation/common/utils.py"
)
trace_stub = types.ModuleType("sglang.srt.observability.trace")
class _Null:
pass
trace_stub.TraceNullContext = _Null
trace_stub.TraceReqContext = _Null
patcher = mock.patch.dict(
sys.modules,
{
"sglang": types.ModuleType("sglang"),
"sglang.srt": types.ModuleType("sglang.srt"),
"sglang.srt.observability": types.ModuleType(
"sglang.srt.observability"
),
"sglang.srt.observability.trace": trace_stub,
},
)
with patcher:
spec = importlib.util.spec_from_file_location("kv_utils_under_test", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
fields = {f.name: f for f in dataclasses.fields(mod.TransferKVChunk)}
for name in (
"chunk_seq",
"loop_seq",
"send_kind",
"enqueue_mono_ns",
"enqueue_wall_ns",
"last_enqueue_mono_ns",
"last_enqueue_wall_ns",
"dequeue_mono_ns",
"queue_wait_accum_ns",
"queue_id",
"requeue_count",
"num_pages",
"approx_bytes",
"observe",
):
self.assertIn(name, fields)
self.assertFalse(fields[name].init is None)
chunk = mod.TransferKVChunk(
room=1,
prefill_kv_indices=None,
index_slice=slice(0, 1),
is_last_chunk=True,
prefill_aux_index=None,
state_indices=None,
chunk_seq=3,
queue_id=2,
observe=True,
)
self.assertEqual(chunk.chunk_seq, 3)
self.assertTrue(chunk.observe)
self.assertEqual(chunk.requeue_count, 0)
if __name__ == "__main__":
unittest.main()