observability: trace KV transfer chunks end-to-end (M2)

Implements M2 of handoff/prefill-cpu-kv-overlap-design-20261106.md, behind
SGLANG_TRACE_MOONCAKE_DETAIL=1 (default off; reuses the existing trace stack).

- TransferKVChunk gains optional observation fields: chunk_seq, loop_seq,
  send_kind, enqueue/last_enqueue monotonic+wall stamps, queue_wait_accum_ns,
  queue_id, requeue_count, num_pages, approx_bytes, observe
- chunk_seq and enqueue timestamps are assigned at the real queue.put()
  point (after every early return), so skipped paths cannot leave holes
- Mooncake transfer worker measures queue wait, source-event wait, indices
  wait, KV/state/aux transfer, worker busy and residual; emits one
  mooncake.kv_chunk span per attempt (+ queue/source/indices wait children)
  and updates a per-room join summary
- request completion emits kv.request_join with chunk_count,
  last_chunk_queue_wait/worker, final_exposed_ms, completion_poll_ms,
  request_join_ms and first-enqueue-to-completion. First enqueue is stamped
  by the Mooncake manager at the real queue.put(); no ReqTimeStats field is
  added and the legacy queue_entry semantics are unchanged.
- nixl/mori add_transfer_request accept the observation kwarg (no-op there)

No scheduling/transfer semantics change; queue wait is excluded from worker
busy, and worker_busy excludes source/indices event waits. Unit tests: test_mooncake_chunk_detail.py (15 cases, including the
abort/worker interleaving and attempt-vs-total queue wait windows).
This commit is contained in:
ymk
2026-11-06 20:01:56 +08:00
parent 72e556ec81
commit fe9c6574fe
7 changed files with 884 additions and 0 deletions
@@ -51,6 +51,7 @@ from sglang.srt.disaggregation.utils import (
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.trace import TraceNullContext, TraceReqContext
from sglang.srt.runtime_context import (
get_device,
@@ -2091,6 +2092,10 @@ class CommonKVSender(BaseKVSender):
self._transfer_num_state_indices = 0
self._transfer_start_time: Optional[float] = None
self._early_send_wait_event: Optional[object] = None
# Detail tracing: chunk classification only. Enqueue timestamps,
# chunk_seq and the per-request first-enqueue are assigned by the
# manager at the real queue.put() point, after all early returns.
self._send_kind = ""
self.trace_ctx = TraceNullContext()
if self.kv_mgr.sender_trace_module and get_observability().enable_trace:
self.trace_ctx = TraceReqContext(
@@ -2289,6 +2294,19 @@ class CommonKVSender(BaseKVSender):
):
self._transfer_start_time = time.perf_counter()
obs = None
if batch_telemetry.mooncake_detail_enabled():
# Metadata only; the manager stamps chunk_seq and enqueue times
# when it is certain the chunk is actually put on the queue.
obs = {
"loop_seq": int(batch_telemetry.current_loop_seq()),
"send_kind": str(self._send_kind or ""),
"num_pages": int(len(kv_indices)),
"approx_bytes": int(
len(kv_indices) * getattr(self.kv_mgr, "kv_item_lens_sum", 0)
),
}
# The early-send event belongs to exactly one chunk; the Mooncake and
# Mori workers synchronize on it before reading source KV.
wait_event = self._early_send_wait_event
@@ -2304,9 +2322,27 @@ class CommonKVSender(BaseKVSender):
wait_event=wait_event,
indices_event=indices_event,
trace_ctx=self.trace_ctx.copy_for_thread(),
obs=obs,
)
self._record_transfer_indices(kv_indices, state_indices)
def pop_chunk_join_stats(self) -> Optional[dict]:
"""Take (and free) this room's chunk join summary."""
state = getattr(self.kv_mgr, "_room_detail", None)
if state is None:
return None
return state.pop_join(self.bootstrap_room)
def reset_chunk_detail_state(self) -> None:
"""Drop per-room detail state and mark it cancelled (abort / retract).
The mark is what prevents a worker attempt that already passed its
status check from recreating stats after this cleanup.
"""
state = getattr(self.kv_mgr, "_room_detail", None)
if state is not None:
state.reset(self.bootstrap_room)
def poll(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status)
@@ -2425,6 +2461,9 @@ class CommonKVSender(BaseKVSender):
def abort(self):
if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
return
# Drop per-room detail bookkeeping: an aborted request never reaches
# the poll path that normally consumes it (review finding).
self.reset_chunk_detail_state()
self.kv_mgr.record_failure(
self.bootstrap_room,
"Aborted by AbortReq.",
@@ -36,6 +36,24 @@ class TransferKVChunk:
# Must complete before reading asynchronously copied host indices.
indices_event: Optional[object] = None
# --- Optional observation fields (SGLANG_TRACE_MOONCAKE_DETAIL=1) ------
# Filled at the real enqueue point (add_transfer_request), never at the
# send_kv_chunk entry, so skipped paths cannot create holes in chunk_seq.
chunk_seq: int = 0 # per-sender monotonic enqueue index
loop_seq: int = 0 # scheduler loop that produced this chunk
send_kind: str = "" # cached_prefix / middle / last
enqueue_mono_ns: int = 0 # first enqueue of this chunk, monotonic clock
enqueue_wall_ns: int = 0 # first enqueue of this chunk, wall clock
last_enqueue_mono_ns: int = 0 # updated on every (re-)enqueue
last_enqueue_wall_ns: int = 0 # wall clock of the last (re-)enqueue
dequeue_mono_ns: int = 0 # current attempt dequeue; 0 outside the worker
queue_wait_accum_ns: int = 0 # completed queue waits from earlier attempts
queue_id: int = -1
requeue_count: int = 0
num_pages: int = 0
approx_bytes: int = 0
observe: bool = False # True when detail tracing populated this chunk
def pack_list_of_buffers(buffers: List[bytes]) -> bytes:
if not buffers:
@@ -60,6 +60,7 @@ from sglang.srt.disaggregation.utils import (
)
from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine
from sglang.srt.environ import envs
from sglang.srt.observability import batch_telemetry
from sglang.srt.observability.mooncake_trace import (
MooncakeRequestStage,
mooncake_trace_func,
@@ -386,6 +387,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# transfer blocks over zmq when the intra-node NVLink transport is
# active (it cannot reach host memory). Populated on bootstrap.
self._session_endpoint_map = {}
# Detail tracing (SGLANG_TRACE_MOONCAKE_DETAIL=1; empty otherwise).
# Lock-protected per-room state: abort/retract and the worker's join
# update must not interleave into resurrected stats.
self._room_detail = batch_telemetry.RoomDetailState()
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
self.max_transfer_batch_indices = (
envs.SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES.get()
@@ -762,6 +767,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# block other rooms on this single worker thread.
with self._staging_ctx.watermark_cv:
self._staging_ctx.watermark_cv.wait(STAGING_WATERMARK_WAIT_S)
if getattr(kv_chunk, "observe", False):
batch_telemetry.note_chunk_requeue(
kv_chunk, time.monotonic_ns(), time.time_ns()
)
queue.put(kv_chunk)
return (-1, True)
@@ -2716,6 +2725,65 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if self.enable_deferred_decode_kv_release:
self.poison_deferred_ack_room(room)
def _emit_chunk_attempt(
self,
kv_chunk: TransferKVChunk,
*,
status: str,
queue_wait_ns: int,
queue_wait_total_ns: int,
source_wait_ns: int,
indices_wait_ns: int,
kv_transfer_ns: int,
state_transfer_ns: int,
aux_transfer_ns: int,
work_start_ns: int,
dequeue_wall_ns: int,
n_destinations: int,
source_wait_wall=None,
indices_wait_wall=None,
) -> None:
"""Close one worker attempt: chunk span + per-room join summary."""
now_mono_ns = time.monotonic_ns()
now_wall_ns = time.time_ns()
attempt_wall_ns = max(0, now_mono_ns - int(work_start_ns))
worker_busy_ns = max(
0, attempt_wall_ns - int(source_wait_ns) - int(indices_wait_ns)
)
batch_telemetry.emit_mooncake_chunk(
chunk=kv_chunk,
status=status,
queue_wait_ns=queue_wait_ns,
queue_wait_total_ns=queue_wait_total_ns,
source_wait_ns=source_wait_ns,
indices_wait_ns=indices_wait_ns,
kv_transfer_ns=kv_transfer_ns,
state_transfer_ns=state_transfer_ns,
aux_transfer_ns=aux_transfer_ns,
attempt_wall_ns=attempt_wall_ns,
dequeue_wall_ns=dequeue_wall_ns,
end_wall_ns=now_wall_ns,
n_destinations=n_destinations,
source_wait_wall=source_wait_wall,
indices_wait_wall=indices_wait_wall,
dp_rank=self.attn_dp_rank,
tp_rank=self.attn_tp_rank,
)
# The status was checked before this call; the registry re-checks the
# cancellation mark under its lock, which is what makes an abort that
# lands in between unable to resurrect stats (review finding).
self._room_detail.record_join(
kv_chunk.room,
status=status,
room_failed=self.check_status(kv_chunk.room) == KVPoll.Failed,
now_wall_ns=now_wall_ns,
queue_wait_total_ns=queue_wait_total_ns,
worker_busy_ns=worker_busy_ns,
enqueue_wall_ns=kv_chunk.enqueue_wall_ns,
is_last_chunk=kv_chunk.is_last_chunk,
chunk_seq=kv_chunk.chunk_seq,
)
def transfer_worker(
self,
queue: FastQueue,
@@ -2740,6 +2808,34 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# _stopped alone can never wake a parked worker during a role switch.
if kv_chunk is None:
break
_obs = bool(getattr(kv_chunk, "observe", False))
_dequeue_mono_ns = time.monotonic_ns() if _obs else 0
_dequeue_wall_ns = time.time_ns() if _obs else 0
if _obs:
# Needed by note_chunk_requeue(): queue wait ends here and
# must not include worker/watermark time.
kv_chunk.dequeue_mono_ns = _dequeue_mono_ns
_queue_wait_ns = 0
_source_wait_ns = 0
_source_wait_wall = None
_indices_wait_ns = 0
_indices_wait_wall = None
_kv_transfer_ns = 0
_state_transfer_ns = 0
_aux_transfer_ns = 0
_n_dest = 0
if _obs:
# queue wait is not part of worker busy time (design §5.2).
# Keep the attempt window (child span) separate from the
# cumulative value across deferred attempts (review finding).
(
_attempt_queue_wait_ns,
_queue_wait_ns,
) = batch_telemetry.split_queue_wait(
kv_chunk.queue_wait_accum_ns,
kv_chunk.last_enqueue_mono_ns,
_dequeue_mono_ns,
)
if self.enable_trace:
kv_chunk.trace_ctx.rebuild_thread_context()
kv_chunk.trace_ctx.trace_slice_start(
@@ -2765,13 +2861,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
kv_chunk.room in self.request_status
and self.check_status(kv_chunk.room) != KVPoll.Failed
):
_wait_t0 = time.monotonic_ns() if _obs else 0
_wait_wall0 = time.time_ns() if _obs else 0
kv_chunk.wait_event.synchronize()
if _wait_t0:
_source_wait_ns += time.monotonic_ns() - _wait_t0
_wait_wall1 = time.time_ns()
_source_wait_wall = (_wait_wall0, _wait_wall1)
kv_chunk.wait_event = None
_ind_t0 = time.monotonic_ns() if _obs else 0
_ind_wall0 = time.time_ns() if _obs else 0
if kv_chunk.indices_event is not None:
kv_chunk.indices_event.synchronize()
kv_chunk.indices_event = None
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
if _ind_t0:
# synchronize + host indices availability, per design §5.2.
_indices_wait_ns += time.monotonic_ns() - _ind_t0
_indices_wait_wall = (_ind_wall0, time.time_ns())
if (
kv_chunk.room not in self.request_status
@@ -2787,6 +2895,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
thread_finish_flag=True,
)
self._staging_outstanding.pop(kv_chunk.room, None)
if _obs:
self._emit_chunk_attempt(
kv_chunk,
status="skipped",
queue_wait_ns=_attempt_queue_wait_ns,
queue_wait_total_ns=_queue_wait_ns,
source_wait_ns=_source_wait_ns,
indices_wait_ns=_indices_wait_ns,
kv_transfer_ns=_kv_transfer_ns,
state_transfer_ns=_state_transfer_ns,
aux_transfer_ns=_aux_transfer_ns,
work_start_ns=_dequeue_mono_ns,
dequeue_wall_ns=_dequeue_wall_ns,
n_destinations=_n_dest,
source_wait_wall=_source_wait_wall,
indices_wait_wall=_indices_wait_wall,
)
if self.enable_deferred_decode_kv_release:
# Skipped => nothing written for this aborted room; ack.
self._maybe_ack_drained_abort(kv_chunk.room)
@@ -2882,6 +3007,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
skip_kv, skip_state = self._get_dsa_cache_transfer_skip_flags(
target_rank_registration_info
)
_kv_t0 = time.monotonic_ns() if _obs else 0
if (
len(kv_chunk.prefill_kv_indices) == 0
or not self.kv_args.kv_data_ptrs
@@ -2954,6 +3080,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
if deferred:
staging_deferred = True
if _kv_t0:
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
_n_dest += 1
# Chunk re-enqueued; stop processing remaining reqs for this chunk
break
else:
@@ -2968,6 +3097,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
executor,
target_rank_registration_info.dst_kv_layer_ids,
)
if _kv_t0:
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
_n_dest += 1
if ret != 0:
with self.session_lock:
self.session_failures[req.mooncake_session_id] += 1
@@ -2989,12 +3121,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if kv_chunk.is_last_chunk:
if kv_chunk.state_indices and not skip_state:
_state_t0 = time.monotonic_ns() if _obs else 0
state_rc = self.maybe_send_extra(
req,
kv_chunk.state_indices,
executor,
target_rank_registration_info,
)
if _state_t0:
_state_transfer_ns += time.monotonic_ns() - _state_t0
# Before the Failed check: an ABORT during the
# send also marks the room Failed.
if state_rc != 0:
@@ -3025,11 +3160,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
break
# Only the last chunk we need to send the aux data
_aux_t0 = time.monotonic_ns() if _obs else 0
ret = self.send_aux(
req,
kv_chunk.prefill_aux_index,
target_rank_registration_info.dst_aux_ptrs,
)
if _aux_t0:
_aux_transfer_ns += time.monotonic_ns() - _aux_t0
if ret != 0:
# The aux write targets decode's metadata
# slot, which the deferred release also frees.
@@ -3071,8 +3209,45 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
if staging_deferred:
if _obs:
self._emit_chunk_attempt(
kv_chunk,
status="deferred",
queue_wait_ns=_attempt_queue_wait_ns,
queue_wait_total_ns=_queue_wait_ns,
source_wait_ns=_source_wait_ns,
indices_wait_ns=_indices_wait_ns,
kv_transfer_ns=_kv_transfer_ns,
state_transfer_ns=_state_transfer_ns,
aux_transfer_ns=_aux_transfer_ns,
work_start_ns=_dequeue_mono_ns,
dequeue_wall_ns=_dequeue_wall_ns,
n_destinations=_n_dest,
source_wait_wall=_source_wait_wall,
indices_wait_wall=_indices_wait_wall,
)
continue
if _obs:
self._emit_chunk_attempt(
kv_chunk,
status="ok"
if self.check_status(kv_chunk.room) != KVPoll.Failed
else "failed",
queue_wait_ns=_attempt_queue_wait_ns,
queue_wait_total_ns=_queue_wait_ns,
source_wait_ns=_source_wait_ns,
indices_wait_ns=_indices_wait_ns,
kv_transfer_ns=_kv_transfer_ns,
state_transfer_ns=_state_transfer_ns,
aux_transfer_ns=_aux_transfer_ns,
work_start_ns=_dequeue_mono_ns,
dequeue_wall_ns=_dequeue_wall_ns,
n_destinations=_n_dest,
source_wait_wall=_source_wait_wall,
indices_wait_wall=_indices_wait_wall,
)
self._staging_outstanding[kv_chunk.room] -= 1
if self.enable_deferred_decode_kv_release:
# In-flight write finished; if aborted and nothing outstanding,
@@ -3096,6 +3271,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self._session_endpoint_map.pop(sid, None)
self.transfer_infos.pop(kv_chunk.room)
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
# Success: the scheduler poll still needs the join summary.
# Failed/absent: nothing will consume it, drop it here.
self._room_detail.teardown(
kv_chunk.room,
keep_join=(
self.request_status.get(kv_chunk.room)
== KVPoll.Success
),
)
if self.enable_staging:
# Purge prefetch bookkeeping for the finished room.
# Snapshot first: the scheduler thread adds concurrently.
@@ -3332,6 +3516,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
wait_event: Optional[object] = None,
indices_event: Optional[object] = None,
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
obs: Optional[dict] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -3361,6 +3546,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if trace_ctx is None:
trace_ctx = TraceNullContext()
if obs is not None:
# Stamp only now: every earlier return (failed room, dummy rank,
# zero-page skip) must not consume a chunk_seq or a first-enqueue
# timestamp (review finding).
now_mono_ns = time.monotonic_ns()
now_wall_ns = time.time_ns()
chunk_seq, _first_wall_ns = self._room_detail.begin_chunk(
bootstrap_room, now_mono_ns, now_wall_ns
)
obs = dict(obs)
obs.update(
chunk_seq=chunk_seq,
enqueue_mono_ns=now_mono_ns,
enqueue_wall_ns=now_wall_ns,
last_enqueue_wall_ns=now_wall_ns,
)
self.transfer_queues[shard_idx].put(
TransferKVChunk(
room=bootstrap_room,
@@ -3373,6 +3575,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
trace_ctx=trace_ctx,
wait_event=wait_event,
indices_event=indices_event,
**(obs or {}),
queue_id=int(shard_idx),
observe=obs is not None,
last_enqueue_mono_ns=int((obs or {}).get("enqueue_mono_ns", 0)),
)
)
@@ -804,6 +804,7 @@ class MoriKVManager(CommonKVManager):
wait_event: Optional[object] = None,
indices_event: Optional[object] = None,
trace_ctx=None,
obs: Optional[dict] = None,
) -> None:
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -3267,6 +3267,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
wait_event=None,
indices_event=None,
trace_ctx=None,
obs: Optional[dict] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -1426,11 +1426,30 @@ class SchedulerDisaggregationPrefillMixin:
self.tree_cache.finish(
req.cache_request_handle, CacheRequestOutcome.SUCCESS
)
# Consume the join summary before clear() releases the room.
_join_stats = None
if batch_telemetry.mooncake_detail_enabled():
_pop_stats = getattr(
req.disagg_kv_sender, "pop_chunk_join_stats", None
)
_join_stats = _pop_stats() if _pop_stats is not None else None
# FIXME: clean up req's data in transfer engine
req.disagg_kv_sender.clear()
done_reqs.append(req)
req.time_stats.set_prefill_kv_transfer_finish_time()
if batch_telemetry.mooncake_detail_enabled():
batch_telemetry.emit_request_kv_join(
req=req,
stats=_join_stats,
dp_rank=get_parallel().attn_dp_rank,
)
elif poll == KVPoll.Failed:
if batch_telemetry.mooncake_detail_enabled():
_pop_stats = getattr(
req.disagg_kv_sender, "pop_chunk_join_stats", None
)
if _pop_stats is not None:
_pop_stats()
self.handle_inflight_transfer_failure(req)
done_reqs.append(req)
else:
@@ -2016,6 +2035,12 @@ class SchedulerDisaggregationPrefillMixin:
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
"""Release KV cache and requeue an optimistic prefill request."""
if batch_telemetry.mooncake_detail_enabled():
# A retry may reuse the same room; stale chunk stats would
# otherwise be merged into the next attempt's join summary.
_reset = getattr(req.disagg_kv_sender, "reset_chunk_detail_state", None)
if _reset is not None:
_reset()
max_attempts = get_disagg().optimistic_prefill_attempts
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
if not uses_write_through_cache: