observability: trace KV transfer chunks end-to-end (M2)
Implements M2 of handoff/prefill-cpu-kv-overlap-design-20261106.md, behind SGLANG_TRACE_MOONCAKE_DETAIL=1 (default off; reuses the existing trace stack). - TransferKVChunk gains optional observation fields: chunk_seq, loop_seq, send_kind, enqueue/last_enqueue monotonic+wall stamps, queue_wait_accum_ns, queue_id, requeue_count, num_pages, approx_bytes, observe - chunk_seq and enqueue timestamps are assigned at the real queue.put() point (after every early return), so skipped paths cannot leave holes - Mooncake transfer worker measures queue wait, source-event wait, indices wait, KV/state/aux transfer, worker busy and residual; emits one mooncake.kv_chunk span per attempt (+ queue/source/indices wait children) and updates a per-room join summary - request completion emits kv.request_join with chunk_count, last_chunk_queue_wait/worker, final_exposed_ms, completion_poll_ms, request_join_ms and first-enqueue-to-completion. First enqueue is stamped by the Mooncake manager at the real queue.put(); no ReqTimeStats field is added and the legacy queue_entry semantics are unchanged. - nixl/mori add_transfer_request accept the observation kwarg (no-op there) No scheduling/transfer semantics change; queue wait is excluded from worker busy, and worker_busy excludes source/indices event waits. Unit tests: test_mooncake_chunk_detail.py (15 cases, including the abort/worker interleaving and attempt-vs-total queue wait windows).
This commit is contained in:
@@ -51,6 +51,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.trace import TraceNullContext, TraceReqContext
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
@@ -2091,6 +2092,10 @@ class CommonKVSender(BaseKVSender):
|
||||
self._transfer_num_state_indices = 0
|
||||
self._transfer_start_time: Optional[float] = None
|
||||
self._early_send_wait_event: Optional[object] = None
|
||||
# Detail tracing: chunk classification only. Enqueue timestamps,
|
||||
# chunk_seq and the per-request first-enqueue are assigned by the
|
||||
# manager at the real queue.put() point, after all early returns.
|
||||
self._send_kind = ""
|
||||
self.trace_ctx = TraceNullContext()
|
||||
if self.kv_mgr.sender_trace_module and get_observability().enable_trace:
|
||||
self.trace_ctx = TraceReqContext(
|
||||
@@ -2289,6 +2294,19 @@ class CommonKVSender(BaseKVSender):
|
||||
):
|
||||
self._transfer_start_time = time.perf_counter()
|
||||
|
||||
obs = None
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
# Metadata only; the manager stamps chunk_seq and enqueue times
|
||||
# when it is certain the chunk is actually put on the queue.
|
||||
obs = {
|
||||
"loop_seq": int(batch_telemetry.current_loop_seq()),
|
||||
"send_kind": str(self._send_kind or ""),
|
||||
"num_pages": int(len(kv_indices)),
|
||||
"approx_bytes": int(
|
||||
len(kv_indices) * getattr(self.kv_mgr, "kv_item_lens_sum", 0)
|
||||
),
|
||||
}
|
||||
|
||||
# The early-send event belongs to exactly one chunk; the Mooncake and
|
||||
# Mori workers synchronize on it before reading source KV.
|
||||
wait_event = self._early_send_wait_event
|
||||
@@ -2304,9 +2322,27 @@ class CommonKVSender(BaseKVSender):
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
obs=obs,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
def pop_chunk_join_stats(self) -> Optional[dict]:
|
||||
"""Take (and free) this room's chunk join summary."""
|
||||
state = getattr(self.kv_mgr, "_room_detail", None)
|
||||
if state is None:
|
||||
return None
|
||||
return state.pop_join(self.bootstrap_room)
|
||||
|
||||
def reset_chunk_detail_state(self) -> None:
|
||||
"""Drop per-room detail state and mark it cancelled (abort / retract).
|
||||
|
||||
The mark is what prevents a worker attempt that already passed its
|
||||
status check from recreating stats after this cleanup.
|
||||
"""
|
||||
state = getattr(self.kv_mgr, "_room_detail", None)
|
||||
if state is not None:
|
||||
state.reset(self.bootstrap_room)
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
@@ -2425,6 +2461,9 @@ class CommonKVSender(BaseKVSender):
|
||||
def abort(self):
|
||||
if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
|
||||
return
|
||||
# Drop per-room detail bookkeeping: an aborted request never reaches
|
||||
# the poll path that normally consumes it (review finding).
|
||||
self.reset_chunk_detail_state()
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
"Aborted by AbortReq.",
|
||||
|
||||
@@ -36,6 +36,24 @@ class TransferKVChunk:
|
||||
# Must complete before reading asynchronously copied host indices.
|
||||
indices_event: Optional[object] = None
|
||||
|
||||
# --- Optional observation fields (SGLANG_TRACE_MOONCAKE_DETAIL=1) ------
|
||||
# Filled at the real enqueue point (add_transfer_request), never at the
|
||||
# send_kv_chunk entry, so skipped paths cannot create holes in chunk_seq.
|
||||
chunk_seq: int = 0 # per-sender monotonic enqueue index
|
||||
loop_seq: int = 0 # scheduler loop that produced this chunk
|
||||
send_kind: str = "" # cached_prefix / middle / last
|
||||
enqueue_mono_ns: int = 0 # first enqueue of this chunk, monotonic clock
|
||||
enqueue_wall_ns: int = 0 # first enqueue of this chunk, wall clock
|
||||
last_enqueue_mono_ns: int = 0 # updated on every (re-)enqueue
|
||||
last_enqueue_wall_ns: int = 0 # wall clock of the last (re-)enqueue
|
||||
dequeue_mono_ns: int = 0 # current attempt dequeue; 0 outside the worker
|
||||
queue_wait_accum_ns: int = 0 # completed queue waits from earlier attempts
|
||||
queue_id: int = -1
|
||||
requeue_count: int = 0
|
||||
num_pages: int = 0
|
||||
approx_bytes: int = 0
|
||||
observe: bool = False # True when detail tracing populated this chunk
|
||||
|
||||
|
||||
def pack_list_of_buffers(buffers: List[bytes]) -> bytes:
|
||||
if not buffers:
|
||||
|
||||
@@ -60,6 +60,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.mooncake_trace import (
|
||||
MooncakeRequestStage,
|
||||
mooncake_trace_func,
|
||||
@@ -386,6 +387,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# transfer blocks over zmq when the intra-node NVLink transport is
|
||||
# active (it cannot reach host memory). Populated on bootstrap.
|
||||
self._session_endpoint_map = {}
|
||||
# Detail tracing (SGLANG_TRACE_MOONCAKE_DETAIL=1; empty otherwise).
|
||||
# Lock-protected per-room state: abort/retract and the worker's join
|
||||
# update must not interleave into resurrected stats.
|
||||
self._room_detail = batch_telemetry.RoomDetailState()
|
||||
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
|
||||
self.max_transfer_batch_indices = (
|
||||
envs.SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES.get()
|
||||
@@ -762,6 +767,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# block other rooms on this single worker thread.
|
||||
with self._staging_ctx.watermark_cv:
|
||||
self._staging_ctx.watermark_cv.wait(STAGING_WATERMARK_WAIT_S)
|
||||
if getattr(kv_chunk, "observe", False):
|
||||
batch_telemetry.note_chunk_requeue(
|
||||
kv_chunk, time.monotonic_ns(), time.time_ns()
|
||||
)
|
||||
queue.put(kv_chunk)
|
||||
return (-1, True)
|
||||
|
||||
@@ -2716,6 +2725,65 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if self.enable_deferred_decode_kv_release:
|
||||
self.poison_deferred_ack_room(room)
|
||||
|
||||
def _emit_chunk_attempt(
|
||||
self,
|
||||
kv_chunk: TransferKVChunk,
|
||||
*,
|
||||
status: str,
|
||||
queue_wait_ns: int,
|
||||
queue_wait_total_ns: int,
|
||||
source_wait_ns: int,
|
||||
indices_wait_ns: int,
|
||||
kv_transfer_ns: int,
|
||||
state_transfer_ns: int,
|
||||
aux_transfer_ns: int,
|
||||
work_start_ns: int,
|
||||
dequeue_wall_ns: int,
|
||||
n_destinations: int,
|
||||
source_wait_wall=None,
|
||||
indices_wait_wall=None,
|
||||
) -> None:
|
||||
"""Close one worker attempt: chunk span + per-room join summary."""
|
||||
now_mono_ns = time.monotonic_ns()
|
||||
now_wall_ns = time.time_ns()
|
||||
attempt_wall_ns = max(0, now_mono_ns - int(work_start_ns))
|
||||
worker_busy_ns = max(
|
||||
0, attempt_wall_ns - int(source_wait_ns) - int(indices_wait_ns)
|
||||
)
|
||||
batch_telemetry.emit_mooncake_chunk(
|
||||
chunk=kv_chunk,
|
||||
status=status,
|
||||
queue_wait_ns=queue_wait_ns,
|
||||
queue_wait_total_ns=queue_wait_total_ns,
|
||||
source_wait_ns=source_wait_ns,
|
||||
indices_wait_ns=indices_wait_ns,
|
||||
kv_transfer_ns=kv_transfer_ns,
|
||||
state_transfer_ns=state_transfer_ns,
|
||||
aux_transfer_ns=aux_transfer_ns,
|
||||
attempt_wall_ns=attempt_wall_ns,
|
||||
dequeue_wall_ns=dequeue_wall_ns,
|
||||
end_wall_ns=now_wall_ns,
|
||||
n_destinations=n_destinations,
|
||||
source_wait_wall=source_wait_wall,
|
||||
indices_wait_wall=indices_wait_wall,
|
||||
dp_rank=self.attn_dp_rank,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
)
|
||||
# The status was checked before this call; the registry re-checks the
|
||||
# cancellation mark under its lock, which is what makes an abort that
|
||||
# lands in between unable to resurrect stats (review finding).
|
||||
self._room_detail.record_join(
|
||||
kv_chunk.room,
|
||||
status=status,
|
||||
room_failed=self.check_status(kv_chunk.room) == KVPoll.Failed,
|
||||
now_wall_ns=now_wall_ns,
|
||||
queue_wait_total_ns=queue_wait_total_ns,
|
||||
worker_busy_ns=worker_busy_ns,
|
||||
enqueue_wall_ns=kv_chunk.enqueue_wall_ns,
|
||||
is_last_chunk=kv_chunk.is_last_chunk,
|
||||
chunk_seq=kv_chunk.chunk_seq,
|
||||
)
|
||||
|
||||
def transfer_worker(
|
||||
self,
|
||||
queue: FastQueue,
|
||||
@@ -2740,6 +2808,34 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# _stopped alone can never wake a parked worker during a role switch.
|
||||
if kv_chunk is None:
|
||||
break
|
||||
_obs = bool(getattr(kv_chunk, "observe", False))
|
||||
_dequeue_mono_ns = time.monotonic_ns() if _obs else 0
|
||||
_dequeue_wall_ns = time.time_ns() if _obs else 0
|
||||
if _obs:
|
||||
# Needed by note_chunk_requeue(): queue wait ends here and
|
||||
# must not include worker/watermark time.
|
||||
kv_chunk.dequeue_mono_ns = _dequeue_mono_ns
|
||||
_queue_wait_ns = 0
|
||||
_source_wait_ns = 0
|
||||
_source_wait_wall = None
|
||||
_indices_wait_ns = 0
|
||||
_indices_wait_wall = None
|
||||
_kv_transfer_ns = 0
|
||||
_state_transfer_ns = 0
|
||||
_aux_transfer_ns = 0
|
||||
_n_dest = 0
|
||||
if _obs:
|
||||
# queue wait is not part of worker busy time (design §5.2).
|
||||
# Keep the attempt window (child span) separate from the
|
||||
# cumulative value across deferred attempts (review finding).
|
||||
(
|
||||
_attempt_queue_wait_ns,
|
||||
_queue_wait_ns,
|
||||
) = batch_telemetry.split_queue_wait(
|
||||
kv_chunk.queue_wait_accum_ns,
|
||||
kv_chunk.last_enqueue_mono_ns,
|
||||
_dequeue_mono_ns,
|
||||
)
|
||||
if self.enable_trace:
|
||||
kv_chunk.trace_ctx.rebuild_thread_context()
|
||||
kv_chunk.trace_ctx.trace_slice_start(
|
||||
@@ -2765,13 +2861,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
kv_chunk.room in self.request_status
|
||||
and self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
):
|
||||
_wait_t0 = time.monotonic_ns() if _obs else 0
|
||||
_wait_wall0 = time.time_ns() if _obs else 0
|
||||
kv_chunk.wait_event.synchronize()
|
||||
if _wait_t0:
|
||||
_source_wait_ns += time.monotonic_ns() - _wait_t0
|
||||
_wait_wall1 = time.time_ns()
|
||||
_source_wait_wall = (_wait_wall0, _wait_wall1)
|
||||
kv_chunk.wait_event = None
|
||||
_ind_t0 = time.monotonic_ns() if _obs else 0
|
||||
_ind_wall0 = time.time_ns() if _obs else 0
|
||||
if kv_chunk.indices_event is not None:
|
||||
kv_chunk.indices_event.synchronize()
|
||||
kv_chunk.indices_event = None
|
||||
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
|
||||
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
|
||||
if _ind_t0:
|
||||
# synchronize + host indices availability, per design §5.2.
|
||||
_indices_wait_ns += time.monotonic_ns() - _ind_t0
|
||||
_indices_wait_wall = (_ind_wall0, time.time_ns())
|
||||
|
||||
if (
|
||||
kv_chunk.room not in self.request_status
|
||||
@@ -2787,6 +2895,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
thread_finish_flag=True,
|
||||
)
|
||||
self._staging_outstanding.pop(kv_chunk.room, None)
|
||||
if _obs:
|
||||
self._emit_chunk_attempt(
|
||||
kv_chunk,
|
||||
status="skipped",
|
||||
queue_wait_ns=_attempt_queue_wait_ns,
|
||||
queue_wait_total_ns=_queue_wait_ns,
|
||||
source_wait_ns=_source_wait_ns,
|
||||
indices_wait_ns=_indices_wait_ns,
|
||||
kv_transfer_ns=_kv_transfer_ns,
|
||||
state_transfer_ns=_state_transfer_ns,
|
||||
aux_transfer_ns=_aux_transfer_ns,
|
||||
work_start_ns=_dequeue_mono_ns,
|
||||
dequeue_wall_ns=_dequeue_wall_ns,
|
||||
n_destinations=_n_dest,
|
||||
source_wait_wall=_source_wait_wall,
|
||||
indices_wait_wall=_indices_wait_wall,
|
||||
)
|
||||
if self.enable_deferred_decode_kv_release:
|
||||
# Skipped => nothing written for this aborted room; ack.
|
||||
self._maybe_ack_drained_abort(kv_chunk.room)
|
||||
@@ -2882,6 +3007,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
skip_kv, skip_state = self._get_dsa_cache_transfer_skip_flags(
|
||||
target_rank_registration_info
|
||||
)
|
||||
_kv_t0 = time.monotonic_ns() if _obs else 0
|
||||
if (
|
||||
len(kv_chunk.prefill_kv_indices) == 0
|
||||
or not self.kv_args.kv_data_ptrs
|
||||
@@ -2954,6 +3080,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if deferred:
|
||||
staging_deferred = True
|
||||
if _kv_t0:
|
||||
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
|
||||
_n_dest += 1
|
||||
# Chunk re-enqueued; stop processing remaining reqs for this chunk
|
||||
break
|
||||
else:
|
||||
@@ -2968,6 +3097,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
executor,
|
||||
target_rank_registration_info.dst_kv_layer_ids,
|
||||
)
|
||||
if _kv_t0:
|
||||
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
|
||||
_n_dest += 1
|
||||
if ret != 0:
|
||||
with self.session_lock:
|
||||
self.session_failures[req.mooncake_session_id] += 1
|
||||
@@ -2989,12 +3121,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
if kv_chunk.is_last_chunk:
|
||||
if kv_chunk.state_indices and not skip_state:
|
||||
_state_t0 = time.monotonic_ns() if _obs else 0
|
||||
state_rc = self.maybe_send_extra(
|
||||
req,
|
||||
kv_chunk.state_indices,
|
||||
executor,
|
||||
target_rank_registration_info,
|
||||
)
|
||||
if _state_t0:
|
||||
_state_transfer_ns += time.monotonic_ns() - _state_t0
|
||||
# Before the Failed check: an ABORT during the
|
||||
# send also marks the room Failed.
|
||||
if state_rc != 0:
|
||||
@@ -3025,11 +3160,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
break
|
||||
|
||||
# Only the last chunk we need to send the aux data
|
||||
_aux_t0 = time.monotonic_ns() if _obs else 0
|
||||
ret = self.send_aux(
|
||||
req,
|
||||
kv_chunk.prefill_aux_index,
|
||||
target_rank_registration_info.dst_aux_ptrs,
|
||||
)
|
||||
if _aux_t0:
|
||||
_aux_transfer_ns += time.monotonic_ns() - _aux_t0
|
||||
if ret != 0:
|
||||
# The aux write targets decode's metadata
|
||||
# slot, which the deferred release also frees.
|
||||
@@ -3071,8 +3209,45 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
|
||||
if staging_deferred:
|
||||
if _obs:
|
||||
self._emit_chunk_attempt(
|
||||
kv_chunk,
|
||||
status="deferred",
|
||||
queue_wait_ns=_attempt_queue_wait_ns,
|
||||
queue_wait_total_ns=_queue_wait_ns,
|
||||
source_wait_ns=_source_wait_ns,
|
||||
indices_wait_ns=_indices_wait_ns,
|
||||
kv_transfer_ns=_kv_transfer_ns,
|
||||
state_transfer_ns=_state_transfer_ns,
|
||||
aux_transfer_ns=_aux_transfer_ns,
|
||||
work_start_ns=_dequeue_mono_ns,
|
||||
dequeue_wall_ns=_dequeue_wall_ns,
|
||||
n_destinations=_n_dest,
|
||||
source_wait_wall=_source_wait_wall,
|
||||
indices_wait_wall=_indices_wait_wall,
|
||||
)
|
||||
continue
|
||||
|
||||
if _obs:
|
||||
self._emit_chunk_attempt(
|
||||
kv_chunk,
|
||||
status="ok"
|
||||
if self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
else "failed",
|
||||
queue_wait_ns=_attempt_queue_wait_ns,
|
||||
queue_wait_total_ns=_queue_wait_ns,
|
||||
source_wait_ns=_source_wait_ns,
|
||||
indices_wait_ns=_indices_wait_ns,
|
||||
kv_transfer_ns=_kv_transfer_ns,
|
||||
state_transfer_ns=_state_transfer_ns,
|
||||
aux_transfer_ns=_aux_transfer_ns,
|
||||
work_start_ns=_dequeue_mono_ns,
|
||||
dequeue_wall_ns=_dequeue_wall_ns,
|
||||
n_destinations=_n_dest,
|
||||
source_wait_wall=_source_wait_wall,
|
||||
indices_wait_wall=_indices_wait_wall,
|
||||
)
|
||||
|
||||
self._staging_outstanding[kv_chunk.room] -= 1
|
||||
if self.enable_deferred_decode_kv_release:
|
||||
# In-flight write finished; if aborted and nothing outstanding,
|
||||
@@ -3096,6 +3271,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._session_endpoint_map.pop(sid, None)
|
||||
self.transfer_infos.pop(kv_chunk.room)
|
||||
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
|
||||
# Success: the scheduler poll still needs the join summary.
|
||||
# Failed/absent: nothing will consume it, drop it here.
|
||||
self._room_detail.teardown(
|
||||
kv_chunk.room,
|
||||
keep_join=(
|
||||
self.request_status.get(kv_chunk.room)
|
||||
== KVPoll.Success
|
||||
),
|
||||
)
|
||||
if self.enable_staging:
|
||||
# Purge prefetch bookkeeping for the finished room.
|
||||
# Snapshot first: the scheduler thread adds concurrently.
|
||||
@@ -3332,6 +3516,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
wait_event: Optional[object] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
obs: Optional[dict] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -3361,6 +3546,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if trace_ctx is None:
|
||||
trace_ctx = TraceNullContext()
|
||||
|
||||
if obs is not None:
|
||||
# Stamp only now: every earlier return (failed room, dummy rank,
|
||||
# zero-page skip) must not consume a chunk_seq or a first-enqueue
|
||||
# timestamp (review finding).
|
||||
now_mono_ns = time.monotonic_ns()
|
||||
now_wall_ns = time.time_ns()
|
||||
chunk_seq, _first_wall_ns = self._room_detail.begin_chunk(
|
||||
bootstrap_room, now_mono_ns, now_wall_ns
|
||||
)
|
||||
obs = dict(obs)
|
||||
obs.update(
|
||||
chunk_seq=chunk_seq,
|
||||
enqueue_mono_ns=now_mono_ns,
|
||||
enqueue_wall_ns=now_wall_ns,
|
||||
last_enqueue_wall_ns=now_wall_ns,
|
||||
)
|
||||
|
||||
self.transfer_queues[shard_idx].put(
|
||||
TransferKVChunk(
|
||||
room=bootstrap_room,
|
||||
@@ -3373,6 +3575,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
trace_ctx=trace_ctx,
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
**(obs or {}),
|
||||
queue_id=int(shard_idx),
|
||||
observe=obs is not None,
|
||||
last_enqueue_mono_ns=int((obs or {}).get("enqueue_mono_ns", 0)),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -804,6 +804,7 @@ class MoriKVManager(CommonKVManager):
|
||||
wait_event: Optional[object] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
trace_ctx=None,
|
||||
obs: Optional[dict] = None,
|
||||
) -> None:
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
|
||||
@@ -3267,6 +3267,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
wait_event=None,
|
||||
indices_event=None,
|
||||
trace_ctx=None,
|
||||
obs: Optional[dict] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
|
||||
@@ -1426,11 +1426,30 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self.tree_cache.finish(
|
||||
req.cache_request_handle, CacheRequestOutcome.SUCCESS
|
||||
)
|
||||
# Consume the join summary before clear() releases the room.
|
||||
_join_stats = None
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
_pop_stats = getattr(
|
||||
req.disagg_kv_sender, "pop_chunk_join_stats", None
|
||||
)
|
||||
_join_stats = _pop_stats() if _pop_stats is not None else None
|
||||
# FIXME: clean up req's data in transfer engine
|
||||
req.disagg_kv_sender.clear()
|
||||
done_reqs.append(req)
|
||||
req.time_stats.set_prefill_kv_transfer_finish_time()
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
batch_telemetry.emit_request_kv_join(
|
||||
req=req,
|
||||
stats=_join_stats,
|
||||
dp_rank=get_parallel().attn_dp_rank,
|
||||
)
|
||||
elif poll == KVPoll.Failed:
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
_pop_stats = getattr(
|
||||
req.disagg_kv_sender, "pop_chunk_join_stats", None
|
||||
)
|
||||
if _pop_stats is not None:
|
||||
_pop_stats()
|
||||
self.handle_inflight_transfer_failure(req)
|
||||
done_reqs.append(req)
|
||||
else:
|
||||
@@ -2016,6 +2035,12 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
|
||||
"""Release KV cache and requeue an optimistic prefill request."""
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
# A retry may reuse the same room; stale chunk stats would
|
||||
# otherwise be merged into the next attempt's join summary.
|
||||
_reset = getattr(req.disagg_kv_sender, "reset_chunk_detail_state", None)
|
||||
if _reset is not None:
|
||||
_reset()
|
||||
max_attempts = get_disagg().optimistic_prefill_attempts
|
||||
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
|
||||
if not uses_write_through_cache:
|
||||
|
||||
@@ -0,0 +1,594 @@
|
||||
"""Unit tests for Mooncake chunk-level tracing (SGLANG_TRACE_MOONCAKE_DETAIL).
|
||||
|
||||
Covers design §5: queue/source/indices/transfer breakdown per worker attempt,
|
||||
request-level join fields, and the additive fields on TransferKVChunk.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def _module_path():
|
||||
p = (
|
||||
pathlib.Path(__file__).resolve().parents[4]
|
||||
/ "python/sglang/srt/observability/batch_telemetry.py"
|
||||
)
|
||||
if p.exists():
|
||||
return p
|
||||
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
|
||||
return pathlib.Path(spec.origin)
|
||||
|
||||
|
||||
def _load(**env):
|
||||
base = {
|
||||
"SGLANG_BATCH_STAGE_TIMING": "1",
|
||||
"SGLANG_TRACE_LOOP_SPAN": "1",
|
||||
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
|
||||
"SGLANG_TRACE_LOOP_DETAIL": "1",
|
||||
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
|
||||
"SGLANG_TRACE_FORWARD_SPAN": "0",
|
||||
}
|
||||
base.update(env)
|
||||
with mock.patch.dict(os.environ, base):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"batch_telemetry_detail_under_test", _module_path()
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
class _FakeCounter:
|
||||
def __init__(self):
|
||||
self.data = {}
|
||||
|
||||
def labels(self, **kw):
|
||||
self._key = tuple(sorted(kw.items()))
|
||||
return self
|
||||
|
||||
def inc(self, value=1.0):
|
||||
self.data[self._key] = self.data.get(self._key, 0.0) + value
|
||||
|
||||
|
||||
def _fake_loop_metrics():
|
||||
return {
|
||||
"no_batch_seconds": _FakeCounter(),
|
||||
"no_batch_total": _FakeCounter(),
|
||||
"lockstep_seconds": _FakeCounter(),
|
||||
}
|
||||
|
||||
|
||||
class _FakeSpan:
|
||||
def __init__(self, name, **kwargs):
|
||||
self.name = name
|
||||
self.kwargs = kwargs
|
||||
self.attributes = dict(kwargs.get("attributes") or {})
|
||||
self.end_time = None
|
||||
|
||||
def set_attributes(self, attrs):
|
||||
self.attributes.update(attrs)
|
||||
|
||||
def get_span_context(self):
|
||||
return types.SimpleNamespace(
|
||||
is_valid=True, trace_id=1, span_id=2
|
||||
)
|
||||
|
||||
def end(self, end_time=None):
|
||||
self.end_time = end_time
|
||||
|
||||
|
||||
class _FakeTracer:
|
||||
def __init__(self):
|
||||
self.spans = []
|
||||
|
||||
def start_span(self, name, **kwargs):
|
||||
span = _FakeSpan(name, **kwargs)
|
||||
self.spans.append(span)
|
||||
return span
|
||||
|
||||
|
||||
def _fake_otel():
|
||||
tracer = _FakeTracer()
|
||||
trace = types.ModuleType("opentelemetry.trace")
|
||||
trace.get_tracer = lambda _name: tracer
|
||||
trace.set_span_in_context = lambda span: ("ctx", span)
|
||||
trace.Link = lambda ctx: ("link", ctx)
|
||||
package = types.ModuleType("opentelemetry")
|
||||
package.trace = trace
|
||||
patcher = mock.patch.dict(
|
||||
sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace}
|
||||
)
|
||||
return tracer, patcher
|
||||
|
||||
|
||||
def _state(**overrides):
|
||||
state = {
|
||||
"waiting": 0,
|
||||
"result_queue": 0,
|
||||
"result_limit": 2,
|
||||
"inflight": 0,
|
||||
"bootstrap": 0,
|
||||
"deferred": 0,
|
||||
"paused": False,
|
||||
"waiting_for_result": False,
|
||||
"skip_batch_creation": False,
|
||||
"forward_active": False,
|
||||
}
|
||||
state.update(overrides)
|
||||
return state
|
||||
|
||||
|
||||
def _span(tracer, name):
|
||||
return next(s for s in tracer.spans if s.name == name)
|
||||
|
||||
|
||||
class MooncakeChunkDetailTest(unittest.TestCase):
|
||||
def _chunk(self, **over):
|
||||
now_wall = time.time_ns()
|
||||
chunk = types.SimpleNamespace(
|
||||
room=17,
|
||||
chunk_seq=3,
|
||||
loop_seq=99,
|
||||
send_kind="last",
|
||||
is_last_chunk=True,
|
||||
queue_id=5,
|
||||
requeue_count=1,
|
||||
num_kv_tokens=4096,
|
||||
num_pages=32,
|
||||
approx_bytes=1 << 20,
|
||||
enqueue_wall_ns=now_wall - 40_000_000,
|
||||
# Attempt-scoped start: after a defer this is the re-enqueue time.
|
||||
last_enqueue_wall_ns=now_wall - 20_000_000,
|
||||
last_enqueue_mono_ns=0,
|
||||
dequeue_mono_ns=0,
|
||||
trace_ctx=None,
|
||||
)
|
||||
for k, v in over.items():
|
||||
setattr(chunk, k, v)
|
||||
return chunk
|
||||
|
||||
def test_disabled_emits_no_span(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="0")
|
||||
self.assertFalse(mod.mooncake_detail_enabled())
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
mod.emit_mooncake_chunk(
|
||||
chunk=self._chunk(),
|
||||
status="ok",
|
||||
queue_wait_ns=1,
|
||||
queue_wait_total_ns=1,
|
||||
source_wait_ns=1,
|
||||
indices_wait_ns=1,
|
||||
kv_transfer_ns=1,
|
||||
state_transfer_ns=1,
|
||||
aux_transfer_ns=1,
|
||||
attempt_wall_ns=10,
|
||||
dequeue_wall_ns=time.time_ns(),
|
||||
end_wall_ns=time.time_ns(),
|
||||
n_destinations=1,
|
||||
)
|
||||
self.assertEqual(tracer.spans, [])
|
||||
|
||||
def test_chunk_breakdown_attributes(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
self.assertTrue(mod.mooncake_detail_enabled())
|
||||
tracer, patcher = _fake_otel()
|
||||
now_wall = time.time_ns()
|
||||
dequeue_wall = now_wall - 16_000_000
|
||||
source_wall = (now_wall - 15_000_000, now_wall - 13_000_000)
|
||||
indices_wall = (now_wall - 12_000_000, now_wall - 11_000_000)
|
||||
chunk = self._chunk(
|
||||
last_enqueue_wall_ns=now_wall - 20_000_000,
|
||||
queue_wait_accum_ns=5_000_000,
|
||||
)
|
||||
with patcher:
|
||||
mod.emit_mooncake_chunk(
|
||||
chunk=chunk,
|
||||
status="ok",
|
||||
queue_wait_ns=4_000_000,
|
||||
queue_wait_total_ns=9_000_000,
|
||||
source_wait_ns=2_000_000,
|
||||
indices_wait_ns=1_000_000,
|
||||
kv_transfer_ns=10_000_000,
|
||||
state_transfer_ns=1_000_000,
|
||||
aux_transfer_ns=1_000_000,
|
||||
attempt_wall_ns=20_000_000,
|
||||
dequeue_wall_ns=dequeue_wall,
|
||||
end_wall_ns=now_wall,
|
||||
n_destinations=2,
|
||||
source_wait_wall=source_wall,
|
||||
indices_wait_wall=indices_wall,
|
||||
dp_rank=1,
|
||||
tp_rank=0,
|
||||
)
|
||||
span = _span(tracer, "mooncake.kv_chunk")
|
||||
attrs = span.attributes
|
||||
self.assertEqual(attrs["sglang.bootstrap_room"], 17)
|
||||
self.assertEqual(attrs["sglang.chunk_seq"], 3)
|
||||
self.assertEqual(attrs["sglang.loop_seq"], 99)
|
||||
self.assertEqual(attrs["sglang.send_kind"], "last")
|
||||
self.assertEqual(attrs["sglang.queue_id"], 5)
|
||||
# Attempt window matches the child span; the cumulative value is
|
||||
# exposed separately (review finding).
|
||||
self.assertAlmostEqual(attrs["sglang.queue_wait_ms"], 4.0)
|
||||
self.assertAlmostEqual(attrs["sglang.queue_wait_total_ms"], 9.0)
|
||||
self.assertAlmostEqual(attrs["sglang.kv_transfer_ms"], 10.0)
|
||||
self.assertAlmostEqual(attrs["sglang.attempt_wall_ms"], 20.0)
|
||||
# worker_busy excludes the source/indices event waits (design §5.3).
|
||||
self.assertAlmostEqual(attrs["sglang.worker_busy_ms"], 17.0)
|
||||
# 17 - (10 + 1 + 1) = 5 ms unexplained active worker time.
|
||||
self.assertAlmostEqual(attrs["sglang.worker_residual_ms"], 5.0)
|
||||
# Parent starts at the attempt's last enqueue, not the first one.
|
||||
self.assertEqual(span.kwargs["start_time"], chunk.last_enqueue_wall_ns)
|
||||
names = [s.name for s in tracer.spans]
|
||||
self.assertIn("mooncake.queue_wait", names)
|
||||
self.assertIn("mooncake.wait_source", names)
|
||||
self.assertIn("mooncake.wait_indices", names)
|
||||
queue_span = _span(tracer, "mooncake.queue_wait")
|
||||
self.assertEqual(queue_span.kwargs["start_time"], chunk.last_enqueue_wall_ns)
|
||||
self.assertEqual(queue_span.end_time, dequeue_wall)
|
||||
self.assertAlmostEqual(
|
||||
(queue_span.end_time - queue_span.kwargs["start_time"]) / 1e6, 4.0
|
||||
)
|
||||
self.assertAlmostEqual(queue_span.attributes["sglang.wait_ms"], 4.0)
|
||||
source_span = _span(tracer, "mooncake.wait_source")
|
||||
self.assertEqual(source_span.kwargs["start_time"], source_wall[0])
|
||||
self.assertEqual(source_span.end_time, source_wall[1])
|
||||
indices_span = _span(tracer, "mooncake.wait_indices")
|
||||
self.assertEqual(indices_span.kwargs["start_time"], indices_wall[0])
|
||||
self.assertEqual(indices_span.end_time, indices_wall[1])
|
||||
|
||||
def test_no_synthetic_wait_span_without_real_window(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
mod.emit_mooncake_chunk(
|
||||
chunk=self._chunk(),
|
||||
status="ok",
|
||||
queue_wait_ns=0,
|
||||
queue_wait_total_ns=0,
|
||||
source_wait_ns=2_000_000,
|
||||
indices_wait_ns=1_000_000,
|
||||
kv_transfer_ns=1_000_000,
|
||||
state_transfer_ns=0,
|
||||
aux_transfer_ns=0,
|
||||
attempt_wall_ns=10_000_000,
|
||||
# queue wait has no valid window: last_enqueue == dequeue.
|
||||
dequeue_wall_ns=time.time_ns(),
|
||||
end_wall_ns=time.time_ns(),
|
||||
n_destinations=1,
|
||||
)
|
||||
names = [s.name for s in tracer.spans]
|
||||
self.assertNotIn("mooncake.queue_wait", names)
|
||||
self.assertNotIn("mooncake.wait_source", names)
|
||||
self.assertNotIn("mooncake.wait_indices", names)
|
||||
attrs = _span(tracer, "mooncake.kv_chunk").attributes
|
||||
self.assertAlmostEqual(attrs["sglang.source_wait_ms"], 2.0)
|
||||
|
||||
def test_queue_wait_split_matches_child_span(self):
|
||||
"""Two defers then a third attempt: 20 us total + 10 us attempt."""
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
attempt_ns, total_ns = mod.split_queue_wait(20_000, 1_000_000, 1_010_000)
|
||||
self.assertEqual(attempt_ns, 10_000)
|
||||
self.assertEqual(total_ns, 30_000)
|
||||
tracer, patcher = _fake_otel()
|
||||
now = time.time_ns()
|
||||
chunk = self._chunk(
|
||||
last_enqueue_wall_ns=now - 10_000,
|
||||
queue_wait_accum_ns=20_000,
|
||||
)
|
||||
with patcher:
|
||||
mod.emit_mooncake_chunk(
|
||||
chunk=chunk,
|
||||
status="ok",
|
||||
queue_wait_ns=attempt_ns,
|
||||
queue_wait_total_ns=total_ns,
|
||||
source_wait_ns=0,
|
||||
indices_wait_ns=0,
|
||||
kv_transfer_ns=1_000,
|
||||
state_transfer_ns=0,
|
||||
aux_transfer_ns=0,
|
||||
attempt_wall_ns=2_000,
|
||||
dequeue_wall_ns=now,
|
||||
end_wall_ns=now,
|
||||
n_destinations=1,
|
||||
)
|
||||
child = _span(tracer, "mooncake.queue_wait")
|
||||
# Wall window == this attempt only; the accumulated value is an
|
||||
# attribute on the parent, never on the child span.
|
||||
self.assertEqual(
|
||||
child.end_time - child.kwargs["start_time"], attempt_ns
|
||||
)
|
||||
self.assertAlmostEqual(
|
||||
child.attributes["sglang.wait_ms"], attempt_ns / 1e6
|
||||
)
|
||||
attrs = _span(tracer, "mooncake.kv_chunk").attributes
|
||||
self.assertAlmostEqual(attrs["sglang.queue_wait_ms"], attempt_ns / 1e6)
|
||||
self.assertAlmostEqual(
|
||||
attrs["sglang.queue_wait_total_ms"], total_ns / 1e6
|
||||
)
|
||||
|
||||
def test_requeue_accumulates_only_queue_wait(self):
|
||||
"""Two consecutive defers must not count worker/watermark time."""
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
chunk = types.SimpleNamespace(
|
||||
observe=True,
|
||||
queue_wait_accum_ns=0,
|
||||
last_enqueue_mono_ns=1_000,
|
||||
last_enqueue_wall_ns=2_000,
|
||||
dequeue_mono_ns=11_000, # attempt 1: 10 ms queue wait
|
||||
requeue_count=0,
|
||||
)
|
||||
# Worker + watermark consumed 30 ms; only the 10 ms queue wait counts.
|
||||
mod.note_chunk_requeue(chunk, 41_000, 42_000)
|
||||
self.assertEqual(chunk.queue_wait_accum_ns, 10_000)
|
||||
self.assertEqual(chunk.requeue_count, 1)
|
||||
self.assertEqual(chunk.last_enqueue_mono_ns, 41_000)
|
||||
self.assertEqual(chunk.last_enqueue_wall_ns, 42_000)
|
||||
self.assertEqual(chunk.dequeue_mono_ns, 0)
|
||||
|
||||
# Attempt 2: dequeued 10 ms later, then another defer.
|
||||
chunk.dequeue_mono_ns = 51_000
|
||||
mod.note_chunk_requeue(chunk, 91_000, 92_000)
|
||||
self.assertEqual(chunk.queue_wait_accum_ns, 20_000)
|
||||
self.assertEqual(chunk.requeue_count, 2)
|
||||
|
||||
def test_requeue_disabled_is_noop(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="0")
|
||||
chunk = types.SimpleNamespace(
|
||||
observe=True,
|
||||
queue_wait_accum_ns=0,
|
||||
last_enqueue_mono_ns=1_000,
|
||||
last_enqueue_wall_ns=2_000,
|
||||
dequeue_mono_ns=11_000,
|
||||
requeue_count=0,
|
||||
)
|
||||
mod.note_chunk_requeue(chunk, 41_000, 42_000)
|
||||
self.assertEqual(chunk.queue_wait_accum_ns, 0)
|
||||
self.assertEqual(chunk.requeue_count, 0)
|
||||
|
||||
|
||||
class RequestJoinSpanTest(unittest.TestCase):
|
||||
def test_request_join_span(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
tracer, patcher = _fake_otel()
|
||||
now_wall = time.time_ns()
|
||||
req = types.SimpleNamespace(rid="req-1", bootstrap_room=42)
|
||||
stats = {
|
||||
"chunk_count": 4,
|
||||
"first_enqueue_wall_ns": now_wall - 600_000_000,
|
||||
"last_chunk_enqueue_wall_ns": now_wall - 30_000_000,
|
||||
"last_chunk_complete_wall_ns": now_wall - 10_000_000,
|
||||
"last_chunk_queue_wait_ms": 2.5,
|
||||
"last_chunk_worker_ms": 8.0,
|
||||
}
|
||||
with patcher:
|
||||
mod.emit_request_kv_join(
|
||||
req=req, stats=stats, dp_rank=3, completion_wall_ns=now_wall
|
||||
)
|
||||
span = _span(tracer, "kv.request_join")
|
||||
attrs = span.attributes
|
||||
self.assertEqual(attrs["sglang.bootstrap_room"], 42)
|
||||
self.assertEqual(attrs["sglang.chunk_count"], 4)
|
||||
self.assertAlmostEqual(attrs["sglang.final_exposed_ms"], 20.0)
|
||||
self.assertAlmostEqual(attrs["sglang.completion_poll_ms"], 10.0)
|
||||
self.assertAlmostEqual(attrs["sglang.request_join_ms"], 30.0)
|
||||
self.assertAlmostEqual(
|
||||
attrs["sglang.first_enqueue_to_completion_ms"], 600.0, delta=1.0
|
||||
)
|
||||
self.assertEqual(
|
||||
span.kwargs["start_time"], stats["first_enqueue_wall_ns"]
|
||||
)
|
||||
self.assertEqual(len(span.kwargs.get("links") or []), 0)
|
||||
|
||||
|
||||
class RoomDetailStateTest(unittest.TestCase):
|
||||
def test_begin_chunk_is_sequential_and_keeps_first_enqueue(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
state = mod.RoomDetailState()
|
||||
seq0, first0 = state.begin_chunk(7, 1_000, 2_000)
|
||||
seq1, first1 = state.begin_chunk(7, 5_000, 6_000)
|
||||
self.assertEqual((seq0, seq1), (0, 1))
|
||||
self.assertEqual(first0, 2_000)
|
||||
self.assertEqual(first1, 2_000) # first enqueue of the room
|
||||
|
||||
def test_abort_between_status_check_and_join_write_cannot_resurrect(self):
|
||||
"""Deterministic interleaving of the abort/worker race (review)."""
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
state = mod.RoomDetailState()
|
||||
state.begin_chunk(7, 1_000, 2_000)
|
||||
# Worker saw status ok, then the scheduler aborts before the write.
|
||||
state.reset(7)
|
||||
written = state.record_join(
|
||||
7,
|
||||
status="ok",
|
||||
room_failed=False, # stale status read from before the abort
|
||||
now_wall_ns=3_000,
|
||||
queue_wait_total_ns=10,
|
||||
worker_busy_ns=20,
|
||||
enqueue_wall_ns=2_000,
|
||||
is_last_chunk=True,
|
||||
chunk_seq=0,
|
||||
)
|
||||
self.assertFalse(written)
|
||||
self.assertIsNone(state.pop_join(7))
|
||||
|
||||
def test_join_write_then_abort_is_cleaned(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
state = mod.RoomDetailState()
|
||||
state.begin_chunk(7, 1_000, 2_000)
|
||||
self.assertTrue(
|
||||
state.record_join(
|
||||
7,
|
||||
status="ok",
|
||||
room_failed=False,
|
||||
now_wall_ns=3_000,
|
||||
queue_wait_total_ns=10,
|
||||
worker_busy_ns=20,
|
||||
enqueue_wall_ns=2_000,
|
||||
is_last_chunk=True,
|
||||
chunk_seq=0,
|
||||
)
|
||||
)
|
||||
state.reset(7)
|
||||
self.assertIsNone(state.pop_join(7))
|
||||
|
||||
def test_failed_worker_never_writes_join(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
state = mod.RoomDetailState()
|
||||
state.begin_chunk(7, 1_000, 2_000)
|
||||
self.assertFalse(
|
||||
state.record_join(
|
||||
7,
|
||||
status="failed",
|
||||
room_failed=True,
|
||||
now_wall_ns=3_000,
|
||||
queue_wait_total_ns=0,
|
||||
worker_busy_ns=0,
|
||||
enqueue_wall_ns=2_000,
|
||||
is_last_chunk=True,
|
||||
chunk_seq=0,
|
||||
)
|
||||
)
|
||||
self.assertIsNone(state.pop_join(7))
|
||||
|
||||
def test_retry_clears_cancellation_mark(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
state = mod.RoomDetailState()
|
||||
state.begin_chunk(7, 1_000, 2_000)
|
||||
state.reset(7)
|
||||
# A retry that reuses the room starts a fresh flow.
|
||||
state.begin_chunk(7, 10_000, 11_000)
|
||||
self.assertTrue(
|
||||
state.record_join(
|
||||
7,
|
||||
status="ok",
|
||||
room_failed=False,
|
||||
now_wall_ns=12_000,
|
||||
queue_wait_total_ns=0,
|
||||
worker_busy_ns=1,
|
||||
enqueue_wall_ns=11_000,
|
||||
is_last_chunk=False,
|
||||
chunk_seq=0,
|
||||
)
|
||||
)
|
||||
self.assertEqual(state.pop_join(7)["chunk_count"], 1)
|
||||
|
||||
def test_cancellation_marks_are_bounded(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
state = mod.RoomDetailState()
|
||||
for room in range(mod.RoomDetailState._MAX_CANCELLED + 10):
|
||||
state.reset(room)
|
||||
self.assertLessEqual(
|
||||
len(state._cancelled), mod.RoomDetailState._MAX_CANCELLED
|
||||
)
|
||||
|
||||
def test_teardown_keeps_success_join_and_drops_failed(self):
|
||||
mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1")
|
||||
state = mod.RoomDetailState()
|
||||
state.begin_chunk(1, 1_000, 2_000)
|
||||
state.record_join(
|
||||
1,
|
||||
status="ok",
|
||||
room_failed=False,
|
||||
now_wall_ns=3_000,
|
||||
queue_wait_total_ns=0,
|
||||
worker_busy_ns=1,
|
||||
enqueue_wall_ns=2_000,
|
||||
is_last_chunk=False,
|
||||
chunk_seq=0,
|
||||
)
|
||||
state.teardown(1, keep_join=True)
|
||||
self.assertIsNotNone(state.pop_join(1))
|
||||
|
||||
state.begin_chunk(2, 1_000, 2_000)
|
||||
state.record_join(
|
||||
2,
|
||||
status="ok",
|
||||
room_failed=False,
|
||||
now_wall_ns=3_000,
|
||||
queue_wait_total_ns=0,
|
||||
worker_busy_ns=1,
|
||||
enqueue_wall_ns=2_000,
|
||||
is_last_chunk=False,
|
||||
chunk_seq=0,
|
||||
)
|
||||
state.teardown(2, keep_join=False)
|
||||
self.assertIsNone(state.pop_join(2))
|
||||
|
||||
|
||||
class ChunkFieldDefaultsTest(unittest.TestCase):
|
||||
"""The observation fields must be additive defaults on TransferKVChunk."""
|
||||
|
||||
def test_dataclass_defaults(self):
|
||||
import dataclasses
|
||||
|
||||
path = (
|
||||
pathlib.Path(__file__).resolve().parents[4]
|
||||
/ "python/sglang/srt/disaggregation/common/utils.py"
|
||||
)
|
||||
trace_stub = types.ModuleType("sglang.srt.observability.trace")
|
||||
|
||||
class _Null:
|
||||
pass
|
||||
|
||||
trace_stub.TraceNullContext = _Null
|
||||
trace_stub.TraceReqContext = _Null
|
||||
patcher = mock.patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"sglang": types.ModuleType("sglang"),
|
||||
"sglang.srt": types.ModuleType("sglang.srt"),
|
||||
"sglang.srt.observability": types.ModuleType(
|
||||
"sglang.srt.observability"
|
||||
),
|
||||
"sglang.srt.observability.trace": trace_stub,
|
||||
},
|
||||
)
|
||||
with patcher:
|
||||
spec = importlib.util.spec_from_file_location("kv_utils_under_test", path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
|
||||
fields = {f.name: f for f in dataclasses.fields(mod.TransferKVChunk)}
|
||||
for name in (
|
||||
"chunk_seq",
|
||||
"loop_seq",
|
||||
"send_kind",
|
||||
"enqueue_mono_ns",
|
||||
"enqueue_wall_ns",
|
||||
"last_enqueue_mono_ns",
|
||||
"last_enqueue_wall_ns",
|
||||
"dequeue_mono_ns",
|
||||
"queue_wait_accum_ns",
|
||||
"queue_id",
|
||||
"requeue_count",
|
||||
"num_pages",
|
||||
"approx_bytes",
|
||||
"observe",
|
||||
):
|
||||
self.assertIn(name, fields)
|
||||
self.assertFalse(fields[name].init is None)
|
||||
chunk = mod.TransferKVChunk(
|
||||
room=1,
|
||||
prefill_kv_indices=None,
|
||||
index_slice=slice(0, 1),
|
||||
is_last_chunk=True,
|
||||
prefill_aux_index=None,
|
||||
state_indices=None,
|
||||
chunk_seq=3,
|
||||
queue_id=2,
|
||||
observe=True,
|
||||
)
|
||||
self.assertEqual(chunk.chunk_seq, 3)
|
||||
self.assertTrue(chunk.observe)
|
||||
self.assertEqual(chunk.requeue_count, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user