diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index c494553ed8..5f78eace00 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -51,6 +51,7 @@ from sglang.srt.disaggregation.utils import ( ) from sglang.srt.environ import envs from sglang.srt.mem_cache.page_interleave import get_kv_shard_group +from sglang.srt.observability import batch_telemetry from sglang.srt.observability.trace import TraceNullContext, TraceReqContext from sglang.srt.runtime_context import ( get_device, @@ -2091,6 +2092,10 @@ class CommonKVSender(BaseKVSender): self._transfer_num_state_indices = 0 self._transfer_start_time: Optional[float] = None self._early_send_wait_event: Optional[object] = None + # Detail tracing: chunk classification only. Enqueue timestamps, + # chunk_seq and the per-request first-enqueue are assigned by the + # manager at the real queue.put() point, after all early returns. + self._send_kind = "" self.trace_ctx = TraceNullContext() if self.kv_mgr.sender_trace_module and get_observability().enable_trace: self.trace_ctx = TraceReqContext( @@ -2289,6 +2294,19 @@ class CommonKVSender(BaseKVSender): ): self._transfer_start_time = time.perf_counter() + obs = None + if batch_telemetry.mooncake_detail_enabled(): + # Metadata only; the manager stamps chunk_seq and enqueue times + # when it is certain the chunk is actually put on the queue. + obs = { + "loop_seq": int(batch_telemetry.current_loop_seq()), + "send_kind": str(self._send_kind or ""), + "num_pages": int(len(kv_indices)), + "approx_bytes": int( + len(kv_indices) * getattr(self.kv_mgr, "kv_item_lens_sum", 0) + ), + } + # The early-send event belongs to exactly one chunk; the Mooncake and # Mori workers synchronize on it before reading source KV. wait_event = self._early_send_wait_event @@ -2304,9 +2322,27 @@ class CommonKVSender(BaseKVSender): wait_event=wait_event, indices_event=indices_event, trace_ctx=self.trace_ctx.copy_for_thread(), + obs=obs, ) self._record_transfer_indices(kv_indices, state_indices) + def pop_chunk_join_stats(self) -> Optional[dict]: + """Take (and free) this room's chunk join summary.""" + state = getattr(self.kv_mgr, "_room_detail", None) + if state is None: + return None + return state.pop_join(self.bootstrap_room) + + def reset_chunk_detail_state(self) -> None: + """Drop per-room detail state and mark it cancelled (abort / retract). + + The mark is what prevents a worker attempt that already passed its + status check from recreating stats after this cleanup. + """ + state = getattr(self.kv_mgr, "_room_detail", None) + if state is not None: + state.reset(self.bootstrap_room) + def poll(self) -> KVPoll: return self._poll_with_status(self.kv_mgr.check_status) @@ -2425,6 +2461,9 @@ class CommonKVSender(BaseKVSender): def abort(self): if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room: return + # Drop per-room detail bookkeeping: an aborted request never reaches + # the poll path that normally consumes it (review finding). + self.reset_chunk_detail_state() self.kv_mgr.record_failure( self.bootstrap_room, "Aborted by AbortReq.", diff --git a/python/sglang/srt/disaggregation/common/utils.py b/python/sglang/srt/disaggregation/common/utils.py index 7946c4aec1..72429290df 100644 --- a/python/sglang/srt/disaggregation/common/utils.py +++ b/python/sglang/srt/disaggregation/common/utils.py @@ -36,6 +36,24 @@ class TransferKVChunk: # Must complete before reading asynchronously copied host indices. indices_event: Optional[object] = None + # --- Optional observation fields (SGLANG_TRACE_MOONCAKE_DETAIL=1) ------ + # Filled at the real enqueue point (add_transfer_request), never at the + # send_kv_chunk entry, so skipped paths cannot create holes in chunk_seq. + chunk_seq: int = 0 # per-sender monotonic enqueue index + loop_seq: int = 0 # scheduler loop that produced this chunk + send_kind: str = "" # cached_prefix / middle / last + enqueue_mono_ns: int = 0 # first enqueue of this chunk, monotonic clock + enqueue_wall_ns: int = 0 # first enqueue of this chunk, wall clock + last_enqueue_mono_ns: int = 0 # updated on every (re-)enqueue + last_enqueue_wall_ns: int = 0 # wall clock of the last (re-)enqueue + dequeue_mono_ns: int = 0 # current attempt dequeue; 0 outside the worker + queue_wait_accum_ns: int = 0 # completed queue waits from earlier attempts + queue_id: int = -1 + requeue_count: int = 0 + num_pages: int = 0 + approx_bytes: int = 0 + observe: bool = False # True when detail tracing populated this chunk + def pack_list_of_buffers(buffers: List[bytes]) -> bytes: if not buffers: diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index a65ec32a63..904529905b 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -60,6 +60,7 @@ from sglang.srt.disaggregation.utils import ( ) from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine from sglang.srt.environ import envs +from sglang.srt.observability import batch_telemetry from sglang.srt.observability.mooncake_trace import ( MooncakeRequestStage, mooncake_trace_func, @@ -386,6 +387,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): # transfer blocks over zmq when the intra-node NVLink transport is # active (it cannot reach host memory). Populated on bootstrap. self._session_endpoint_map = {} + # Detail tracing (SGLANG_TRACE_MOONCAKE_DETAIL=1; empty otherwise). + # Lock-protected per-room state: abort/retract and the worker's join + # update must not interleave into resurrected stats. + self._room_detail = batch_telemetry.RoomDetailState() self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get() self.max_transfer_batch_indices = ( envs.SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES.get() @@ -762,6 +767,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): # block other rooms on this single worker thread. with self._staging_ctx.watermark_cv: self._staging_ctx.watermark_cv.wait(STAGING_WATERMARK_WAIT_S) + if getattr(kv_chunk, "observe", False): + batch_telemetry.note_chunk_requeue( + kv_chunk, time.monotonic_ns(), time.time_ns() + ) queue.put(kv_chunk) return (-1, True) @@ -2716,6 +2725,65 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): if self.enable_deferred_decode_kv_release: self.poison_deferred_ack_room(room) + def _emit_chunk_attempt( + self, + kv_chunk: TransferKVChunk, + *, + status: str, + queue_wait_ns: int, + queue_wait_total_ns: int, + source_wait_ns: int, + indices_wait_ns: int, + kv_transfer_ns: int, + state_transfer_ns: int, + aux_transfer_ns: int, + work_start_ns: int, + dequeue_wall_ns: int, + n_destinations: int, + source_wait_wall=None, + indices_wait_wall=None, + ) -> None: + """Close one worker attempt: chunk span + per-room join summary.""" + now_mono_ns = time.monotonic_ns() + now_wall_ns = time.time_ns() + attempt_wall_ns = max(0, now_mono_ns - int(work_start_ns)) + worker_busy_ns = max( + 0, attempt_wall_ns - int(source_wait_ns) - int(indices_wait_ns) + ) + batch_telemetry.emit_mooncake_chunk( + chunk=kv_chunk, + status=status, + queue_wait_ns=queue_wait_ns, + queue_wait_total_ns=queue_wait_total_ns, + source_wait_ns=source_wait_ns, + indices_wait_ns=indices_wait_ns, + kv_transfer_ns=kv_transfer_ns, + state_transfer_ns=state_transfer_ns, + aux_transfer_ns=aux_transfer_ns, + attempt_wall_ns=attempt_wall_ns, + dequeue_wall_ns=dequeue_wall_ns, + end_wall_ns=now_wall_ns, + n_destinations=n_destinations, + source_wait_wall=source_wait_wall, + indices_wait_wall=indices_wait_wall, + dp_rank=self.attn_dp_rank, + tp_rank=self.attn_tp_rank, + ) + # The status was checked before this call; the registry re-checks the + # cancellation mark under its lock, which is what makes an abort that + # lands in between unable to resurrect stats (review finding). + self._room_detail.record_join( + kv_chunk.room, + status=status, + room_failed=self.check_status(kv_chunk.room) == KVPoll.Failed, + now_wall_ns=now_wall_ns, + queue_wait_total_ns=queue_wait_total_ns, + worker_busy_ns=worker_busy_ns, + enqueue_wall_ns=kv_chunk.enqueue_wall_ns, + is_last_chunk=kv_chunk.is_last_chunk, + chunk_seq=kv_chunk.chunk_seq, + ) + def transfer_worker( self, queue: FastQueue, @@ -2740,6 +2808,34 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): # _stopped alone can never wake a parked worker during a role switch. if kv_chunk is None: break + _obs = bool(getattr(kv_chunk, "observe", False)) + _dequeue_mono_ns = time.monotonic_ns() if _obs else 0 + _dequeue_wall_ns = time.time_ns() if _obs else 0 + if _obs: + # Needed by note_chunk_requeue(): queue wait ends here and + # must not include worker/watermark time. + kv_chunk.dequeue_mono_ns = _dequeue_mono_ns + _queue_wait_ns = 0 + _source_wait_ns = 0 + _source_wait_wall = None + _indices_wait_ns = 0 + _indices_wait_wall = None + _kv_transfer_ns = 0 + _state_transfer_ns = 0 + _aux_transfer_ns = 0 + _n_dest = 0 + if _obs: + # queue wait is not part of worker busy time (design §5.2). + # Keep the attempt window (child span) separate from the + # cumulative value across deferred attempts (review finding). + ( + _attempt_queue_wait_ns, + _queue_wait_ns, + ) = batch_telemetry.split_queue_wait( + kv_chunk.queue_wait_accum_ns, + kv_chunk.last_enqueue_mono_ns, + _dequeue_mono_ns, + ) if self.enable_trace: kv_chunk.trace_ctx.rebuild_thread_context() kv_chunk.trace_ctx.trace_slice_start( @@ -2765,13 +2861,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): kv_chunk.room in self.request_status and self.check_status(kv_chunk.room) != KVPoll.Failed ): + _wait_t0 = time.monotonic_ns() if _obs else 0 + _wait_wall0 = time.time_ns() if _obs else 0 kv_chunk.wait_event.synchronize() + if _wait_t0: + _source_wait_ns += time.monotonic_ns() - _wait_t0 + _wait_wall1 = time.time_ns() + _source_wait_wall = (_wait_wall0, _wait_wall1) kv_chunk.wait_event = None + _ind_t0 = time.monotonic_ns() if _obs else 0 + _ind_wall0 = time.time_ns() if _obs else 0 if kv_chunk.indices_event is not None: kv_chunk.indices_event.synchronize() kv_chunk.indices_event = None if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor): kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy() + if _ind_t0: + # synchronize + host indices availability, per design §5.2. + _indices_wait_ns += time.monotonic_ns() - _ind_t0 + _indices_wait_wall = (_ind_wall0, time.time_ns()) if ( kv_chunk.room not in self.request_status @@ -2787,6 +2895,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): thread_finish_flag=True, ) self._staging_outstanding.pop(kv_chunk.room, None) + if _obs: + self._emit_chunk_attempt( + kv_chunk, + status="skipped", + queue_wait_ns=_attempt_queue_wait_ns, + queue_wait_total_ns=_queue_wait_ns, + source_wait_ns=_source_wait_ns, + indices_wait_ns=_indices_wait_ns, + kv_transfer_ns=_kv_transfer_ns, + state_transfer_ns=_state_transfer_ns, + aux_transfer_ns=_aux_transfer_ns, + work_start_ns=_dequeue_mono_ns, + dequeue_wall_ns=_dequeue_wall_ns, + n_destinations=_n_dest, + source_wait_wall=_source_wait_wall, + indices_wait_wall=_indices_wait_wall, + ) if self.enable_deferred_decode_kv_release: # Skipped => nothing written for this aborted room; ack. self._maybe_ack_drained_abort(kv_chunk.room) @@ -2882,6 +3007,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): skip_kv, skip_state = self._get_dsa_cache_transfer_skip_flags( target_rank_registration_info ) + _kv_t0 = time.monotonic_ns() if _obs else 0 if ( len(kv_chunk.prefill_kv_indices) == 0 or not self.kv_args.kv_data_ptrs @@ -2954,6 +3080,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ) if deferred: staging_deferred = True + if _kv_t0: + _kv_transfer_ns += time.monotonic_ns() - _kv_t0 + _n_dest += 1 # Chunk re-enqueued; stop processing remaining reqs for this chunk break else: @@ -2968,6 +3097,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): executor, target_rank_registration_info.dst_kv_layer_ids, ) + if _kv_t0: + _kv_transfer_ns += time.monotonic_ns() - _kv_t0 + _n_dest += 1 if ret != 0: with self.session_lock: self.session_failures[req.mooncake_session_id] += 1 @@ -2989,12 +3121,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): if kv_chunk.is_last_chunk: if kv_chunk.state_indices and not skip_state: + _state_t0 = time.monotonic_ns() if _obs else 0 state_rc = self.maybe_send_extra( req, kv_chunk.state_indices, executor, target_rank_registration_info, ) + if _state_t0: + _state_transfer_ns += time.monotonic_ns() - _state_t0 # Before the Failed check: an ABORT during the # send also marks the room Failed. if state_rc != 0: @@ -3025,11 +3160,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): break # Only the last chunk we need to send the aux data + _aux_t0 = time.monotonic_ns() if _obs else 0 ret = self.send_aux( req, kv_chunk.prefill_aux_index, target_rank_registration_info.dst_aux_ptrs, ) + if _aux_t0: + _aux_transfer_ns += time.monotonic_ns() - _aux_t0 if ret != 0: # The aux write targets decode's metadata # slot, which the deferred release also frees. @@ -3071,8 +3209,45 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ) if staging_deferred: + if _obs: + self._emit_chunk_attempt( + kv_chunk, + status="deferred", + queue_wait_ns=_attempt_queue_wait_ns, + queue_wait_total_ns=_queue_wait_ns, + source_wait_ns=_source_wait_ns, + indices_wait_ns=_indices_wait_ns, + kv_transfer_ns=_kv_transfer_ns, + state_transfer_ns=_state_transfer_ns, + aux_transfer_ns=_aux_transfer_ns, + work_start_ns=_dequeue_mono_ns, + dequeue_wall_ns=_dequeue_wall_ns, + n_destinations=_n_dest, + source_wait_wall=_source_wait_wall, + indices_wait_wall=_indices_wait_wall, + ) continue + if _obs: + self._emit_chunk_attempt( + kv_chunk, + status="ok" + if self.check_status(kv_chunk.room) != KVPoll.Failed + else "failed", + queue_wait_ns=_attempt_queue_wait_ns, + queue_wait_total_ns=_queue_wait_ns, + source_wait_ns=_source_wait_ns, + indices_wait_ns=_indices_wait_ns, + kv_transfer_ns=_kv_transfer_ns, + state_transfer_ns=_state_transfer_ns, + aux_transfer_ns=_aux_transfer_ns, + work_start_ns=_dequeue_mono_ns, + dequeue_wall_ns=_dequeue_wall_ns, + n_destinations=_n_dest, + source_wait_wall=_source_wait_wall, + indices_wait_wall=_indices_wait_wall, + ) + self._staging_outstanding[kv_chunk.room] -= 1 if self.enable_deferred_decode_kv_release: # In-flight write finished; if aborted and nothing outstanding, @@ -3096,6 +3271,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): self._session_endpoint_map.pop(sid, None) self.transfer_infos.pop(kv_chunk.room) self.req_to_decode_prefix_len.pop(kv_chunk.room, None) + # Success: the scheduler poll still needs the join summary. + # Failed/absent: nothing will consume it, drop it here. + self._room_detail.teardown( + kv_chunk.room, + keep_join=( + self.request_status.get(kv_chunk.room) + == KVPoll.Success + ), + ) if self.enable_staging: # Purge prefetch bookkeeping for the finished room. # Snapshot first: the scheduler thread adds concurrently. @@ -3332,6 +3516,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): wait_event: Optional[object] = None, indices_event: Optional[object] = None, trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None, + obs: Optional[dict] = None, ): assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) @@ -3361,6 +3546,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): if trace_ctx is None: trace_ctx = TraceNullContext() + if obs is not None: + # Stamp only now: every earlier return (failed room, dummy rank, + # zero-page skip) must not consume a chunk_seq or a first-enqueue + # timestamp (review finding). + now_mono_ns = time.monotonic_ns() + now_wall_ns = time.time_ns() + chunk_seq, _first_wall_ns = self._room_detail.begin_chunk( + bootstrap_room, now_mono_ns, now_wall_ns + ) + obs = dict(obs) + obs.update( + chunk_seq=chunk_seq, + enqueue_mono_ns=now_mono_ns, + enqueue_wall_ns=now_wall_ns, + last_enqueue_wall_ns=now_wall_ns, + ) + self.transfer_queues[shard_idx].put( TransferKVChunk( room=bootstrap_room, @@ -3373,6 +3575,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): trace_ctx=trace_ctx, wait_event=wait_event, indices_event=indices_event, + **(obs or {}), + queue_id=int(shard_idx), + observe=obs is not None, + last_enqueue_mono_ns=int((obs or {}).get("enqueue_mono_ns", 0)), ) ) diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index 6700bf59b9..1c8cd85d67 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -804,6 +804,7 @@ class MoriKVManager(CommonKVManager): wait_event: Optional[object] = None, indices_event: Optional[object] = None, trace_ctx=None, + obs: Optional[dict] = None, ) -> None: assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 35c8c62d19..503517ed08 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -3267,6 +3267,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): wait_event=None, indices_event=None, trace_ctx=None, + obs: Optional[dict] = None, ): assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index bec0777d64..be47d02e7d 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1426,11 +1426,30 @@ class SchedulerDisaggregationPrefillMixin: self.tree_cache.finish( req.cache_request_handle, CacheRequestOutcome.SUCCESS ) + # Consume the join summary before clear() releases the room. + _join_stats = None + if batch_telemetry.mooncake_detail_enabled(): + _pop_stats = getattr( + req.disagg_kv_sender, "pop_chunk_join_stats", None + ) + _join_stats = _pop_stats() if _pop_stats is not None else None # FIXME: clean up req's data in transfer engine req.disagg_kv_sender.clear() done_reqs.append(req) req.time_stats.set_prefill_kv_transfer_finish_time() + if batch_telemetry.mooncake_detail_enabled(): + batch_telemetry.emit_request_kv_join( + req=req, + stats=_join_stats, + dp_rank=get_parallel().attn_dp_rank, + ) elif poll == KVPoll.Failed: + if batch_telemetry.mooncake_detail_enabled(): + _pop_stats = getattr( + req.disagg_kv_sender, "pop_chunk_join_stats", None + ) + if _pop_stats is not None: + _pop_stats() self.handle_inflight_transfer_failure(req) done_reqs.append(req) else: @@ -2016,6 +2035,12 @@ class SchedulerDisaggregationPrefillMixin: def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None: """Release KV cache and requeue an optimistic prefill request.""" + if batch_telemetry.mooncake_detail_enabled(): + # A retry may reuse the same room; stale chunk stats would + # otherwise be merged into the next attempt's join summary. + _reset = getattr(req.disagg_kv_sender, "reset_chunk_detail_state", None) + if _reset is not None: + _reset() max_attempts = get_disagg().optimistic_prefill_attempts uses_write_through_cache = _uses_write_through_cache(self.tree_cache) if not uses_write_through_cache: diff --git a/test/registered/unit/observability/test_mooncake_chunk_detail.py b/test/registered/unit/observability/test_mooncake_chunk_detail.py new file mode 100644 index 0000000000..7892870cd0 --- /dev/null +++ b/test/registered/unit/observability/test_mooncake_chunk_detail.py @@ -0,0 +1,594 @@ +"""Unit tests for Mooncake chunk-level tracing (SGLANG_TRACE_MOONCAKE_DETAIL). + +Covers design §5: queue/source/indices/transfer breakdown per worker attempt, +request-level join fields, and the additive fields on TransferKVChunk. +""" +import importlib.util +import os +import pathlib +import sys +import time +import types +import unittest +from unittest import mock + + +def _module_path(): + p = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/observability/batch_telemetry.py" + ) + if p.exists(): + return p + spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry") + return pathlib.Path(spec.origin) + + +def _load(**env): + base = { + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_TRACE_LOOP_SPAN": "1", + "SGLANG_TRACE_LOOP_SAMPLE": "1.0", + "SGLANG_TRACE_LOOP_DETAIL": "1", + "SGLANG_TRACE_MOONCAKE_DETAIL": "0", + "SGLANG_TRACE_FORWARD_SPAN": "0", + } + base.update(env) + with mock.patch.dict(os.environ, base): + spec = importlib.util.spec_from_file_location( + "batch_telemetry_detail_under_test", _module_path() + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class _FakeCounter: + def __init__(self): + self.data = {} + + def labels(self, **kw): + self._key = tuple(sorted(kw.items())) + return self + + def inc(self, value=1.0): + self.data[self._key] = self.data.get(self._key, 0.0) + value + + +def _fake_loop_metrics(): + return { + "no_batch_seconds": _FakeCounter(), + "no_batch_total": _FakeCounter(), + "lockstep_seconds": _FakeCounter(), + } + + +class _FakeSpan: + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs + self.attributes = dict(kwargs.get("attributes") or {}) + self.end_time = None + + def set_attributes(self, attrs): + self.attributes.update(attrs) + + def get_span_context(self): + return types.SimpleNamespace( + is_valid=True, trace_id=1, span_id=2 + ) + + def end(self, end_time=None): + self.end_time = end_time + + +class _FakeTracer: + def __init__(self): + self.spans = [] + + def start_span(self, name, **kwargs): + span = _FakeSpan(name, **kwargs) + self.spans.append(span) + return span + + +def _fake_otel(): + tracer = _FakeTracer() + trace = types.ModuleType("opentelemetry.trace") + trace.get_tracer = lambda _name: tracer + trace.set_span_in_context = lambda span: ("ctx", span) + trace.Link = lambda ctx: ("link", ctx) + package = types.ModuleType("opentelemetry") + package.trace = trace + patcher = mock.patch.dict( + sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace} + ) + return tracer, patcher + + +def _state(**overrides): + state = { + "waiting": 0, + "result_queue": 0, + "result_limit": 2, + "inflight": 0, + "bootstrap": 0, + "deferred": 0, + "paused": False, + "waiting_for_result": False, + "skip_batch_creation": False, + "forward_active": False, + } + state.update(overrides) + return state + + +def _span(tracer, name): + return next(s for s in tracer.spans if s.name == name) + + +class MooncakeChunkDetailTest(unittest.TestCase): + def _chunk(self, **over): + now_wall = time.time_ns() + chunk = types.SimpleNamespace( + room=17, + chunk_seq=3, + loop_seq=99, + send_kind="last", + is_last_chunk=True, + queue_id=5, + requeue_count=1, + num_kv_tokens=4096, + num_pages=32, + approx_bytes=1 << 20, + enqueue_wall_ns=now_wall - 40_000_000, + # Attempt-scoped start: after a defer this is the re-enqueue time. + last_enqueue_wall_ns=now_wall - 20_000_000, + last_enqueue_mono_ns=0, + dequeue_mono_ns=0, + trace_ctx=None, + ) + for k, v in over.items(): + setattr(chunk, k, v) + return chunk + + def test_disabled_emits_no_span(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="0") + self.assertFalse(mod.mooncake_detail_enabled()) + tracer, patcher = _fake_otel() + with patcher: + mod.emit_mooncake_chunk( + chunk=self._chunk(), + status="ok", + queue_wait_ns=1, + queue_wait_total_ns=1, + source_wait_ns=1, + indices_wait_ns=1, + kv_transfer_ns=1, + state_transfer_ns=1, + aux_transfer_ns=1, + attempt_wall_ns=10, + dequeue_wall_ns=time.time_ns(), + end_wall_ns=time.time_ns(), + n_destinations=1, + ) + self.assertEqual(tracer.spans, []) + + def test_chunk_breakdown_attributes(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + self.assertTrue(mod.mooncake_detail_enabled()) + tracer, patcher = _fake_otel() + now_wall = time.time_ns() + dequeue_wall = now_wall - 16_000_000 + source_wall = (now_wall - 15_000_000, now_wall - 13_000_000) + indices_wall = (now_wall - 12_000_000, now_wall - 11_000_000) + chunk = self._chunk( + last_enqueue_wall_ns=now_wall - 20_000_000, + queue_wait_accum_ns=5_000_000, + ) + with patcher: + mod.emit_mooncake_chunk( + chunk=chunk, + status="ok", + queue_wait_ns=4_000_000, + queue_wait_total_ns=9_000_000, + source_wait_ns=2_000_000, + indices_wait_ns=1_000_000, + kv_transfer_ns=10_000_000, + state_transfer_ns=1_000_000, + aux_transfer_ns=1_000_000, + attempt_wall_ns=20_000_000, + dequeue_wall_ns=dequeue_wall, + end_wall_ns=now_wall, + n_destinations=2, + source_wait_wall=source_wall, + indices_wait_wall=indices_wall, + dp_rank=1, + tp_rank=0, + ) + span = _span(tracer, "mooncake.kv_chunk") + attrs = span.attributes + self.assertEqual(attrs["sglang.bootstrap_room"], 17) + self.assertEqual(attrs["sglang.chunk_seq"], 3) + self.assertEqual(attrs["sglang.loop_seq"], 99) + self.assertEqual(attrs["sglang.send_kind"], "last") + self.assertEqual(attrs["sglang.queue_id"], 5) + # Attempt window matches the child span; the cumulative value is + # exposed separately (review finding). + self.assertAlmostEqual(attrs["sglang.queue_wait_ms"], 4.0) + self.assertAlmostEqual(attrs["sglang.queue_wait_total_ms"], 9.0) + self.assertAlmostEqual(attrs["sglang.kv_transfer_ms"], 10.0) + self.assertAlmostEqual(attrs["sglang.attempt_wall_ms"], 20.0) + # worker_busy excludes the source/indices event waits (design §5.3). + self.assertAlmostEqual(attrs["sglang.worker_busy_ms"], 17.0) + # 17 - (10 + 1 + 1) = 5 ms unexplained active worker time. + self.assertAlmostEqual(attrs["sglang.worker_residual_ms"], 5.0) + # Parent starts at the attempt's last enqueue, not the first one. + self.assertEqual(span.kwargs["start_time"], chunk.last_enqueue_wall_ns) + names = [s.name for s in tracer.spans] + self.assertIn("mooncake.queue_wait", names) + self.assertIn("mooncake.wait_source", names) + self.assertIn("mooncake.wait_indices", names) + queue_span = _span(tracer, "mooncake.queue_wait") + self.assertEqual(queue_span.kwargs["start_time"], chunk.last_enqueue_wall_ns) + self.assertEqual(queue_span.end_time, dequeue_wall) + self.assertAlmostEqual( + (queue_span.end_time - queue_span.kwargs["start_time"]) / 1e6, 4.0 + ) + self.assertAlmostEqual(queue_span.attributes["sglang.wait_ms"], 4.0) + source_span = _span(tracer, "mooncake.wait_source") + self.assertEqual(source_span.kwargs["start_time"], source_wall[0]) + self.assertEqual(source_span.end_time, source_wall[1]) + indices_span = _span(tracer, "mooncake.wait_indices") + self.assertEqual(indices_span.kwargs["start_time"], indices_wall[0]) + self.assertEqual(indices_span.end_time, indices_wall[1]) + + def test_no_synthetic_wait_span_without_real_window(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + tracer, patcher = _fake_otel() + with patcher: + mod.emit_mooncake_chunk( + chunk=self._chunk(), + status="ok", + queue_wait_ns=0, + queue_wait_total_ns=0, + source_wait_ns=2_000_000, + indices_wait_ns=1_000_000, + kv_transfer_ns=1_000_000, + state_transfer_ns=0, + aux_transfer_ns=0, + attempt_wall_ns=10_000_000, + # queue wait has no valid window: last_enqueue == dequeue. + dequeue_wall_ns=time.time_ns(), + end_wall_ns=time.time_ns(), + n_destinations=1, + ) + names = [s.name for s in tracer.spans] + self.assertNotIn("mooncake.queue_wait", names) + self.assertNotIn("mooncake.wait_source", names) + self.assertNotIn("mooncake.wait_indices", names) + attrs = _span(tracer, "mooncake.kv_chunk").attributes + self.assertAlmostEqual(attrs["sglang.source_wait_ms"], 2.0) + + def test_queue_wait_split_matches_child_span(self): + """Two defers then a third attempt: 20 us total + 10 us attempt.""" + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + attempt_ns, total_ns = mod.split_queue_wait(20_000, 1_000_000, 1_010_000) + self.assertEqual(attempt_ns, 10_000) + self.assertEqual(total_ns, 30_000) + tracer, patcher = _fake_otel() + now = time.time_ns() + chunk = self._chunk( + last_enqueue_wall_ns=now - 10_000, + queue_wait_accum_ns=20_000, + ) + with patcher: + mod.emit_mooncake_chunk( + chunk=chunk, + status="ok", + queue_wait_ns=attempt_ns, + queue_wait_total_ns=total_ns, + source_wait_ns=0, + indices_wait_ns=0, + kv_transfer_ns=1_000, + state_transfer_ns=0, + aux_transfer_ns=0, + attempt_wall_ns=2_000, + dequeue_wall_ns=now, + end_wall_ns=now, + n_destinations=1, + ) + child = _span(tracer, "mooncake.queue_wait") + # Wall window == this attempt only; the accumulated value is an + # attribute on the parent, never on the child span. + self.assertEqual( + child.end_time - child.kwargs["start_time"], attempt_ns + ) + self.assertAlmostEqual( + child.attributes["sglang.wait_ms"], attempt_ns / 1e6 + ) + attrs = _span(tracer, "mooncake.kv_chunk").attributes + self.assertAlmostEqual(attrs["sglang.queue_wait_ms"], attempt_ns / 1e6) + self.assertAlmostEqual( + attrs["sglang.queue_wait_total_ms"], total_ns / 1e6 + ) + + def test_requeue_accumulates_only_queue_wait(self): + """Two consecutive defers must not count worker/watermark time.""" + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + chunk = types.SimpleNamespace( + observe=True, + queue_wait_accum_ns=0, + last_enqueue_mono_ns=1_000, + last_enqueue_wall_ns=2_000, + dequeue_mono_ns=11_000, # attempt 1: 10 ms queue wait + requeue_count=0, + ) + # Worker + watermark consumed 30 ms; only the 10 ms queue wait counts. + mod.note_chunk_requeue(chunk, 41_000, 42_000) + self.assertEqual(chunk.queue_wait_accum_ns, 10_000) + self.assertEqual(chunk.requeue_count, 1) + self.assertEqual(chunk.last_enqueue_mono_ns, 41_000) + self.assertEqual(chunk.last_enqueue_wall_ns, 42_000) + self.assertEqual(chunk.dequeue_mono_ns, 0) + + # Attempt 2: dequeued 10 ms later, then another defer. + chunk.dequeue_mono_ns = 51_000 + mod.note_chunk_requeue(chunk, 91_000, 92_000) + self.assertEqual(chunk.queue_wait_accum_ns, 20_000) + self.assertEqual(chunk.requeue_count, 2) + + def test_requeue_disabled_is_noop(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="0") + chunk = types.SimpleNamespace( + observe=True, + queue_wait_accum_ns=0, + last_enqueue_mono_ns=1_000, + last_enqueue_wall_ns=2_000, + dequeue_mono_ns=11_000, + requeue_count=0, + ) + mod.note_chunk_requeue(chunk, 41_000, 42_000) + self.assertEqual(chunk.queue_wait_accum_ns, 0) + self.assertEqual(chunk.requeue_count, 0) + + +class RequestJoinSpanTest(unittest.TestCase): + def test_request_join_span(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + tracer, patcher = _fake_otel() + now_wall = time.time_ns() + req = types.SimpleNamespace(rid="req-1", bootstrap_room=42) + stats = { + "chunk_count": 4, + "first_enqueue_wall_ns": now_wall - 600_000_000, + "last_chunk_enqueue_wall_ns": now_wall - 30_000_000, + "last_chunk_complete_wall_ns": now_wall - 10_000_000, + "last_chunk_queue_wait_ms": 2.5, + "last_chunk_worker_ms": 8.0, + } + with patcher: + mod.emit_request_kv_join( + req=req, stats=stats, dp_rank=3, completion_wall_ns=now_wall + ) + span = _span(tracer, "kv.request_join") + attrs = span.attributes + self.assertEqual(attrs["sglang.bootstrap_room"], 42) + self.assertEqual(attrs["sglang.chunk_count"], 4) + self.assertAlmostEqual(attrs["sglang.final_exposed_ms"], 20.0) + self.assertAlmostEqual(attrs["sglang.completion_poll_ms"], 10.0) + self.assertAlmostEqual(attrs["sglang.request_join_ms"], 30.0) + self.assertAlmostEqual( + attrs["sglang.first_enqueue_to_completion_ms"], 600.0, delta=1.0 + ) + self.assertEqual( + span.kwargs["start_time"], stats["first_enqueue_wall_ns"] + ) + self.assertEqual(len(span.kwargs.get("links") or []), 0) + + +class RoomDetailStateTest(unittest.TestCase): + def test_begin_chunk_is_sequential_and_keeps_first_enqueue(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + state = mod.RoomDetailState() + seq0, first0 = state.begin_chunk(7, 1_000, 2_000) + seq1, first1 = state.begin_chunk(7, 5_000, 6_000) + self.assertEqual((seq0, seq1), (0, 1)) + self.assertEqual(first0, 2_000) + self.assertEqual(first1, 2_000) # first enqueue of the room + + def test_abort_between_status_check_and_join_write_cannot_resurrect(self): + """Deterministic interleaving of the abort/worker race (review).""" + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + state = mod.RoomDetailState() + state.begin_chunk(7, 1_000, 2_000) + # Worker saw status ok, then the scheduler aborts before the write. + state.reset(7) + written = state.record_join( + 7, + status="ok", + room_failed=False, # stale status read from before the abort + now_wall_ns=3_000, + queue_wait_total_ns=10, + worker_busy_ns=20, + enqueue_wall_ns=2_000, + is_last_chunk=True, + chunk_seq=0, + ) + self.assertFalse(written) + self.assertIsNone(state.pop_join(7)) + + def test_join_write_then_abort_is_cleaned(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + state = mod.RoomDetailState() + state.begin_chunk(7, 1_000, 2_000) + self.assertTrue( + state.record_join( + 7, + status="ok", + room_failed=False, + now_wall_ns=3_000, + queue_wait_total_ns=10, + worker_busy_ns=20, + enqueue_wall_ns=2_000, + is_last_chunk=True, + chunk_seq=0, + ) + ) + state.reset(7) + self.assertIsNone(state.pop_join(7)) + + def test_failed_worker_never_writes_join(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + state = mod.RoomDetailState() + state.begin_chunk(7, 1_000, 2_000) + self.assertFalse( + state.record_join( + 7, + status="failed", + room_failed=True, + now_wall_ns=3_000, + queue_wait_total_ns=0, + worker_busy_ns=0, + enqueue_wall_ns=2_000, + is_last_chunk=True, + chunk_seq=0, + ) + ) + self.assertIsNone(state.pop_join(7)) + + def test_retry_clears_cancellation_mark(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + state = mod.RoomDetailState() + state.begin_chunk(7, 1_000, 2_000) + state.reset(7) + # A retry that reuses the room starts a fresh flow. + state.begin_chunk(7, 10_000, 11_000) + self.assertTrue( + state.record_join( + 7, + status="ok", + room_failed=False, + now_wall_ns=12_000, + queue_wait_total_ns=0, + worker_busy_ns=1, + enqueue_wall_ns=11_000, + is_last_chunk=False, + chunk_seq=0, + ) + ) + self.assertEqual(state.pop_join(7)["chunk_count"], 1) + + def test_cancellation_marks_are_bounded(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + state = mod.RoomDetailState() + for room in range(mod.RoomDetailState._MAX_CANCELLED + 10): + state.reset(room) + self.assertLessEqual( + len(state._cancelled), mod.RoomDetailState._MAX_CANCELLED + ) + + def test_teardown_keeps_success_join_and_drops_failed(self): + mod = _load(SGLANG_TRACE_MOONCAKE_DETAIL="1") + state = mod.RoomDetailState() + state.begin_chunk(1, 1_000, 2_000) + state.record_join( + 1, + status="ok", + room_failed=False, + now_wall_ns=3_000, + queue_wait_total_ns=0, + worker_busy_ns=1, + enqueue_wall_ns=2_000, + is_last_chunk=False, + chunk_seq=0, + ) + state.teardown(1, keep_join=True) + self.assertIsNotNone(state.pop_join(1)) + + state.begin_chunk(2, 1_000, 2_000) + state.record_join( + 2, + status="ok", + room_failed=False, + now_wall_ns=3_000, + queue_wait_total_ns=0, + worker_busy_ns=1, + enqueue_wall_ns=2_000, + is_last_chunk=False, + chunk_seq=0, + ) + state.teardown(2, keep_join=False) + self.assertIsNone(state.pop_join(2)) + + +class ChunkFieldDefaultsTest(unittest.TestCase): + """The observation fields must be additive defaults on TransferKVChunk.""" + + def test_dataclass_defaults(self): + import dataclasses + + path = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/disaggregation/common/utils.py" + ) + trace_stub = types.ModuleType("sglang.srt.observability.trace") + + class _Null: + pass + + trace_stub.TraceNullContext = _Null + trace_stub.TraceReqContext = _Null + patcher = mock.patch.dict( + sys.modules, + { + "sglang": types.ModuleType("sglang"), + "sglang.srt": types.ModuleType("sglang.srt"), + "sglang.srt.observability": types.ModuleType( + "sglang.srt.observability" + ), + "sglang.srt.observability.trace": trace_stub, + }, + ) + with patcher: + spec = importlib.util.spec_from_file_location("kv_utils_under_test", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + + fields = {f.name: f for f in dataclasses.fields(mod.TransferKVChunk)} + for name in ( + "chunk_seq", + "loop_seq", + "send_kind", + "enqueue_mono_ns", + "enqueue_wall_ns", + "last_enqueue_mono_ns", + "last_enqueue_wall_ns", + "dequeue_mono_ns", + "queue_wait_accum_ns", + "queue_id", + "requeue_count", + "num_pages", + "approx_bytes", + "observe", + ): + self.assertIn(name, fields) + self.assertFalse(fields[name].init is None) + chunk = mod.TransferKVChunk( + room=1, + prefill_kv_indices=None, + index_slice=slice(0, 1), + is_last_chunk=True, + prefill_aux_index=None, + state_indices=None, + chunk_seq=3, + queue_id=2, + observe=True, + ) + self.assertEqual(chunk.chunk_seq, 3) + self.assertTrue(chunk.observe) + self.assertEqual(chunk.requeue_count, 0) + + +if __name__ == "__main__": + unittest.main()