observability: trace KV transfer chunks end-to-end (M2)
Implements M2 of handoff/prefill-cpu-kv-overlap-design-20261106.md, behind SGLANG_TRACE_MOONCAKE_DETAIL=1 (default off; reuses the existing trace stack). - TransferKVChunk gains optional observation fields: chunk_seq, loop_seq, send_kind, enqueue/last_enqueue monotonic+wall stamps, queue_wait_accum_ns, queue_id, requeue_count, num_pages, approx_bytes, observe - chunk_seq and enqueue timestamps are assigned at the real queue.put() point (after every early return), so skipped paths cannot leave holes - Mooncake transfer worker measures queue wait, source-event wait, indices wait, KV/state/aux transfer, worker busy and residual; emits one mooncake.kv_chunk span per attempt (+ queue/source/indices wait children) and updates a per-room join summary - request completion emits kv.request_join with chunk_count, last_chunk_queue_wait/worker, final_exposed_ms, completion_poll_ms, request_join_ms and first-enqueue-to-completion. First enqueue is stamped by the Mooncake manager at the real queue.put(); no ReqTimeStats field is added and the legacy queue_entry semantics are unchanged. - nixl/mori add_transfer_request accept the observation kwarg (no-op there) No scheduling/transfer semantics change; queue wait is excluded from worker busy, and worker_busy excludes source/indices event waits. Unit tests: test_mooncake_chunk_detail.py (15 cases, including the abort/worker interleaving and attempt-vs-total queue wait windows).
This commit is contained in:
@@ -51,6 +51,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.trace import TraceNullContext, TraceReqContext
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
@@ -2091,6 +2092,10 @@ class CommonKVSender(BaseKVSender):
|
||||
self._transfer_num_state_indices = 0
|
||||
self._transfer_start_time: Optional[float] = None
|
||||
self._early_send_wait_event: Optional[object] = None
|
||||
# Detail tracing: chunk classification only. Enqueue timestamps,
|
||||
# chunk_seq and the per-request first-enqueue are assigned by the
|
||||
# manager at the real queue.put() point, after all early returns.
|
||||
self._send_kind = ""
|
||||
self.trace_ctx = TraceNullContext()
|
||||
if self.kv_mgr.sender_trace_module and get_observability().enable_trace:
|
||||
self.trace_ctx = TraceReqContext(
|
||||
@@ -2289,6 +2294,19 @@ class CommonKVSender(BaseKVSender):
|
||||
):
|
||||
self._transfer_start_time = time.perf_counter()
|
||||
|
||||
obs = None
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
# Metadata only; the manager stamps chunk_seq and enqueue times
|
||||
# when it is certain the chunk is actually put on the queue.
|
||||
obs = {
|
||||
"loop_seq": int(batch_telemetry.current_loop_seq()),
|
||||
"send_kind": str(self._send_kind or ""),
|
||||
"num_pages": int(len(kv_indices)),
|
||||
"approx_bytes": int(
|
||||
len(kv_indices) * getattr(self.kv_mgr, "kv_item_lens_sum", 0)
|
||||
),
|
||||
}
|
||||
|
||||
# The early-send event belongs to exactly one chunk; the Mooncake and
|
||||
# Mori workers synchronize on it before reading source KV.
|
||||
wait_event = self._early_send_wait_event
|
||||
@@ -2304,9 +2322,27 @@ class CommonKVSender(BaseKVSender):
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
obs=obs,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
def pop_chunk_join_stats(self) -> Optional[dict]:
|
||||
"""Take (and free) this room's chunk join summary."""
|
||||
state = getattr(self.kv_mgr, "_room_detail", None)
|
||||
if state is None:
|
||||
return None
|
||||
return state.pop_join(self.bootstrap_room)
|
||||
|
||||
def reset_chunk_detail_state(self) -> None:
|
||||
"""Drop per-room detail state and mark it cancelled (abort / retract).
|
||||
|
||||
The mark is what prevents a worker attempt that already passed its
|
||||
status check from recreating stats after this cleanup.
|
||||
"""
|
||||
state = getattr(self.kv_mgr, "_room_detail", None)
|
||||
if state is not None:
|
||||
state.reset(self.bootstrap_room)
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
@@ -2425,6 +2461,9 @@ class CommonKVSender(BaseKVSender):
|
||||
def abort(self):
|
||||
if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
|
||||
return
|
||||
# Drop per-room detail bookkeeping: an aborted request never reaches
|
||||
# the poll path that normally consumes it (review finding).
|
||||
self.reset_chunk_detail_state()
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
"Aborted by AbortReq.",
|
||||
|
||||
@@ -36,6 +36,24 @@ class TransferKVChunk:
|
||||
# Must complete before reading asynchronously copied host indices.
|
||||
indices_event: Optional[object] = None
|
||||
|
||||
# --- Optional observation fields (SGLANG_TRACE_MOONCAKE_DETAIL=1) ------
|
||||
# Filled at the real enqueue point (add_transfer_request), never at the
|
||||
# send_kv_chunk entry, so skipped paths cannot create holes in chunk_seq.
|
||||
chunk_seq: int = 0 # per-sender monotonic enqueue index
|
||||
loop_seq: int = 0 # scheduler loop that produced this chunk
|
||||
send_kind: str = "" # cached_prefix / middle / last
|
||||
enqueue_mono_ns: int = 0 # first enqueue of this chunk, monotonic clock
|
||||
enqueue_wall_ns: int = 0 # first enqueue of this chunk, wall clock
|
||||
last_enqueue_mono_ns: int = 0 # updated on every (re-)enqueue
|
||||
last_enqueue_wall_ns: int = 0 # wall clock of the last (re-)enqueue
|
||||
dequeue_mono_ns: int = 0 # current attempt dequeue; 0 outside the worker
|
||||
queue_wait_accum_ns: int = 0 # completed queue waits from earlier attempts
|
||||
queue_id: int = -1
|
||||
requeue_count: int = 0
|
||||
num_pages: int = 0
|
||||
approx_bytes: int = 0
|
||||
observe: bool = False # True when detail tracing populated this chunk
|
||||
|
||||
|
||||
def pack_list_of_buffers(buffers: List[bytes]) -> bytes:
|
||||
if not buffers:
|
||||
|
||||
@@ -60,6 +60,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.observability.mooncake_trace import (
|
||||
MooncakeRequestStage,
|
||||
mooncake_trace_func,
|
||||
@@ -386,6 +387,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# transfer blocks over zmq when the intra-node NVLink transport is
|
||||
# active (it cannot reach host memory). Populated on bootstrap.
|
||||
self._session_endpoint_map = {}
|
||||
# Detail tracing (SGLANG_TRACE_MOONCAKE_DETAIL=1; empty otherwise).
|
||||
# Lock-protected per-room state: abort/retract and the worker's join
|
||||
# update must not interleave into resurrected stats.
|
||||
self._room_detail = batch_telemetry.RoomDetailState()
|
||||
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
|
||||
self.max_transfer_batch_indices = (
|
||||
envs.SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES.get()
|
||||
@@ -762,6 +767,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# block other rooms on this single worker thread.
|
||||
with self._staging_ctx.watermark_cv:
|
||||
self._staging_ctx.watermark_cv.wait(STAGING_WATERMARK_WAIT_S)
|
||||
if getattr(kv_chunk, "observe", False):
|
||||
batch_telemetry.note_chunk_requeue(
|
||||
kv_chunk, time.monotonic_ns(), time.time_ns()
|
||||
)
|
||||
queue.put(kv_chunk)
|
||||
return (-1, True)
|
||||
|
||||
@@ -2716,6 +2725,65 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if self.enable_deferred_decode_kv_release:
|
||||
self.poison_deferred_ack_room(room)
|
||||
|
||||
def _emit_chunk_attempt(
|
||||
self,
|
||||
kv_chunk: TransferKVChunk,
|
||||
*,
|
||||
status: str,
|
||||
queue_wait_ns: int,
|
||||
queue_wait_total_ns: int,
|
||||
source_wait_ns: int,
|
||||
indices_wait_ns: int,
|
||||
kv_transfer_ns: int,
|
||||
state_transfer_ns: int,
|
||||
aux_transfer_ns: int,
|
||||
work_start_ns: int,
|
||||
dequeue_wall_ns: int,
|
||||
n_destinations: int,
|
||||
source_wait_wall=None,
|
||||
indices_wait_wall=None,
|
||||
) -> None:
|
||||
"""Close one worker attempt: chunk span + per-room join summary."""
|
||||
now_mono_ns = time.monotonic_ns()
|
||||
now_wall_ns = time.time_ns()
|
||||
attempt_wall_ns = max(0, now_mono_ns - int(work_start_ns))
|
||||
worker_busy_ns = max(
|
||||
0, attempt_wall_ns - int(source_wait_ns) - int(indices_wait_ns)
|
||||
)
|
||||
batch_telemetry.emit_mooncake_chunk(
|
||||
chunk=kv_chunk,
|
||||
status=status,
|
||||
queue_wait_ns=queue_wait_ns,
|
||||
queue_wait_total_ns=queue_wait_total_ns,
|
||||
source_wait_ns=source_wait_ns,
|
||||
indices_wait_ns=indices_wait_ns,
|
||||
kv_transfer_ns=kv_transfer_ns,
|
||||
state_transfer_ns=state_transfer_ns,
|
||||
aux_transfer_ns=aux_transfer_ns,
|
||||
attempt_wall_ns=attempt_wall_ns,
|
||||
dequeue_wall_ns=dequeue_wall_ns,
|
||||
end_wall_ns=now_wall_ns,
|
||||
n_destinations=n_destinations,
|
||||
source_wait_wall=source_wait_wall,
|
||||
indices_wait_wall=indices_wait_wall,
|
||||
dp_rank=self.attn_dp_rank,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
)
|
||||
# The status was checked before this call; the registry re-checks the
|
||||
# cancellation mark under its lock, which is what makes an abort that
|
||||
# lands in between unable to resurrect stats (review finding).
|
||||
self._room_detail.record_join(
|
||||
kv_chunk.room,
|
||||
status=status,
|
||||
room_failed=self.check_status(kv_chunk.room) == KVPoll.Failed,
|
||||
now_wall_ns=now_wall_ns,
|
||||
queue_wait_total_ns=queue_wait_total_ns,
|
||||
worker_busy_ns=worker_busy_ns,
|
||||
enqueue_wall_ns=kv_chunk.enqueue_wall_ns,
|
||||
is_last_chunk=kv_chunk.is_last_chunk,
|
||||
chunk_seq=kv_chunk.chunk_seq,
|
||||
)
|
||||
|
||||
def transfer_worker(
|
||||
self,
|
||||
queue: FastQueue,
|
||||
@@ -2740,6 +2808,34 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# _stopped alone can never wake a parked worker during a role switch.
|
||||
if kv_chunk is None:
|
||||
break
|
||||
_obs = bool(getattr(kv_chunk, "observe", False))
|
||||
_dequeue_mono_ns = time.monotonic_ns() if _obs else 0
|
||||
_dequeue_wall_ns = time.time_ns() if _obs else 0
|
||||
if _obs:
|
||||
# Needed by note_chunk_requeue(): queue wait ends here and
|
||||
# must not include worker/watermark time.
|
||||
kv_chunk.dequeue_mono_ns = _dequeue_mono_ns
|
||||
_queue_wait_ns = 0
|
||||
_source_wait_ns = 0
|
||||
_source_wait_wall = None
|
||||
_indices_wait_ns = 0
|
||||
_indices_wait_wall = None
|
||||
_kv_transfer_ns = 0
|
||||
_state_transfer_ns = 0
|
||||
_aux_transfer_ns = 0
|
||||
_n_dest = 0
|
||||
if _obs:
|
||||
# queue wait is not part of worker busy time (design §5.2).
|
||||
# Keep the attempt window (child span) separate from the
|
||||
# cumulative value across deferred attempts (review finding).
|
||||
(
|
||||
_attempt_queue_wait_ns,
|
||||
_queue_wait_ns,
|
||||
) = batch_telemetry.split_queue_wait(
|
||||
kv_chunk.queue_wait_accum_ns,
|
||||
kv_chunk.last_enqueue_mono_ns,
|
||||
_dequeue_mono_ns,
|
||||
)
|
||||
if self.enable_trace:
|
||||
kv_chunk.trace_ctx.rebuild_thread_context()
|
||||
kv_chunk.trace_ctx.trace_slice_start(
|
||||
@@ -2765,13 +2861,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
kv_chunk.room in self.request_status
|
||||
and self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
):
|
||||
_wait_t0 = time.monotonic_ns() if _obs else 0
|
||||
_wait_wall0 = time.time_ns() if _obs else 0
|
||||
kv_chunk.wait_event.synchronize()
|
||||
if _wait_t0:
|
||||
_source_wait_ns += time.monotonic_ns() - _wait_t0
|
||||
_wait_wall1 = time.time_ns()
|
||||
_source_wait_wall = (_wait_wall0, _wait_wall1)
|
||||
kv_chunk.wait_event = None
|
||||
_ind_t0 = time.monotonic_ns() if _obs else 0
|
||||
_ind_wall0 = time.time_ns() if _obs else 0
|
||||
if kv_chunk.indices_event is not None:
|
||||
kv_chunk.indices_event.synchronize()
|
||||
kv_chunk.indices_event = None
|
||||
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
|
||||
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
|
||||
if _ind_t0:
|
||||
# synchronize + host indices availability, per design §5.2.
|
||||
_indices_wait_ns += time.monotonic_ns() - _ind_t0
|
||||
_indices_wait_wall = (_ind_wall0, time.time_ns())
|
||||
|
||||
if (
|
||||
kv_chunk.room not in self.request_status
|
||||
@@ -2787,6 +2895,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
thread_finish_flag=True,
|
||||
)
|
||||
self._staging_outstanding.pop(kv_chunk.room, None)
|
||||
if _obs:
|
||||
self._emit_chunk_attempt(
|
||||
kv_chunk,
|
||||
status="skipped",
|
||||
queue_wait_ns=_attempt_queue_wait_ns,
|
||||
queue_wait_total_ns=_queue_wait_ns,
|
||||
source_wait_ns=_source_wait_ns,
|
||||
indices_wait_ns=_indices_wait_ns,
|
||||
kv_transfer_ns=_kv_transfer_ns,
|
||||
state_transfer_ns=_state_transfer_ns,
|
||||
aux_transfer_ns=_aux_transfer_ns,
|
||||
work_start_ns=_dequeue_mono_ns,
|
||||
dequeue_wall_ns=_dequeue_wall_ns,
|
||||
n_destinations=_n_dest,
|
||||
source_wait_wall=_source_wait_wall,
|
||||
indices_wait_wall=_indices_wait_wall,
|
||||
)
|
||||
if self.enable_deferred_decode_kv_release:
|
||||
# Skipped => nothing written for this aborted room; ack.
|
||||
self._maybe_ack_drained_abort(kv_chunk.room)
|
||||
@@ -2882,6 +3007,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
skip_kv, skip_state = self._get_dsa_cache_transfer_skip_flags(
|
||||
target_rank_registration_info
|
||||
)
|
||||
_kv_t0 = time.monotonic_ns() if _obs else 0
|
||||
if (
|
||||
len(kv_chunk.prefill_kv_indices) == 0
|
||||
or not self.kv_args.kv_data_ptrs
|
||||
@@ -2954,6 +3080,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if deferred:
|
||||
staging_deferred = True
|
||||
if _kv_t0:
|
||||
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
|
||||
_n_dest += 1
|
||||
# Chunk re-enqueued; stop processing remaining reqs for this chunk
|
||||
break
|
||||
else:
|
||||
@@ -2968,6 +3097,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
executor,
|
||||
target_rank_registration_info.dst_kv_layer_ids,
|
||||
)
|
||||
if _kv_t0:
|
||||
_kv_transfer_ns += time.monotonic_ns() - _kv_t0
|
||||
_n_dest += 1
|
||||
if ret != 0:
|
||||
with self.session_lock:
|
||||
self.session_failures[req.mooncake_session_id] += 1
|
||||
@@ -2989,12 +3121,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
if kv_chunk.is_last_chunk:
|
||||
if kv_chunk.state_indices and not skip_state:
|
||||
_state_t0 = time.monotonic_ns() if _obs else 0
|
||||
state_rc = self.maybe_send_extra(
|
||||
req,
|
||||
kv_chunk.state_indices,
|
||||
executor,
|
||||
target_rank_registration_info,
|
||||
)
|
||||
if _state_t0:
|
||||
_state_transfer_ns += time.monotonic_ns() - _state_t0
|
||||
# Before the Failed check: an ABORT during the
|
||||
# send also marks the room Failed.
|
||||
if state_rc != 0:
|
||||
@@ -3025,11 +3160,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
break
|
||||
|
||||
# Only the last chunk we need to send the aux data
|
||||
_aux_t0 = time.monotonic_ns() if _obs else 0
|
||||
ret = self.send_aux(
|
||||
req,
|
||||
kv_chunk.prefill_aux_index,
|
||||
target_rank_registration_info.dst_aux_ptrs,
|
||||
)
|
||||
if _aux_t0:
|
||||
_aux_transfer_ns += time.monotonic_ns() - _aux_t0
|
||||
if ret != 0:
|
||||
# The aux write targets decode's metadata
|
||||
# slot, which the deferred release also frees.
|
||||
@@ -3071,8 +3209,45 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
|
||||
if staging_deferred:
|
||||
if _obs:
|
||||
self._emit_chunk_attempt(
|
||||
kv_chunk,
|
||||
status="deferred",
|
||||
queue_wait_ns=_attempt_queue_wait_ns,
|
||||
queue_wait_total_ns=_queue_wait_ns,
|
||||
source_wait_ns=_source_wait_ns,
|
||||
indices_wait_ns=_indices_wait_ns,
|
||||
kv_transfer_ns=_kv_transfer_ns,
|
||||
state_transfer_ns=_state_transfer_ns,
|
||||
aux_transfer_ns=_aux_transfer_ns,
|
||||
work_start_ns=_dequeue_mono_ns,
|
||||
dequeue_wall_ns=_dequeue_wall_ns,
|
||||
n_destinations=_n_dest,
|
||||
source_wait_wall=_source_wait_wall,
|
||||
indices_wait_wall=_indices_wait_wall,
|
||||
)
|
||||
continue
|
||||
|
||||
if _obs:
|
||||
self._emit_chunk_attempt(
|
||||
kv_chunk,
|
||||
status="ok"
|
||||
if self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
else "failed",
|
||||
queue_wait_ns=_attempt_queue_wait_ns,
|
||||
queue_wait_total_ns=_queue_wait_ns,
|
||||
source_wait_ns=_source_wait_ns,
|
||||
indices_wait_ns=_indices_wait_ns,
|
||||
kv_transfer_ns=_kv_transfer_ns,
|
||||
state_transfer_ns=_state_transfer_ns,
|
||||
aux_transfer_ns=_aux_transfer_ns,
|
||||
work_start_ns=_dequeue_mono_ns,
|
||||
dequeue_wall_ns=_dequeue_wall_ns,
|
||||
n_destinations=_n_dest,
|
||||
source_wait_wall=_source_wait_wall,
|
||||
indices_wait_wall=_indices_wait_wall,
|
||||
)
|
||||
|
||||
self._staging_outstanding[kv_chunk.room] -= 1
|
||||
if self.enable_deferred_decode_kv_release:
|
||||
# In-flight write finished; if aborted and nothing outstanding,
|
||||
@@ -3096,6 +3271,15 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._session_endpoint_map.pop(sid, None)
|
||||
self.transfer_infos.pop(kv_chunk.room)
|
||||
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
|
||||
# Success: the scheduler poll still needs the join summary.
|
||||
# Failed/absent: nothing will consume it, drop it here.
|
||||
self._room_detail.teardown(
|
||||
kv_chunk.room,
|
||||
keep_join=(
|
||||
self.request_status.get(kv_chunk.room)
|
||||
== KVPoll.Success
|
||||
),
|
||||
)
|
||||
if self.enable_staging:
|
||||
# Purge prefetch bookkeeping for the finished room.
|
||||
# Snapshot first: the scheduler thread adds concurrently.
|
||||
@@ -3332,6 +3516,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
wait_event: Optional[object] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
obs: Optional[dict] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -3361,6 +3546,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if trace_ctx is None:
|
||||
trace_ctx = TraceNullContext()
|
||||
|
||||
if obs is not None:
|
||||
# Stamp only now: every earlier return (failed room, dummy rank,
|
||||
# zero-page skip) must not consume a chunk_seq or a first-enqueue
|
||||
# timestamp (review finding).
|
||||
now_mono_ns = time.monotonic_ns()
|
||||
now_wall_ns = time.time_ns()
|
||||
chunk_seq, _first_wall_ns = self._room_detail.begin_chunk(
|
||||
bootstrap_room, now_mono_ns, now_wall_ns
|
||||
)
|
||||
obs = dict(obs)
|
||||
obs.update(
|
||||
chunk_seq=chunk_seq,
|
||||
enqueue_mono_ns=now_mono_ns,
|
||||
enqueue_wall_ns=now_wall_ns,
|
||||
last_enqueue_wall_ns=now_wall_ns,
|
||||
)
|
||||
|
||||
self.transfer_queues[shard_idx].put(
|
||||
TransferKVChunk(
|
||||
room=bootstrap_room,
|
||||
@@ -3373,6 +3575,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
trace_ctx=trace_ctx,
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
**(obs or {}),
|
||||
queue_id=int(shard_idx),
|
||||
observe=obs is not None,
|
||||
last_enqueue_mono_ns=int((obs or {}).get("enqueue_mono_ns", 0)),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -804,6 +804,7 @@ class MoriKVManager(CommonKVManager):
|
||||
wait_event: Optional[object] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
trace_ctx=None,
|
||||
obs: Optional[dict] = None,
|
||||
) -> None:
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
|
||||
@@ -3267,6 +3267,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
wait_event=None,
|
||||
indices_event=None,
|
||||
trace_ctx=None,
|
||||
obs: Optional[dict] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
|
||||
@@ -1426,11 +1426,30 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self.tree_cache.finish(
|
||||
req.cache_request_handle, CacheRequestOutcome.SUCCESS
|
||||
)
|
||||
# Consume the join summary before clear() releases the room.
|
||||
_join_stats = None
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
_pop_stats = getattr(
|
||||
req.disagg_kv_sender, "pop_chunk_join_stats", None
|
||||
)
|
||||
_join_stats = _pop_stats() if _pop_stats is not None else None
|
||||
# FIXME: clean up req's data in transfer engine
|
||||
req.disagg_kv_sender.clear()
|
||||
done_reqs.append(req)
|
||||
req.time_stats.set_prefill_kv_transfer_finish_time()
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
batch_telemetry.emit_request_kv_join(
|
||||
req=req,
|
||||
stats=_join_stats,
|
||||
dp_rank=get_parallel().attn_dp_rank,
|
||||
)
|
||||
elif poll == KVPoll.Failed:
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
_pop_stats = getattr(
|
||||
req.disagg_kv_sender, "pop_chunk_join_stats", None
|
||||
)
|
||||
if _pop_stats is not None:
|
||||
_pop_stats()
|
||||
self.handle_inflight_transfer_failure(req)
|
||||
done_reqs.append(req)
|
||||
else:
|
||||
@@ -2016,6 +2035,12 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
|
||||
"""Release KV cache and requeue an optimistic prefill request."""
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
# A retry may reuse the same room; stale chunk stats would
|
||||
# otherwise be merged into the next attempt's join summary.
|
||||
_reset = getattr(req.disagg_kv_sender, "reset_chunk_detail_state", None)
|
||||
if _reset is not None:
|
||||
_reset()
|
||||
max_attempts = get_disagg().optimistic_prefill_attempts
|
||||
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
|
||||
if not uses_write_through_cache:
|
||||
|
||||
Reference in New Issue
Block a user