[PD] Wait for prefill completion before Mooncake early KV transfer (#41395)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sasha Sidorov
2026-10-06 15:17:12 +08:00
committed by GitHub
co-authored by Shangming Cai Claude Opus 5
parent 8c273d6813
commit c683190b40
4 changed files with 160 additions and 6 deletions
@@ -32,7 +32,7 @@ class TransferKVChunk:
# Set when the staging worker first counts this chunk toward the per-room
# outstanding count; stays set across re-enqueue on a watermark defer.
staging_counted: bool = False
# Mori early-send: CUDA event to synchronize before RDMA (optional).
# Early-send: GPU event to synchronize before reading KV (optional).
wait_event: Optional[object] = None
@@ -2331,6 +2331,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self._staging_outstanding[kv_chunk.room] += 1
kv_chunk.staging_counted = True
# Under overlap scheduling an early cached-prefix send can
# precede completion of the previous prefill forward. Wait on
# the worker, not the scheduler, before reading source KV.
# A room that already failed skips the wait; the status check
# below then drops the chunk, including on an abort during it.
if kv_chunk.wait_event is not None:
if (
kv_chunk.room in self.request_status
and self.check_status(kv_chunk.room) != KVPoll.Failed
):
kv_chunk.wait_event.synchronize()
kv_chunk.wait_event = None
if (
kv_chunk.room not in self.request_status
or self.check_status(kv_chunk.room) == KVPoll.Failed
@@ -2882,6 +2895,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
state_indices: Optional[List] = None,
num_kv_tokens: Optional[int] = None,
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
wait_event: Optional[object] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -2921,6 +2935,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
state_indices=state_indices,
num_kv_tokens=num_kv_tokens,
trace_ctx=trace_ctx,
wait_event=wait_event,
)
)
@@ -3022,6 +3037,8 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
if should_skip:
return
wait_event = getattr(self, "_early_send_wait_event", None)
self._early_send_wait_event = None
if not is_last_chunk:
self.kv_mgr.add_transfer_request(
self.bootstrap_room,
@@ -3030,6 +3047,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
False,
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
)
else:
self.kv_mgr.add_transfer_request(
@@ -3041,6 +3059,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
state_indices=state_indices,
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
)
self._record_transfer_indices(kv_indices, state_indices)