[PD] Wait for prefill completion before Mooncake early KV transfer (#41395)
Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Shangming Cai
Claude Opus 5
parent
8c273d6813
commit
c683190b40
@@ -32,7 +32,7 @@ class TransferKVChunk:
|
||||
# Set when the staging worker first counts this chunk toward the per-room
|
||||
# outstanding count; stays set across re-enqueue on a watermark defer.
|
||||
staging_counted: bool = False
|
||||
# Mori early-send: CUDA event to synchronize before RDMA (optional).
|
||||
# Early-send: GPU event to synchronize before reading KV (optional).
|
||||
wait_event: Optional[object] = None
|
||||
|
||||
|
||||
|
||||
@@ -2331,6 +2331,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._staging_outstanding[kv_chunk.room] += 1
|
||||
kv_chunk.staging_counted = True
|
||||
|
||||
# Under overlap scheduling an early cached-prefix send can
|
||||
# precede completion of the previous prefill forward. Wait on
|
||||
# the worker, not the scheduler, before reading source KV.
|
||||
# A room that already failed skips the wait; the status check
|
||||
# below then drops the chunk, including on an abort during it.
|
||||
if kv_chunk.wait_event is not None:
|
||||
if (
|
||||
kv_chunk.room in self.request_status
|
||||
and self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
):
|
||||
kv_chunk.wait_event.synchronize()
|
||||
kv_chunk.wait_event = None
|
||||
|
||||
if (
|
||||
kv_chunk.room not in self.request_status
|
||||
or self.check_status(kv_chunk.room) == KVPoll.Failed
|
||||
@@ -2882,6 +2895,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
wait_event: Optional[object] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -2921,6 +2935,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=trace_ctx,
|
||||
wait_event=wait_event,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3022,6 +3037,8 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
if should_skip:
|
||||
return
|
||||
|
||||
wait_event = getattr(self, "_early_send_wait_event", None)
|
||||
self._early_send_wait_event = None
|
||||
if not is_last_chunk:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
@@ -3030,6 +3047,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
False,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
else:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
@@ -3041,6 +3059,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user