[PD] Poison the deferred abort ack when a Mooncake send fails (#43719)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
da4bf1ef8e
commit
4195980f5a
@@ -910,6 +910,10 @@ class CommonKVManager(BaseKVManager):
|
||||
(outstanding == 0). pop() makes it fire at most once."""
|
||||
if self._staging_outstanding.get(room, 0) > 0:
|
||||
return
|
||||
if room in self._deferred_ack_poisoned_rooms:
|
||||
# A target registered concurrently with the poison must not fire.
|
||||
self._deferred_ack_targets.pop(room, None)
|
||||
return
|
||||
self._send_abort_acks(room, self._deferred_ack_targets.pop(room, None))
|
||||
|
||||
def _send_abort_acks(
|
||||
|
||||
@@ -2292,6 +2292,16 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
return self._transfer_data(req.mooncake_session_id, transfer_blocks)
|
||||
|
||||
def _poison_unsettled_send(self, room: int) -> None:
|
||||
"""Keep the drain ack from firing after a submitted transfer failed.
|
||||
|
||||
batch_transfer_sync returns -1 on its wall-clock timeout with the batch
|
||||
still running (freeBatchID refuses a busy batch and the handle is
|
||||
dropped), so a failed send may still be writing decode's pages; decode
|
||||
falls back to its release timeout instead."""
|
||||
if self.enable_deferred_decode_kv_release:
|
||||
self.poison_deferred_ack_room(room)
|
||||
|
||||
def transfer_worker(
|
||||
self,
|
||||
queue: FastQueue,
|
||||
@@ -2546,6 +2556,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
logger.error(
|
||||
f"Session {req.mooncake_session_id} failed."
|
||||
)
|
||||
self._poison_unsettled_send(kv_chunk.room)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=kv_chunk.room,
|
||||
failure_reason=(
|
||||
@@ -2563,6 +2574,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
executor,
|
||||
target_rank_registration_info,
|
||||
)
|
||||
# Before the Failed check: an ABORT during the
|
||||
# send also marks the room Failed.
|
||||
if state_rc != 0:
|
||||
self._poison_unsettled_send(kv_chunk.room)
|
||||
if (
|
||||
self.request_status.get(
|
||||
kv_chunk.room, KVPoll.Failed
|
||||
@@ -2594,6 +2609,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
kv_chunk.prefill_aux_index,
|
||||
target_rank_registration_info.dst_aux_ptrs,
|
||||
)
|
||||
if ret != 0:
|
||||
# The aux write targets decode's metadata
|
||||
# slot, which the deferred release also frees.
|
||||
self._poison_unsettled_send(req.room)
|
||||
polls.append(True if ret == 0 else False)
|
||||
dst_ranks_infos.append((req.endpoint, req.dst_port))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user