[PD] Poison the deferred abort ack when a Mooncake send fails (#43719)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Shangming Cai
2026-10-12 15:33:09 +08:00
committed by GitHub
co-authored by Claude Fable 5.1
parent da4bf1ef8e
commit 4195980f5a
5 changed files with 191 additions and 1 deletions
@@ -910,6 +910,10 @@ class CommonKVManager(BaseKVManager):
(outstanding == 0). pop() makes it fire at most once."""
if self._staging_outstanding.get(room, 0) > 0:
return
if room in self._deferred_ack_poisoned_rooms:
# A target registered concurrently with the poison must not fire.
self._deferred_ack_targets.pop(room, None)
return
self._send_abort_acks(room, self._deferred_ack_targets.pop(room, None))
def _send_abort_acks(
@@ -2292,6 +2292,16 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
return self._transfer_data(req.mooncake_session_id, transfer_blocks)
def _poison_unsettled_send(self, room: int) -> None:
"""Keep the drain ack from firing after a submitted transfer failed.
batch_transfer_sync returns -1 on its wall-clock timeout with the batch
still running (freeBatchID refuses a busy batch and the handle is
dropped), so a failed send may still be writing decode's pages; decode
falls back to its release timeout instead."""
if self.enable_deferred_decode_kv_release:
self.poison_deferred_ack_room(room)
def transfer_worker(
self,
queue: FastQueue,
@@ -2546,6 +2556,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
logger.error(
f"Session {req.mooncake_session_id} failed."
)
self._poison_unsettled_send(kv_chunk.room)
self.conclude_failure(
bootstrap_room=kv_chunk.room,
failure_reason=(
@@ -2563,6 +2574,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
executor,
target_rank_registration_info,
)
# Before the Failed check: an ABORT during the
# send also marks the room Failed.
if state_rc != 0:
self._poison_unsettled_send(kv_chunk.room)
if (
self.request_status.get(
kv_chunk.room, KVPoll.Failed
@@ -2594,6 +2609,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
kv_chunk.prefill_aux_index,
target_rank_registration_info.dst_aux_ptrs,
)
if ret != 0:
# The aux write targets decode's metadata
# slot, which the deferred release also frees.
self._poison_unsettled_send(req.room)
polls.append(True if ret == 0 else False)
dst_ranks_infos.append((req.endpoint, req.dst_port))