[PD] Fan drain abort ACKs out to every decode peer of the room (#41402)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Shangming Cai
2026-09-28 05:16:23 +08:00
committed by GitHub
co-authored by Claude Fable 5
parent b252aceffe
commit 14ab74ef95
5 changed files with 195 additions and 20 deletions
@@ -262,6 +262,10 @@ class CommonKVManager(BaseKVManager):
# Deferred KV release: aborted room -> (decode_ip, decode_port);
# ack held until the transfer drains.
self._deferred_ack_targets: Dict[int, Tuple[str, int]] = {}
# Fan-out peers snapshotted at registration: the sender's clear()
# can pop transfer_infos before the worker drains, and the drain
# ack would otherwise reach the registered target alone.
self._deferred_ack_fanout_snapshots: Dict[int, List[Tuple[str, int]]] = {}
self.req_to_decode_prefix_len: Dict[int, int] = {}
self.decode_kv_args_table = {}
self.pp_group = get_parallel().pp_group
@@ -682,14 +686,45 @@ class CommonKVManager(BaseKVManager):
except Exception as e:
logger.debug(f"Failed to send drained ABORT_ACK for room {room}: {e}")
def _abort_ack_fanout_targets(self, room: int) -> List[Tuple[str, int]]:
"""Every decode peer of the room, dummy pairings included: each decode
rank counts drain acks from every prefill rank it notified, and a dummy
pairing still expects its ack (nothing was written, so it is trivially
drained). Snapshot: the control thread can register a late peer while
we walk the dict."""
infos = self.transfer_infos.get(room)
if not infos:
return []
targets: List[Tuple[str, int]] = []
for info in list(infos.values()):
target = (info.endpoint, info.dst_port)
if target not in targets:
targets.append(target)
return targets
def _maybe_ack_drained_abort(self, room: int) -> None:
"""Send the deferred ack once an aborted room's chunks have drained
(outstanding == 0). pop() makes it fire at most once."""
if self._staging_outstanding.get(room, 0) > 0:
return
target = self._deferred_ack_targets.pop(room, None)
if target is not None:
self._send_abort_ack(target[0], target[1], room)
if target is None:
return
# With prefill TP < decode TP several decode ranks share one room, but
# the registry keeps only the last ABORT sender -- the earlier ranks
# would hold their pages for the full release timeout. Fan the ack out
# to every known peer: the live transfer_infos (catches peers that
# registered after the ABORT), the snapshot taken at registration
# (survives a teardown that popped transfer_infos mid-flight), and the
# registered sender as the guaranteed floor.
targets = self._abort_ack_fanout_targets(room)
for peer in self._deferred_ack_fanout_snapshots.pop(room, ()):
if peer not in targets:
targets.append(peer)
if target not in targets:
targets.append(target)
for decode_ip, decode_port in targets:
self._send_abort_ack(decode_ip, decode_port, room)
def register_deferred_ack_target(
self, room: int, decode_ip: str, decode_port: int
@@ -697,6 +732,10 @@ class CommonKVManager(BaseKVManager):
"""Hold this room's ack until its transfer drains. Callers must mark the
room Failed FIRST -- registering while it still accepts chunks lets the
worker ack, then a new chunk writes pages the decode already released."""
# Snapshot before target: the worker pops the target first, so writing
# the target first would let a drain racing this registration consume
# the target and strand the snapshot written after it.
self._deferred_ack_fanout_snapshots[room] = self._abort_ack_fanout_targets(room)
self._deferred_ack_targets[room] = (decode_ip, decode_port)
def get_kv_replica_factor(self) -> int:
@@ -1665,6 +1704,9 @@ class CommonKVSender(BaseKVSender):
self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room)
else:
self.kv_mgr._deferred_ack_targets.pop(self.bootstrap_room, None)
self.kv_mgr._deferred_ack_fanout_snapshots.pop(
self.bootstrap_room, None
)
def abort(self):
self.kv_mgr.record_failure(
@@ -1987,6 +2029,14 @@ class CommonKVReceiver(BaseKVReceiver):
self.abort_notified = True
def _send_abort_notification(self):
# Once metadata is published (init_time set) prefill may already be
# writing; arm the drain-ack tracker BEFORE the ABORT goes out, so an
# ack racing back -- or fanned out by a peer rank's earlier abort of
# the same room -- is counted instead of dropped. Prealloc-queue
# receivers (init_time None) never enter the deferred-release flow
# that would clean the tracker up, so they stay unarmed.
if self.kv_mgr.enable_deferred_decode_kv_release and self.init_time is not None:
self.kv_mgr.register_deferred_abort_room(self.bootstrap_room)
for bootstrap_info in self.bootstrap_infos:
# Best-effort notification to prefill side that this request was aborted.
try:
+4 -15
View File
@@ -5538,21 +5538,10 @@ class Scheduler(
# Keep the host destination alive until prefill stops writing.
prepare_abort(decode_req.req, "Aborted by AbortReq.")
continue
receiver = decode_req.kv_receiver
receiver.abort()
# Arm drain-ack accounting once the ABORT is sent, so acks
# arriving before this req is deferred (e.g. during the next
# forward step) are captured. A fresh set also drops stale acks
# from a prior request that reused this bootstrap_room. A
# redundant abort only re-wipes -- holds longer, never releases
# early -- so no transition guard is needed.
if (
receiver.kv_mgr.enable_deferred_decode_kv_release
and receiver.abort_notified
):
receiver.kv_mgr.register_deferred_abort_room(
decode_req.req.bootstrap_room
)
# The receiver arms drain-ack accounting before sending the
# ABORT (see CommonKVReceiver._send_abort_notification), so
# an ack racing back is never dropped.
decode_req.kv_receiver.abort()
# Abort requests whose KV is already backed up for retraction.
if self.disagg_decode_prealloc_queue.retracted_queue: