[PD] Fan drain abort ACKs out to every decode peer of the room (#41402)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
b252aceffe
commit
14ab74ef95
@@ -262,6 +262,10 @@ class CommonKVManager(BaseKVManager):
|
||||
# Deferred KV release: aborted room -> (decode_ip, decode_port);
|
||||
# ack held until the transfer drains.
|
||||
self._deferred_ack_targets: Dict[int, Tuple[str, int]] = {}
|
||||
# Fan-out peers snapshotted at registration: the sender's clear()
|
||||
# can pop transfer_infos before the worker drains, and the drain
|
||||
# ack would otherwise reach the registered target alone.
|
||||
self._deferred_ack_fanout_snapshots: Dict[int, List[Tuple[str, int]]] = {}
|
||||
self.req_to_decode_prefix_len: Dict[int, int] = {}
|
||||
self.decode_kv_args_table = {}
|
||||
self.pp_group = get_parallel().pp_group
|
||||
@@ -682,14 +686,45 @@ class CommonKVManager(BaseKVManager):
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to send drained ABORT_ACK for room {room}: {e}")
|
||||
|
||||
def _abort_ack_fanout_targets(self, room: int) -> List[Tuple[str, int]]:
|
||||
"""Every decode peer of the room, dummy pairings included: each decode
|
||||
rank counts drain acks from every prefill rank it notified, and a dummy
|
||||
pairing still expects its ack (nothing was written, so it is trivially
|
||||
drained). Snapshot: the control thread can register a late peer while
|
||||
we walk the dict."""
|
||||
infos = self.transfer_infos.get(room)
|
||||
if not infos:
|
||||
return []
|
||||
targets: List[Tuple[str, int]] = []
|
||||
for info in list(infos.values()):
|
||||
target = (info.endpoint, info.dst_port)
|
||||
if target not in targets:
|
||||
targets.append(target)
|
||||
return targets
|
||||
|
||||
def _maybe_ack_drained_abort(self, room: int) -> None:
|
||||
"""Send the deferred ack once an aborted room's chunks have drained
|
||||
(outstanding == 0). pop() makes it fire at most once."""
|
||||
if self._staging_outstanding.get(room, 0) > 0:
|
||||
return
|
||||
target = self._deferred_ack_targets.pop(room, None)
|
||||
if target is not None:
|
||||
self._send_abort_ack(target[0], target[1], room)
|
||||
if target is None:
|
||||
return
|
||||
# With prefill TP < decode TP several decode ranks share one room, but
|
||||
# the registry keeps only the last ABORT sender -- the earlier ranks
|
||||
# would hold their pages for the full release timeout. Fan the ack out
|
||||
# to every known peer: the live transfer_infos (catches peers that
|
||||
# registered after the ABORT), the snapshot taken at registration
|
||||
# (survives a teardown that popped transfer_infos mid-flight), and the
|
||||
# registered sender as the guaranteed floor.
|
||||
targets = self._abort_ack_fanout_targets(room)
|
||||
for peer in self._deferred_ack_fanout_snapshots.pop(room, ()):
|
||||
if peer not in targets:
|
||||
targets.append(peer)
|
||||
if target not in targets:
|
||||
targets.append(target)
|
||||
for decode_ip, decode_port in targets:
|
||||
self._send_abort_ack(decode_ip, decode_port, room)
|
||||
|
||||
def register_deferred_ack_target(
|
||||
self, room: int, decode_ip: str, decode_port: int
|
||||
@@ -697,6 +732,10 @@ class CommonKVManager(BaseKVManager):
|
||||
"""Hold this room's ack until its transfer drains. Callers must mark the
|
||||
room Failed FIRST -- registering while it still accepts chunks lets the
|
||||
worker ack, then a new chunk writes pages the decode already released."""
|
||||
# Snapshot before target: the worker pops the target first, so writing
|
||||
# the target first would let a drain racing this registration consume
|
||||
# the target and strand the snapshot written after it.
|
||||
self._deferred_ack_fanout_snapshots[room] = self._abort_ack_fanout_targets(room)
|
||||
self._deferred_ack_targets[room] = (decode_ip, decode_port)
|
||||
|
||||
def get_kv_replica_factor(self) -> int:
|
||||
@@ -1665,6 +1704,9 @@ class CommonKVSender(BaseKVSender):
|
||||
self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room)
|
||||
else:
|
||||
self.kv_mgr._deferred_ack_targets.pop(self.bootstrap_room, None)
|
||||
self.kv_mgr._deferred_ack_fanout_snapshots.pop(
|
||||
self.bootstrap_room, None
|
||||
)
|
||||
|
||||
def abort(self):
|
||||
self.kv_mgr.record_failure(
|
||||
@@ -1987,6 +2029,14 @@ class CommonKVReceiver(BaseKVReceiver):
|
||||
self.abort_notified = True
|
||||
|
||||
def _send_abort_notification(self):
|
||||
# Once metadata is published (init_time set) prefill may already be
|
||||
# writing; arm the drain-ack tracker BEFORE the ABORT goes out, so an
|
||||
# ack racing back -- or fanned out by a peer rank's earlier abort of
|
||||
# the same room -- is counted instead of dropped. Prealloc-queue
|
||||
# receivers (init_time None) never enter the deferred-release flow
|
||||
# that would clean the tracker up, so they stay unarmed.
|
||||
if self.kv_mgr.enable_deferred_decode_kv_release and self.init_time is not None:
|
||||
self.kv_mgr.register_deferred_abort_room(self.bootstrap_room)
|
||||
for bootstrap_info in self.bootstrap_infos:
|
||||
# Best-effort notification to prefill side that this request was aborted.
|
||||
try:
|
||||
|
||||
@@ -5538,21 +5538,10 @@ class Scheduler(
|
||||
# Keep the host destination alive until prefill stops writing.
|
||||
prepare_abort(decode_req.req, "Aborted by AbortReq.")
|
||||
continue
|
||||
receiver = decode_req.kv_receiver
|
||||
receiver.abort()
|
||||
# Arm drain-ack accounting once the ABORT is sent, so acks
|
||||
# arriving before this req is deferred (e.g. during the next
|
||||
# forward step) are captured. A fresh set also drops stale acks
|
||||
# from a prior request that reused this bootstrap_room. A
|
||||
# redundant abort only re-wipes -- holds longer, never releases
|
||||
# early -- so no transition guard is needed.
|
||||
if (
|
||||
receiver.kv_mgr.enable_deferred_decode_kv_release
|
||||
and receiver.abort_notified
|
||||
):
|
||||
receiver.kv_mgr.register_deferred_abort_room(
|
||||
decode_req.req.bootstrap_room
|
||||
)
|
||||
# The receiver arms drain-ack accounting before sending the
|
||||
# ABORT (see CommonKVReceiver._send_abort_notification), so
|
||||
# an ack racing back is never dropped.
|
||||
decode_req.kv_receiver.abort()
|
||||
|
||||
# Abort requests whose KV is already backed up for retraction.
|
||||
if self.disagg_decode_prealloc_queue.retracted_queue:
|
||||
|
||||
Reference in New Issue
Block a user