From 14ab74ef951e837da9d8a1dc5cb98292a21cd727 Mon Sep 17 00:00:00 2001 From: Shangming Cai Date: Mon, 28 Sep 2026 05:16:23 +0800 Subject: [PATCH] [PD] Fan drain abort ACKs out to every decode peer of the room (#41402) Co-authored-by: Claude Fable 5 --- .../sglang/srt/disaggregation/common/conn.py | 54 +++++++++++++- python/sglang/srt/managers/scheduler.py | 19 ++--- .../test_deferred_decode_kv_release.py | 70 ++++++++++++++++++- .../test_disaggregation_wire.py | 2 + .../test_nixl_deferred_kv_release.py | 70 +++++++++++++++++++ 5 files changed, 195 insertions(+), 20 deletions(-) diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 77e324b7be..594feaa278 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -262,6 +262,10 @@ class CommonKVManager(BaseKVManager): # Deferred KV release: aborted room -> (decode_ip, decode_port); # ack held until the transfer drains. self._deferred_ack_targets: Dict[int, Tuple[str, int]] = {} + # Fan-out peers snapshotted at registration: the sender's clear() + # can pop transfer_infos before the worker drains, and the drain + # ack would otherwise reach the registered target alone. + self._deferred_ack_fanout_snapshots: Dict[int, List[Tuple[str, int]]] = {} self.req_to_decode_prefix_len: Dict[int, int] = {} self.decode_kv_args_table = {} self.pp_group = get_parallel().pp_group @@ -682,14 +686,45 @@ class CommonKVManager(BaseKVManager): except Exception as e: logger.debug(f"Failed to send drained ABORT_ACK for room {room}: {e}") + def _abort_ack_fanout_targets(self, room: int) -> List[Tuple[str, int]]: + """Every decode peer of the room, dummy pairings included: each decode + rank counts drain acks from every prefill rank it notified, and a dummy + pairing still expects its ack (nothing was written, so it is trivially + drained). Snapshot: the control thread can register a late peer while + we walk the dict.""" + infos = self.transfer_infos.get(room) + if not infos: + return [] + targets: List[Tuple[str, int]] = [] + for info in list(infos.values()): + target = (info.endpoint, info.dst_port) + if target not in targets: + targets.append(target) + return targets + def _maybe_ack_drained_abort(self, room: int) -> None: """Send the deferred ack once an aborted room's chunks have drained (outstanding == 0). pop() makes it fire at most once.""" if self._staging_outstanding.get(room, 0) > 0: return target = self._deferred_ack_targets.pop(room, None) - if target is not None: - self._send_abort_ack(target[0], target[1], room) + if target is None: + return + # With prefill TP < decode TP several decode ranks share one room, but + # the registry keeps only the last ABORT sender -- the earlier ranks + # would hold their pages for the full release timeout. Fan the ack out + # to every known peer: the live transfer_infos (catches peers that + # registered after the ABORT), the snapshot taken at registration + # (survives a teardown that popped transfer_infos mid-flight), and the + # registered sender as the guaranteed floor. + targets = self._abort_ack_fanout_targets(room) + for peer in self._deferred_ack_fanout_snapshots.pop(room, ()): + if peer not in targets: + targets.append(peer) + if target not in targets: + targets.append(target) + for decode_ip, decode_port in targets: + self._send_abort_ack(decode_ip, decode_port, room) def register_deferred_ack_target( self, room: int, decode_ip: str, decode_port: int @@ -697,6 +732,10 @@ class CommonKVManager(BaseKVManager): """Hold this room's ack until its transfer drains. Callers must mark the room Failed FIRST -- registering while it still accepts chunks lets the worker ack, then a new chunk writes pages the decode already released.""" + # Snapshot before target: the worker pops the target first, so writing + # the target first would let a drain racing this registration consume + # the target and strand the snapshot written after it. + self._deferred_ack_fanout_snapshots[room] = self._abort_ack_fanout_targets(room) self._deferred_ack_targets[room] = (decode_ip, decode_port) def get_kv_replica_factor(self) -> int: @@ -1665,6 +1704,9 @@ class CommonKVSender(BaseKVSender): self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room) else: self.kv_mgr._deferred_ack_targets.pop(self.bootstrap_room, None) + self.kv_mgr._deferred_ack_fanout_snapshots.pop( + self.bootstrap_room, None + ) def abort(self): self.kv_mgr.record_failure( @@ -1987,6 +2029,14 @@ class CommonKVReceiver(BaseKVReceiver): self.abort_notified = True def _send_abort_notification(self): + # Once metadata is published (init_time set) prefill may already be + # writing; arm the drain-ack tracker BEFORE the ABORT goes out, so an + # ack racing back -- or fanned out by a peer rank's earlier abort of + # the same room -- is counted instead of dropped. Prealloc-queue + # receivers (init_time None) never enter the deferred-release flow + # that would clean the tracker up, so they stay unarmed. + if self.kv_mgr.enable_deferred_decode_kv_release and self.init_time is not None: + self.kv_mgr.register_deferred_abort_room(self.bootstrap_room) for bootstrap_info in self.bootstrap_infos: # Best-effort notification to prefill side that this request was aborted. try: diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index e1ce5e02d4..0da72d79e1 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -5538,21 +5538,10 @@ class Scheduler( # Keep the host destination alive until prefill stops writing. prepare_abort(decode_req.req, "Aborted by AbortReq.") continue - receiver = decode_req.kv_receiver - receiver.abort() - # Arm drain-ack accounting once the ABORT is sent, so acks - # arriving before this req is deferred (e.g. during the next - # forward step) are captured. A fresh set also drops stale acks - # from a prior request that reused this bootstrap_room. A - # redundant abort only re-wipes -- holds longer, never releases - # early -- so no transition guard is needed. - if ( - receiver.kv_mgr.enable_deferred_decode_kv_release - and receiver.abort_notified - ): - receiver.kv_mgr.register_deferred_abort_room( - decode_req.req.bootstrap_room - ) + # The receiver arms drain-ack accounting before sending the + # ABORT (see CommonKVReceiver._send_abort_notification), so + # an ack racing back is never dropped. + decode_req.kv_receiver.abort() # Abort requests whose KV is already backed up for retraction. if self.disagg_decode_prealloc_queue.retracted_queue: diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index f3deecd390..03b59d238f 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -8,13 +8,14 @@ that its transfer drained (CommonKVManager.is_abort_release_safe). Device destinations may also release on timeout; host destinations require the ack. """ +import threading import unittest from types import SimpleNamespace from unittest.mock import patch from sglang.srt.disaggregation import decode as decode_mod from sglang.srt.disaggregation.base.conn import BaseKVManager -from sglang.srt.disaggregation.common.conn import CommonKVManager +from sglang.srt.disaggregation.common.conn import CommonKVManager, CommonKVReceiver from sglang.srt.disaggregation.decode import DecodeTransferQueue from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager from sglang.srt.disaggregation.nixl.conn import NixlKVManager @@ -117,6 +118,69 @@ class TestAbortAckAggregation(CustomTestCase): self.assertFalse(mgr.is_abort_release_safe(room, required_acks=2)) +class _BareReceiver(CommonKVReceiver): + """Concrete shell: the ABC check blocks CommonKVReceiver.__new__.""" + + def poll(self): + raise NotImplementedError + + def failure_exception(self): + raise NotImplementedError + + +class TestAbortArmsTrackerBeforeSend(CustomTestCase): + """Prefill can ack the moment the ABORT lands (already-drained room), and a + peer rank's earlier abort of the same room can fan an ack out even sooner; + an ack arriving before the tracker is armed is dropped and the rank waits + out the full release timeout. So the receiver must arm BEFORE sending.""" + + def _abort_receiver(self, mgr, init_time): + recv = _BareReceiver.__new__(_BareReceiver) + recv.kv_mgr = mgr + recv.bootstrap_room = 500 + recv.init_time = init_time + recv.bootstrap_infos = [{"rank_ip": "10.0.0.9", "rank_port": 7000}] + armed_at_send = [] + sock = SimpleNamespace( + send_multipart=lambda parts: armed_at_send.append( + 500 in mgr._deferred_abort_ack_tracker + ) + ) + recv._connect_to_bootstrap_server = lambda info: (sock, threading.Lock()) + return recv, armed_at_send + + def _make_decode_manager(self, enabled=True): + mgr = _make_manager() + mgr.enable_deferred_decode_kv_release = enabled + mgr.local_ip, mgr.rank_port = "10.0.0.1", 6000 + return mgr + + def test_tracker_armed_before_the_abort_is_sent(self): + mgr = self._make_decode_manager() + recv, armed_at_send = self._abort_receiver(mgr, init_time=123.0) + recv._send_abort_notification() + self.assertEqual(armed_at_send, [True]) + + def test_prealloc_abort_does_not_arm(self): + # A receiver that never published metadata (init_time None) does not + # enter the deferred-release flow that cleans the tracker up; arming + # it would leak one set per aborted prealloc request. + mgr = self._make_decode_manager() + recv, armed_at_send = self._abort_receiver(mgr, init_time=None) + recv._send_abort_notification() + self.assertEqual(armed_at_send, [False]) + self.assertNotIn(500, mgr._deferred_abort_ack_tracker) + + def test_opted_out_backend_does_not_arm(self): + # Backends without a drain ack (mori) send the ABORT but must not arm: + # nothing would ever clean the tracker up. + mgr = self._make_decode_manager(enabled=False) + recv, armed_at_send = self._abort_receiver(mgr, init_time=123.0) + recv._send_abort_notification() + self.assertEqual(armed_at_send, [False]) + self.assertNotIn(500, mgr._deferred_abort_ack_tracker) + + class _FakeIdxAllocator: def __init__(self): self.freed = [] @@ -196,8 +260,8 @@ class TestResolveDeferredReleases(CustomTestCase): room, idx = 200, 7 q = _make_queue() dreq = _make_decode_req(room, idx, mgr, n_prefill_ranks=2) - # In production the room is armed in abort_request when the ABORT is - # sent, before the scheduler defers here. + # In production the receiver arms the room just before it sends the + # ABORT (_send_abort_notification), before the scheduler defers here. mgr.register_deferred_abort_room(room) q._defer_release(dreq) diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index ccd765278f..e85c0a94e6 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -80,6 +80,8 @@ class TestDisaggregationWire(unittest.TestCase): manager.request_status = {42: KVPoll.Failed} manager._staging_outstanding = {42: outstanding} manager._deferred_ack_targets = {42: ("127.0.0.1", 1234)} + manager._deferred_ack_fanout_snapshots = {} + manager.transfer_infos = {} with patch.object(manager, "_send_abort_ack") as ack: sender.clear() if outstanding: diff --git a/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py b/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py index 752787665e..d8e8823e43 100644 --- a/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py +++ b/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py @@ -8,6 +8,7 @@ after its DONE barrier -- never from the bootstrap thread for an active room. """ import unittest +from types import SimpleNamespace from unittest.mock import MagicMock from sglang.srt.disaggregation.base.conn import KVPoll @@ -24,8 +25,10 @@ def _prefill_mgr(cls=CommonKVManager, enabled=True): mgr = cls.__new__(cls) mgr.enable_deferred_decode_kv_release = enabled mgr._deferred_ack_targets = {} + mgr._deferred_ack_fanout_snapshots = {} mgr._staging_outstanding = {} mgr.request_status = {} + mgr.transfer_infos = {} mgr._sent = [] # Capture acks instead of opening a socket. mgr._send_abort_ack = lambda ip, port, room: mgr._sent.append((ip, port, room)) @@ -58,6 +61,73 @@ class TestDeferredAckTargets(CustomTestCase): mgr._maybe_ack_drained_abort(999) self.assertEqual(mgr._sent, []) + def test_drain_ack_fans_out_to_every_room_peer(self): + """With prefill TP < decode TP the room has several decode peers but the + registry keeps only the last ABORT sender; the drain ack must reach every + peer (dummy pairings included) or the others hold until the timeout.""" + mgr = _prefill_mgr() + mgr.transfer_infos[7] = { + "sess0": SimpleNamespace(endpoint="10.0.0.1", dst_port=5000), + "sess1": SimpleNamespace(endpoint="10.0.0.2", dst_port=5001), + # Dummy pairing: the decode rank still counts this prefill's ack. + "sess2": SimpleNamespace(endpoint="10.0.0.3", dst_port=5002), + } + # Rank 1 registered last and overwrote rank 0's registration. + mgr.register_deferred_ack_target(7, "10.0.0.2", 5001) + + mgr._maybe_ack_drained_abort(7) + self.assertEqual( + sorted(mgr._sent), + [ + ("10.0.0.1", 5000, 7), + ("10.0.0.2", 5001, 7), + ("10.0.0.3", 5002, 7), + ], + ) + + # pop() semantics survive the fan-out: a second drain acks nobody. + mgr._maybe_ack_drained_abort(7) + self.assertEqual(len(mgr._sent), 3) + + def test_drain_ack_fanout_survives_mid_flight_teardown(self): + """The sender's clear() can pop transfer_infos while a chunk is still in + flight; the peers snapshotted when the ABORT registered must still be + acked when the worker finally drains, or they hold until the timeout.""" + mgr = _prefill_mgr() + mgr.transfer_infos[7] = { + "sess0": SimpleNamespace(endpoint="10.0.0.1", dst_port=5000), + "sess1": SimpleNamespace(endpoint="10.0.0.2", dst_port=5001), + } + mgr._staging_outstanding[7] = 1 + mgr.register_deferred_ack_target(7, "10.0.0.2", 5001) + mgr._maybe_ack_drained_abort(7) + self.assertEqual(mgr._sent, []) # still writing -> held + + # Scheduler clears the sender mid-flight; the worker drains after. + mgr.transfer_infos.clear() + mgr._staging_outstanding[7] = 0 + mgr._maybe_ack_drained_abort(7) + self.assertEqual( + sorted(mgr._sent), + [("10.0.0.1", 5000, 7), ("10.0.0.2", 5001, 7)], + ) + self.assertNotIn(7, mgr._deferred_ack_fanout_snapshots) + + def test_drain_ack_after_teardown_falls_back_to_registered_target(self): + mgr = _prefill_mgr() + mgr.register_deferred_ack_target(9, "10.0.0.4", 5003) + mgr._maybe_ack_drained_abort(9) + self.assertEqual(mgr._sent, [("10.0.0.4", 5003, 9)]) + + def test_populated_room_without_registration_acks_nobody(self): + """A normal (non-aborted) drain must not fan out spurious acks.""" + mgr = _prefill_mgr() + mgr.transfer_infos[12] = { + "sess0": SimpleNamespace(endpoint="10.0.0.5", dst_port=5004), + } + mgr._maybe_ack_drained_abort(12) + self.assertEqual(mgr._sent, []) + def test_prefill_unique_rank_matches_success_sync_formula(self): mgr = CommonKVManager.__new__(CommonKVManager) mgr.attn_tp_rank, mgr.pp_size, mgr.attn_cp_size = 2, 3, 4