[Fix] disaggregation: Poison the deferred abort ack when a NIXL or Mori write fails (#45973)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Banruo Liu
2026-10-23 14:47:59 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 6c844eb12e
commit 6e667c4310
4 changed files with 171 additions and 34 deletions
+17 -1
View File
@@ -603,9 +603,21 @@ class MoriKVManager(CommonKVManager):
self._staging_outstanding[room] -= 1
if self._staging_outstanding[room] <= 0:
self._staging_outstanding.pop(room)
targets = self._deferred_ack_targets.pop(room, None)
if room not in self._deferred_ack_poisoned_rooms:
targets = self._deferred_ack_targets.pop(room, None)
self._send_abort_acks(room, targets)
def _poison_failed_transfer(
self, room: int, statuses: List[TransferStatus]
) -> None:
# Failed() makes no claim on what the write still puts into the decode's
# pages, so the room never acks and decode falls back to its timeout.
if self.enable_deferred_decode_kv_release and any(
status.Failed() for status in statuses
):
with self._abort_ack_lock:
self.poison_deferred_ack_room(room)
def _process_transfer_chunk(self, kv_chunk: TransferKVChunk) -> bool:
room = kv_chunk.room
if self._should_skip_transfer(room):
@@ -633,6 +645,7 @@ class MoriKVManager(CommonKVManager):
if not is_quiescent:
# The drainer now owns the decrement.
return False
self._poison_failed_transfer(room, statuses)
if self._should_skip_transfer(room):
return True
if failure_reason is not None:
@@ -733,6 +746,7 @@ class MoriKVManager(CommonKVManager):
for status in statuses:
if status.InProgress():
status.Wait()
self._poison_failed_transfer(kv_chunk.room, statuses)
self._mark_transfer_quiescent(kv_chunk)
def _should_skip_transfer(self, room: int) -> bool:
@@ -941,6 +955,8 @@ class MoriKVManager(CommonKVManager):
return
with self._abort_ack_lock:
if bootstrap_room in self._deferred_ack_poisoned_rooms:
return
if self._staging_outstanding.get(bootstrap_room, 0) > 0:
self.register_deferred_ack_target(bootstrap_room, ack_target)
return
+38 -33
View File
@@ -175,6 +175,12 @@ class _StateLayoutMismatch(ValueError):
pass
# A post NIXL rejected. NIXL may already have writes of the request on the wire
# and does not wait for them, so the room must never be acked.
class _NixlPostError(RuntimeError):
pass
# Once one handle of a batch reports ERR, its siblings settle only when NIXL
# notices their peer is gone, which for UCX means waiting out peer keepalive.
# This worker serves other rooms, so bound that wait.
@@ -979,8 +985,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
raise RuntimeError("Failed to create prepared state transfer")
# The chunk owns the handle before posting, so a failed post settles too.
yield xfer_handle
if self.agent.transfer(xfer_handle) == "ERR":
raise RuntimeError("Failed to post prepared state transfer")
self._post_xfer(xfer_handle, "Failed to post prepared state transfer")
def _init_staging_prefill_ctx(self):
from sglang.srt.disaggregation.common.staging_handler import (
@@ -1132,6 +1137,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
def check_status(self, bootstrap_room: int):
return self.request_status.get(bootstrap_room, KVPoll.WaitingForInput)
def _post_xfer(self, xfer_handle: Any, error_msg: str) -> None:
# The NIXL Python API raises on a failed post rather than returning "ERR".
try:
state = self.agent.transfer(xfer_handle)
except Exception as e:
raise _NixlPostError(f"{error_msg}: {e}") from e
if state == "ERR":
raise _NixlPostError(error_msg)
def _await_handles(
self, handles: List[Any], *, failure_seen: bool
) -> Tuple[bool, bool]:
@@ -1991,7 +2005,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
except Exception as e:
# Catch all exceptions to prevent silently killing this
# worker thread, but still propagate via failure_exception().
if isinstance(e, _NIXL_TRANSPORT_ERRORS):
if isinstance(e, _NIXL_TRANSPORT_ERRORS) or isinstance(
e.__cause__, _NIXL_TRANSPORT_ERRORS
):
logger.warning(f"NIXL transport error for room {room}: {e}")
else:
logger.exception(
@@ -2005,6 +2021,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
if not settle_timed_out:
notify, _ = self._await_handles(handles, failure_seen=True)
if notify:
if (
isinstance(e, _NixlPostError)
and self.enable_deferred_decode_kv_release
):
# The rejected request is not in `handles`, so its writes
# may still land; leave the room to decode's release timeout.
self.poison_deferred_ack_room(room)
self.conclude_failure(bootstrap_room=room, failure_reason=str(e))
# Every handle settled => the writes are done, but the
# normal-path decrement was never reached, so without this
@@ -2200,9 +2223,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create prepped transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post prepped transfer")
self._post_xfer(xfer_handle, "KVSender failed to post prepped transfer")
return xfer_handle
# Non-prepped path: used for state transfers (SWA/NSA) via maybe_send_extra.
@@ -2356,9 +2377,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post transfer")
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
return xfer_handle
def send_kvcache(
@@ -2534,9 +2553,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create mixed prepped transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post mixed prepped transfer")
self._post_xfer(
xfer_handle, "KVSender failed to post mixed prepped transfer"
)
handles.append(xfer_handle)
return handles
@@ -2581,9 +2600,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create prepped slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post prepped slice transfer")
self._post_xfer(xfer_handle, "KVSender failed to post prepped slice transfer")
return xfer_handle
def send_kvcache_staged(
@@ -2692,9 +2709,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
f"(src=0x{staging_buffer.get_ptr():x}, dst=0x{dst_write_ptr:x}, "
f"size={per_rank_bytes})"
)
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise RuntimeError("[Staging] NIXL bulk transfer failed to post")
self._post_xfer(xfer_handle, "[Staging] NIXL bulk transfer failed to post")
return xfer_handle
def _try_create_staging_strategy(self, staging_buffer):
@@ -2827,9 +2842,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post transfer")
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
return xfer_handle
def _send_slot_state(
@@ -2881,9 +2894,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create dsa_tail transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post dsa_tail transfer")
self._post_xfer(xfer_handle, "KVSender failed to post dsa_tail transfer")
return xfer_handle
def _send_mamba_state(
@@ -2937,9 +2948,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("Failed to create Mamba state transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post Mamba state transfer")
self._post_xfer(xfer_handle, "Failed to post Mamba state transfer")
return xfer_handle
def _send_mamba_state_slice(
@@ -3067,9 +3076,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("Failed to create Mamba state slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post Mamba state slice transfer")
self._post_xfer(xfer_handle, "Failed to post Mamba state slice transfer")
return xfer_handle
def _send_swa_state_slice(
@@ -3124,9 +3131,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("Failed to create SWA state slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post SWA state slice transfer")
self._post_xfer(xfer_handle, "Failed to post SWA state slice transfer")
return xfer_handle
def maybe_send_extra(
@@ -136,6 +136,7 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase):
manager.request_status[11] = KVPoll.Transferring
status = MagicMock()
status.InProgress.return_value = False
status.Failed.return_value = False
manager._submit_kv_transfer = MagicMock(return_value=[status])
wait_results = iter((StatusCode.IN_PROGRESS, StatusCode.SUCCESS))
@@ -170,6 +171,7 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase):
manager.engine.wait_all.return_value = StatusCode.IN_PROGRESS
status = MagicMock()
status.InProgress.return_value = True
status.Failed.return_value = False
manager._submit_kv_transfer = MagicMock(return_value=[status])
manager.conclude_failure = MagicMock()
chunk = _chunk()
@@ -201,6 +203,45 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase):
self.assertNotIn(11, manager._staging_outstanding)
manager.conclude_failure.assert_called_once()
def test_failed_status_never_acks(self):
"""A write that ended Failed() may still land in decode's pages, so
neither the pending ABORT nor a later one may be ACKed."""
for via_drainer in (False, True):
with self.subTest(via_drainer=via_drainer):
manager = _manager()
manager.request_status[11] = KVPoll.Transferring
manager.conclude_failure = MagicMock()
status = MagicMock()
status.InProgress.return_value = via_drainer
status.Failed.return_value = True
manager._submit_kv_transfer = MagicMock(return_value=[status])
manager.engine = MagicMock()
manager.engine.wait_all.return_value = (
StatusCode.IN_PROGRESS if via_drainer else StatusCode.FAILED
)
manager._transfer_timeout_ms = 1 if via_drainer else 0
chunk = _chunk()
manager._mark_transfer_started(chunk)
with patch(
"sglang.srt.disaggregation.mori.conn.time.perf_counter",
side_effect=(0.0, 0.002),
):
is_quiescent = manager._process_transfer_chunk(chunk)
manager._handle_abort_message(_abort_message())
if via_drainer:
self.assertFalse(is_quiescent)
queued_chunk, queued_statuses = manager._drain_queue.get_nowait()
manager._drain_transfer_statuses(queued_chunk, queued_statuses)
else:
self.assertTrue(is_quiescent)
manager._mark_transfer_quiescent(chunk)
manager._handle_abort_message(_abort_message())
self.assertEqual(manager._sent, [])
self.assertIn(11, manager._deferred_ack_poisoned_rooms)
self.assertNotIn(11, manager._staging_outstanding)
def test_room_cleared_after_drain_handoff_keeps_single_owner(self):
manager = _manager()
manager.request_status[11] = KVPoll.Transferring
@@ -21,6 +21,7 @@ from test_deferred_decode_kv_release import (
from sglang.srt.disaggregation.base.conn import KVPoll
from sglang.srt.disaggregation.common.conn import AckTarget
from sglang.srt.disaggregation.common.utils import TransferKVChunk
from sglang.srt.disaggregation.nixl import conn as nixl_conn
from sglang.srt.disaggregation.nixl.conn import NixlKVManager
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -153,6 +154,80 @@ class TestNixlAbortNotification(
)
self.assertEqual(manager._deferred_ack_targets, {})
def _prepare_worker(self, manager, num_peers: int) -> None:
"""Wire transfer_worker to MLA decode peers over the prepped path."""
manager.enable_staging = False
manager._staging_ctx = None
manager._staging_outstanding = defaultdict(int)
manager.exceptions = {}
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.attn_tp_size = 1
manager.transfer_source_rank = 0
manager.src_mem_kind = "VRAM"
manager._num_slots_src = 4
manager.kv_args = SimpleNamespace(kv_data_ptrs=[0x1000], kv_item_lens=[16])
manager.prep_handles = {"": "src-prep"}
manager.decode_kv_args_table = {}
manager.transfer_infos = {ROOM: {}}
for i in range(num_peers):
agent_name = f"decode-{i}"
manager.prep_handles[agent_name] = f"dst-prep-{i}"
manager.decode_kv_args_table[agent_name] = SimpleNamespace(
decode_tp_size=1,
requires_dcp_relayout=False,
kv_xfer_segments=None,
dst_homogeneous_mem_kind="VRAM",
dst_kv_ptrs=[0x2000],
dst_num_slots=None,
gpu_id=0,
)
manager.transfer_infos[ROOM][agent_name] = SimpleNamespace(
room=ROOM,
is_dummy=False,
agent_name=agent_name,
dst_kv_indices=np.array([1], dtype=np.int32),
)
manager.conclude_failure = MagicMock(
side_effect=lambda **kw: manager.update_status(
kw["bootstrap_room"], KVPoll.Failed
)
)
manager._reload_invalidated_peers = MagicMock()
manager.agent = MagicMock()
manager.agent.make_prepped_xfer.side_effect = lambda *args: object()
def test_rejected_post_never_acks(self):
"""NIXL raises when it rejects a post and does not wait for writes it
already put on the wire, so after the first decode peer's write is DONE
and the second peer's post raises, neither an ABORT that arrived
mid-transfer nor a later one may be ACKed."""
manager = _manager(status=KVPoll.Transferring)
self._prepare_worker(manager, num_peers=2)
posts = iter(("PROC", nixl_conn._NIXL_TRANSPORT_ERRORS[0]("NIXL_ERR_BACKEND")))
def post(handle):
result = next(posts)
if isinstance(result, Exception):
self._dispatch_abort(manager)
raise result
return result
manager.agent.transfer.side_effect = post
manager.agent.check_xfer_state.return_value = "DONE"
queue = SimpleNamespace(
get=MagicMock(side_effect=(_chunk(), KeyboardInterrupt()))
)
with self.assertRaises(KeyboardInterrupt):
manager.transfer_worker(queue)
self._dispatch_abort(manager)
self.assertEqual(manager.agent.transfer.call_count, 2)
manager.conclude_failure.assert_called_once()
self.assertEqual(manager._staging_outstanding[ROOM], 0)
self.assertEqual(manager._sent, [])
if __name__ == "__main__":
unittest.main()