[Fix] disaggregation: Poison the deferred abort ack when a NIXL or Mori write fails (#45973)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6c844eb12e
commit
6e667c4310
@@ -603,9 +603,21 @@ class MoriKVManager(CommonKVManager):
|
||||
self._staging_outstanding[room] -= 1
|
||||
if self._staging_outstanding[room] <= 0:
|
||||
self._staging_outstanding.pop(room)
|
||||
targets = self._deferred_ack_targets.pop(room, None)
|
||||
if room not in self._deferred_ack_poisoned_rooms:
|
||||
targets = self._deferred_ack_targets.pop(room, None)
|
||||
self._send_abort_acks(room, targets)
|
||||
|
||||
def _poison_failed_transfer(
|
||||
self, room: int, statuses: List[TransferStatus]
|
||||
) -> None:
|
||||
# Failed() makes no claim on what the write still puts into the decode's
|
||||
# pages, so the room never acks and decode falls back to its timeout.
|
||||
if self.enable_deferred_decode_kv_release and any(
|
||||
status.Failed() for status in statuses
|
||||
):
|
||||
with self._abort_ack_lock:
|
||||
self.poison_deferred_ack_room(room)
|
||||
|
||||
def _process_transfer_chunk(self, kv_chunk: TransferKVChunk) -> bool:
|
||||
room = kv_chunk.room
|
||||
if self._should_skip_transfer(room):
|
||||
@@ -633,6 +645,7 @@ class MoriKVManager(CommonKVManager):
|
||||
if not is_quiescent:
|
||||
# The drainer now owns the decrement.
|
||||
return False
|
||||
self._poison_failed_transfer(room, statuses)
|
||||
if self._should_skip_transfer(room):
|
||||
return True
|
||||
if failure_reason is not None:
|
||||
@@ -733,6 +746,7 @@ class MoriKVManager(CommonKVManager):
|
||||
for status in statuses:
|
||||
if status.InProgress():
|
||||
status.Wait()
|
||||
self._poison_failed_transfer(kv_chunk.room, statuses)
|
||||
self._mark_transfer_quiescent(kv_chunk)
|
||||
|
||||
def _should_skip_transfer(self, room: int) -> bool:
|
||||
@@ -941,6 +955,8 @@ class MoriKVManager(CommonKVManager):
|
||||
return
|
||||
|
||||
with self._abort_ack_lock:
|
||||
if bootstrap_room in self._deferred_ack_poisoned_rooms:
|
||||
return
|
||||
if self._staging_outstanding.get(bootstrap_room, 0) > 0:
|
||||
self.register_deferred_ack_target(bootstrap_room, ack_target)
|
||||
return
|
||||
|
||||
@@ -175,6 +175,12 @@ class _StateLayoutMismatch(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
# A post NIXL rejected. NIXL may already have writes of the request on the wire
|
||||
# and does not wait for them, so the room must never be acked.
|
||||
class _NixlPostError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
# Once one handle of a batch reports ERR, its siblings settle only when NIXL
|
||||
# notices their peer is gone, which for UCX means waiting out peer keepalive.
|
||||
# This worker serves other rooms, so bound that wait.
|
||||
@@ -979,8 +985,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
raise RuntimeError("Failed to create prepared state transfer")
|
||||
# The chunk owns the handle before posting, so a failed post settles too.
|
||||
yield xfer_handle
|
||||
if self.agent.transfer(xfer_handle) == "ERR":
|
||||
raise RuntimeError("Failed to post prepared state transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post prepared state transfer")
|
||||
|
||||
def _init_staging_prefill_ctx(self):
|
||||
from sglang.srt.disaggregation.common.staging_handler import (
|
||||
@@ -1132,6 +1137,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
def check_status(self, bootstrap_room: int):
|
||||
return self.request_status.get(bootstrap_room, KVPoll.WaitingForInput)
|
||||
|
||||
def _post_xfer(self, xfer_handle: Any, error_msg: str) -> None:
|
||||
# The NIXL Python API raises on a failed post rather than returning "ERR".
|
||||
try:
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
except Exception as e:
|
||||
raise _NixlPostError(f"{error_msg}: {e}") from e
|
||||
if state == "ERR":
|
||||
raise _NixlPostError(error_msg)
|
||||
|
||||
def _await_handles(
|
||||
self, handles: List[Any], *, failure_seen: bool
|
||||
) -> Tuple[bool, bool]:
|
||||
@@ -1991,7 +2005,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
except Exception as e:
|
||||
# Catch all exceptions to prevent silently killing this
|
||||
# worker thread, but still propagate via failure_exception().
|
||||
if isinstance(e, _NIXL_TRANSPORT_ERRORS):
|
||||
if isinstance(e, _NIXL_TRANSPORT_ERRORS) or isinstance(
|
||||
e.__cause__, _NIXL_TRANSPORT_ERRORS
|
||||
):
|
||||
logger.warning(f"NIXL transport error for room {room}: {e}")
|
||||
else:
|
||||
logger.exception(
|
||||
@@ -2005,6 +2021,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if not settle_timed_out:
|
||||
notify, _ = self._await_handles(handles, failure_seen=True)
|
||||
if notify:
|
||||
if (
|
||||
isinstance(e, _NixlPostError)
|
||||
and self.enable_deferred_decode_kv_release
|
||||
):
|
||||
# The rejected request is not in `handles`, so its writes
|
||||
# may still land; leave the room to decode's release timeout.
|
||||
self.poison_deferred_ack_room(room)
|
||||
self.conclude_failure(bootstrap_room=room, failure_reason=str(e))
|
||||
# Every handle settled => the writes are done, but the
|
||||
# normal-path decrement was never reached, so without this
|
||||
@@ -2200,9 +2223,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create prepped transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post prepped transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post prepped transfer")
|
||||
return xfer_handle
|
||||
|
||||
# Non-prepped path: used for state transfers (SWA/NSA) via maybe_send_extra.
|
||||
@@ -2356,9 +2377,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
|
||||
return xfer_handle
|
||||
|
||||
def send_kvcache(
|
||||
@@ -2534,9 +2553,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create mixed prepped transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post mixed prepped transfer")
|
||||
self._post_xfer(
|
||||
xfer_handle, "KVSender failed to post mixed prepped transfer"
|
||||
)
|
||||
handles.append(xfer_handle)
|
||||
return handles
|
||||
|
||||
@@ -2581,9 +2600,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create prepped slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post prepped slice transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post prepped slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def send_kvcache_staged(
|
||||
@@ -2692,9 +2709,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
f"(src=0x{staging_buffer.get_ptr():x}, dst=0x{dst_write_ptr:x}, "
|
||||
f"size={per_rank_bytes})"
|
||||
)
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise RuntimeError("[Staging] NIXL bulk transfer failed to post")
|
||||
self._post_xfer(xfer_handle, "[Staging] NIXL bulk transfer failed to post")
|
||||
return xfer_handle
|
||||
|
||||
def _try_create_staging_strategy(self, staging_buffer):
|
||||
@@ -2827,9 +2842,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_slot_state(
|
||||
@@ -2881,9 +2894,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create dsa_tail transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post dsa_tail transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post dsa_tail transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_mamba_state(
|
||||
@@ -2937,9 +2948,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create Mamba state transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post Mamba state transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post Mamba state transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_mamba_state_slice(
|
||||
@@ -3067,9 +3076,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create Mamba state slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post Mamba state slice transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post Mamba state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_swa_state_slice(
|
||||
@@ -3124,9 +3131,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create SWA state slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post SWA state slice transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post SWA state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def maybe_send_extra(
|
||||
|
||||
@@ -136,6 +136,7 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase):
|
||||
manager.request_status[11] = KVPoll.Transferring
|
||||
status = MagicMock()
|
||||
status.InProgress.return_value = False
|
||||
status.Failed.return_value = False
|
||||
manager._submit_kv_transfer = MagicMock(return_value=[status])
|
||||
wait_results = iter((StatusCode.IN_PROGRESS, StatusCode.SUCCESS))
|
||||
|
||||
@@ -170,6 +171,7 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase):
|
||||
manager.engine.wait_all.return_value = StatusCode.IN_PROGRESS
|
||||
status = MagicMock()
|
||||
status.InProgress.return_value = True
|
||||
status.Failed.return_value = False
|
||||
manager._submit_kv_transfer = MagicMock(return_value=[status])
|
||||
manager.conclude_failure = MagicMock()
|
||||
chunk = _chunk()
|
||||
@@ -201,6 +203,45 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase):
|
||||
self.assertNotIn(11, manager._staging_outstanding)
|
||||
manager.conclude_failure.assert_called_once()
|
||||
|
||||
def test_failed_status_never_acks(self):
|
||||
"""A write that ended Failed() may still land in decode's pages, so
|
||||
neither the pending ABORT nor a later one may be ACKed."""
|
||||
for via_drainer in (False, True):
|
||||
with self.subTest(via_drainer=via_drainer):
|
||||
manager = _manager()
|
||||
manager.request_status[11] = KVPoll.Transferring
|
||||
manager.conclude_failure = MagicMock()
|
||||
status = MagicMock()
|
||||
status.InProgress.return_value = via_drainer
|
||||
status.Failed.return_value = True
|
||||
manager._submit_kv_transfer = MagicMock(return_value=[status])
|
||||
manager.engine = MagicMock()
|
||||
manager.engine.wait_all.return_value = (
|
||||
StatusCode.IN_PROGRESS if via_drainer else StatusCode.FAILED
|
||||
)
|
||||
manager._transfer_timeout_ms = 1 if via_drainer else 0
|
||||
chunk = _chunk()
|
||||
manager._mark_transfer_started(chunk)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.disaggregation.mori.conn.time.perf_counter",
|
||||
side_effect=(0.0, 0.002),
|
||||
):
|
||||
is_quiescent = manager._process_transfer_chunk(chunk)
|
||||
manager._handle_abort_message(_abort_message())
|
||||
if via_drainer:
|
||||
self.assertFalse(is_quiescent)
|
||||
queued_chunk, queued_statuses = manager._drain_queue.get_nowait()
|
||||
manager._drain_transfer_statuses(queued_chunk, queued_statuses)
|
||||
else:
|
||||
self.assertTrue(is_quiescent)
|
||||
manager._mark_transfer_quiescent(chunk)
|
||||
manager._handle_abort_message(_abort_message())
|
||||
|
||||
self.assertEqual(manager._sent, [])
|
||||
self.assertIn(11, manager._deferred_ack_poisoned_rooms)
|
||||
self.assertNotIn(11, manager._staging_outstanding)
|
||||
|
||||
def test_room_cleared_after_drain_handoff_keeps_single_owner(self):
|
||||
manager = _manager()
|
||||
manager.request_status[11] = KVPoll.Transferring
|
||||
|
||||
@@ -21,6 +21,7 @@ from test_deferred_decode_kv_release import (
|
||||
from sglang.srt.disaggregation.base.conn import KVPoll
|
||||
from sglang.srt.disaggregation.common.conn import AckTarget
|
||||
from sglang.srt.disaggregation.common.utils import TransferKVChunk
|
||||
from sglang.srt.disaggregation.nixl import conn as nixl_conn
|
||||
from sglang.srt.disaggregation.nixl.conn import NixlKVManager
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -153,6 +154,80 @@ class TestNixlAbortNotification(
|
||||
)
|
||||
self.assertEqual(manager._deferred_ack_targets, {})
|
||||
|
||||
def _prepare_worker(self, manager, num_peers: int) -> None:
|
||||
"""Wire transfer_worker to MLA decode peers over the prepped path."""
|
||||
manager.enable_staging = False
|
||||
manager._staging_ctx = None
|
||||
manager._staging_outstanding = defaultdict(int)
|
||||
manager.exceptions = {}
|
||||
manager.is_mla_backend = True
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.attn_tp_size = 1
|
||||
manager.transfer_source_rank = 0
|
||||
manager.src_mem_kind = "VRAM"
|
||||
manager._num_slots_src = 4
|
||||
manager.kv_args = SimpleNamespace(kv_data_ptrs=[0x1000], kv_item_lens=[16])
|
||||
manager.prep_handles = {"": "src-prep"}
|
||||
manager.decode_kv_args_table = {}
|
||||
manager.transfer_infos = {ROOM: {}}
|
||||
for i in range(num_peers):
|
||||
agent_name = f"decode-{i}"
|
||||
manager.prep_handles[agent_name] = f"dst-prep-{i}"
|
||||
manager.decode_kv_args_table[agent_name] = SimpleNamespace(
|
||||
decode_tp_size=1,
|
||||
requires_dcp_relayout=False,
|
||||
kv_xfer_segments=None,
|
||||
dst_homogeneous_mem_kind="VRAM",
|
||||
dst_kv_ptrs=[0x2000],
|
||||
dst_num_slots=None,
|
||||
gpu_id=0,
|
||||
)
|
||||
manager.transfer_infos[ROOM][agent_name] = SimpleNamespace(
|
||||
room=ROOM,
|
||||
is_dummy=False,
|
||||
agent_name=agent_name,
|
||||
dst_kv_indices=np.array([1], dtype=np.int32),
|
||||
)
|
||||
manager.conclude_failure = MagicMock(
|
||||
side_effect=lambda **kw: manager.update_status(
|
||||
kw["bootstrap_room"], KVPoll.Failed
|
||||
)
|
||||
)
|
||||
manager._reload_invalidated_peers = MagicMock()
|
||||
manager.agent = MagicMock()
|
||||
manager.agent.make_prepped_xfer.side_effect = lambda *args: object()
|
||||
|
||||
def test_rejected_post_never_acks(self):
|
||||
"""NIXL raises when it rejects a post and does not wait for writes it
|
||||
already put on the wire, so after the first decode peer's write is DONE
|
||||
and the second peer's post raises, neither an ABORT that arrived
|
||||
mid-transfer nor a later one may be ACKed."""
|
||||
manager = _manager(status=KVPoll.Transferring)
|
||||
self._prepare_worker(manager, num_peers=2)
|
||||
posts = iter(("PROC", nixl_conn._NIXL_TRANSPORT_ERRORS[0]("NIXL_ERR_BACKEND")))
|
||||
|
||||
def post(handle):
|
||||
result = next(posts)
|
||||
if isinstance(result, Exception):
|
||||
self._dispatch_abort(manager)
|
||||
raise result
|
||||
return result
|
||||
|
||||
manager.agent.transfer.side_effect = post
|
||||
manager.agent.check_xfer_state.return_value = "DONE"
|
||||
queue = SimpleNamespace(
|
||||
get=MagicMock(side_effect=(_chunk(), KeyboardInterrupt()))
|
||||
)
|
||||
|
||||
with self.assertRaises(KeyboardInterrupt):
|
||||
manager.transfer_worker(queue)
|
||||
self._dispatch_abort(manager)
|
||||
|
||||
self.assertEqual(manager.agent.transfer.call_count, 2)
|
||||
manager.conclude_failure.assert_called_once()
|
||||
self.assertEqual(manager._staging_outstanding[ROOM], 0)
|
||||
self.assertEqual(manager._sent, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user