[Fix] disaggregation: Poison the deferred abort ack when a NIXL or Mori write fails (#45973)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Banruo Liu
2026-10-23 14:47:59 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 6c844eb12e
commit 6e667c4310
4 changed files with 171 additions and 34 deletions
+17 -1
View File
@@ -603,9 +603,21 @@ class MoriKVManager(CommonKVManager):
self._staging_outstanding[room] -= 1
if self._staging_outstanding[room] <= 0:
self._staging_outstanding.pop(room)
targets = self._deferred_ack_targets.pop(room, None)
if room not in self._deferred_ack_poisoned_rooms:
targets = self._deferred_ack_targets.pop(room, None)
self._send_abort_acks(room, targets)
def _poison_failed_transfer(
self, room: int, statuses: List[TransferStatus]
) -> None:
# Failed() makes no claim on what the write still puts into the decode's
# pages, so the room never acks and decode falls back to its timeout.
if self.enable_deferred_decode_kv_release and any(
status.Failed() for status in statuses
):
with self._abort_ack_lock:
self.poison_deferred_ack_room(room)
def _process_transfer_chunk(self, kv_chunk: TransferKVChunk) -> bool:
room = kv_chunk.room
if self._should_skip_transfer(room):
@@ -633,6 +645,7 @@ class MoriKVManager(CommonKVManager):
if not is_quiescent:
# The drainer now owns the decrement.
return False
self._poison_failed_transfer(room, statuses)
if self._should_skip_transfer(room):
return True
if failure_reason is not None:
@@ -733,6 +746,7 @@ class MoriKVManager(CommonKVManager):
for status in statuses:
if status.InProgress():
status.Wait()
self._poison_failed_transfer(kv_chunk.room, statuses)
self._mark_transfer_quiescent(kv_chunk)
def _should_skip_transfer(self, room: int) -> bool:
@@ -941,6 +955,8 @@ class MoriKVManager(CommonKVManager):
return
with self._abort_ack_lock:
if bootstrap_room in self._deferred_ack_poisoned_rooms:
return
if self._staging_outstanding.get(bootstrap_room, 0) > 0:
self.register_deferred_ack_target(bootstrap_room, ack_target)
return
+38 -33
View File
@@ -175,6 +175,12 @@ class _StateLayoutMismatch(ValueError):
pass
# A post NIXL rejected. NIXL may already have writes of the request on the wire
# and does not wait for them, so the room must never be acked.
class _NixlPostError(RuntimeError):
pass
# Once one handle of a batch reports ERR, its siblings settle only when NIXL
# notices their peer is gone, which for UCX means waiting out peer keepalive.
# This worker serves other rooms, so bound that wait.
@@ -979,8 +985,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
raise RuntimeError("Failed to create prepared state transfer")
# The chunk owns the handle before posting, so a failed post settles too.
yield xfer_handle
if self.agent.transfer(xfer_handle) == "ERR":
raise RuntimeError("Failed to post prepared state transfer")
self._post_xfer(xfer_handle, "Failed to post prepared state transfer")
def _init_staging_prefill_ctx(self):
from sglang.srt.disaggregation.common.staging_handler import (
@@ -1132,6 +1137,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
def check_status(self, bootstrap_room: int):
return self.request_status.get(bootstrap_room, KVPoll.WaitingForInput)
def _post_xfer(self, xfer_handle: Any, error_msg: str) -> None:
# The NIXL Python API raises on a failed post rather than returning "ERR".
try:
state = self.agent.transfer(xfer_handle)
except Exception as e:
raise _NixlPostError(f"{error_msg}: {e}") from e
if state == "ERR":
raise _NixlPostError(error_msg)
def _await_handles(
self, handles: List[Any], *, failure_seen: bool
) -> Tuple[bool, bool]:
@@ -1991,7 +2005,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
except Exception as e:
# Catch all exceptions to prevent silently killing this
# worker thread, but still propagate via failure_exception().
if isinstance(e, _NIXL_TRANSPORT_ERRORS):
if isinstance(e, _NIXL_TRANSPORT_ERRORS) or isinstance(
e.__cause__, _NIXL_TRANSPORT_ERRORS
):
logger.warning(f"NIXL transport error for room {room}: {e}")
else:
logger.exception(
@@ -2005,6 +2021,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
if not settle_timed_out:
notify, _ = self._await_handles(handles, failure_seen=True)
if notify:
if (
isinstance(e, _NixlPostError)
and self.enable_deferred_decode_kv_release
):
# The rejected request is not in `handles`, so its writes
# may still land; leave the room to decode's release timeout.
self.poison_deferred_ack_room(room)
self.conclude_failure(bootstrap_room=room, failure_reason=str(e))
# Every handle settled => the writes are done, but the
# normal-path decrement was never reached, so without this
@@ -2200,9 +2223,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create prepped transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post prepped transfer")
self._post_xfer(xfer_handle, "KVSender failed to post prepped transfer")
return xfer_handle
# Non-prepped path: used for state transfers (SWA/NSA) via maybe_send_extra.
@@ -2356,9 +2377,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post transfer")
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
return xfer_handle
def send_kvcache(
@@ -2534,9 +2553,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create mixed prepped transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post mixed prepped transfer")
self._post_xfer(
xfer_handle, "KVSender failed to post mixed prepped transfer"
)
handles.append(xfer_handle)
return handles
@@ -2581,9 +2600,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create prepped slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post prepped slice transfer")
self._post_xfer(xfer_handle, "KVSender failed to post prepped slice transfer")
return xfer_handle
def send_kvcache_staged(
@@ -2692,9 +2709,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
f"(src=0x{staging_buffer.get_ptr():x}, dst=0x{dst_write_ptr:x}, "
f"size={per_rank_bytes})"
)
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise RuntimeError("[Staging] NIXL bulk transfer failed to post")
self._post_xfer(xfer_handle, "[Staging] NIXL bulk transfer failed to post")
return xfer_handle
def _try_create_staging_strategy(self, staging_buffer):
@@ -2827,9 +2842,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post transfer")
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
return xfer_handle
def _send_slot_state(
@@ -2881,9 +2894,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("KVSender failed to create dsa_tail transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("KVSender failed to post dsa_tail transfer")
self._post_xfer(xfer_handle, "KVSender failed to post dsa_tail transfer")
return xfer_handle
def _send_mamba_state(
@@ -2937,9 +2948,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("Failed to create Mamba state transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post Mamba state transfer")
self._post_xfer(xfer_handle, "Failed to post Mamba state transfer")
return xfer_handle
def _send_mamba_state_slice(
@@ -3067,9 +3076,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("Failed to create Mamba state slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post Mamba state slice transfer")
self._post_xfer(xfer_handle, "Failed to post Mamba state slice transfer")
return xfer_handle
def _send_swa_state_slice(
@@ -3124,9 +3131,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
)
if not xfer_handle:
raise Exception("Failed to create SWA state slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post SWA state slice transfer")
self._post_xfer(xfer_handle, "Failed to post SWA state slice transfer")
return xfer_handle
def maybe_send_extra(