[Fix] disaggregation: Poison the deferred abort ack when a NIXL or Mori write fails (#45973)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6c844eb12e
commit
6e667c4310
@@ -603,9 +603,21 @@ class MoriKVManager(CommonKVManager):
|
||||
self._staging_outstanding[room] -= 1
|
||||
if self._staging_outstanding[room] <= 0:
|
||||
self._staging_outstanding.pop(room)
|
||||
targets = self._deferred_ack_targets.pop(room, None)
|
||||
if room not in self._deferred_ack_poisoned_rooms:
|
||||
targets = self._deferred_ack_targets.pop(room, None)
|
||||
self._send_abort_acks(room, targets)
|
||||
|
||||
def _poison_failed_transfer(
|
||||
self, room: int, statuses: List[TransferStatus]
|
||||
) -> None:
|
||||
# Failed() makes no claim on what the write still puts into the decode's
|
||||
# pages, so the room never acks and decode falls back to its timeout.
|
||||
if self.enable_deferred_decode_kv_release and any(
|
||||
status.Failed() for status in statuses
|
||||
):
|
||||
with self._abort_ack_lock:
|
||||
self.poison_deferred_ack_room(room)
|
||||
|
||||
def _process_transfer_chunk(self, kv_chunk: TransferKVChunk) -> bool:
|
||||
room = kv_chunk.room
|
||||
if self._should_skip_transfer(room):
|
||||
@@ -633,6 +645,7 @@ class MoriKVManager(CommonKVManager):
|
||||
if not is_quiescent:
|
||||
# The drainer now owns the decrement.
|
||||
return False
|
||||
self._poison_failed_transfer(room, statuses)
|
||||
if self._should_skip_transfer(room):
|
||||
return True
|
||||
if failure_reason is not None:
|
||||
@@ -733,6 +746,7 @@ class MoriKVManager(CommonKVManager):
|
||||
for status in statuses:
|
||||
if status.InProgress():
|
||||
status.Wait()
|
||||
self._poison_failed_transfer(kv_chunk.room, statuses)
|
||||
self._mark_transfer_quiescent(kv_chunk)
|
||||
|
||||
def _should_skip_transfer(self, room: int) -> bool:
|
||||
@@ -941,6 +955,8 @@ class MoriKVManager(CommonKVManager):
|
||||
return
|
||||
|
||||
with self._abort_ack_lock:
|
||||
if bootstrap_room in self._deferred_ack_poisoned_rooms:
|
||||
return
|
||||
if self._staging_outstanding.get(bootstrap_room, 0) > 0:
|
||||
self.register_deferred_ack_target(bootstrap_room, ack_target)
|
||||
return
|
||||
|
||||
@@ -175,6 +175,12 @@ class _StateLayoutMismatch(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
# A post NIXL rejected. NIXL may already have writes of the request on the wire
|
||||
# and does not wait for them, so the room must never be acked.
|
||||
class _NixlPostError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
# Once one handle of a batch reports ERR, its siblings settle only when NIXL
|
||||
# notices their peer is gone, which for UCX means waiting out peer keepalive.
|
||||
# This worker serves other rooms, so bound that wait.
|
||||
@@ -979,8 +985,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
raise RuntimeError("Failed to create prepared state transfer")
|
||||
# The chunk owns the handle before posting, so a failed post settles too.
|
||||
yield xfer_handle
|
||||
if self.agent.transfer(xfer_handle) == "ERR":
|
||||
raise RuntimeError("Failed to post prepared state transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post prepared state transfer")
|
||||
|
||||
def _init_staging_prefill_ctx(self):
|
||||
from sglang.srt.disaggregation.common.staging_handler import (
|
||||
@@ -1132,6 +1137,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
def check_status(self, bootstrap_room: int):
|
||||
return self.request_status.get(bootstrap_room, KVPoll.WaitingForInput)
|
||||
|
||||
def _post_xfer(self, xfer_handle: Any, error_msg: str) -> None:
|
||||
# The NIXL Python API raises on a failed post rather than returning "ERR".
|
||||
try:
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
except Exception as e:
|
||||
raise _NixlPostError(f"{error_msg}: {e}") from e
|
||||
if state == "ERR":
|
||||
raise _NixlPostError(error_msg)
|
||||
|
||||
def _await_handles(
|
||||
self, handles: List[Any], *, failure_seen: bool
|
||||
) -> Tuple[bool, bool]:
|
||||
@@ -1991,7 +2005,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
except Exception as e:
|
||||
# Catch all exceptions to prevent silently killing this
|
||||
# worker thread, but still propagate via failure_exception().
|
||||
if isinstance(e, _NIXL_TRANSPORT_ERRORS):
|
||||
if isinstance(e, _NIXL_TRANSPORT_ERRORS) or isinstance(
|
||||
e.__cause__, _NIXL_TRANSPORT_ERRORS
|
||||
):
|
||||
logger.warning(f"NIXL transport error for room {room}: {e}")
|
||||
else:
|
||||
logger.exception(
|
||||
@@ -2005,6 +2021,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if not settle_timed_out:
|
||||
notify, _ = self._await_handles(handles, failure_seen=True)
|
||||
if notify:
|
||||
if (
|
||||
isinstance(e, _NixlPostError)
|
||||
and self.enable_deferred_decode_kv_release
|
||||
):
|
||||
# The rejected request is not in `handles`, so its writes
|
||||
# may still land; leave the room to decode's release timeout.
|
||||
self.poison_deferred_ack_room(room)
|
||||
self.conclude_failure(bootstrap_room=room, failure_reason=str(e))
|
||||
# Every handle settled => the writes are done, but the
|
||||
# normal-path decrement was never reached, so without this
|
||||
@@ -2200,9 +2223,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create prepped transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post prepped transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post prepped transfer")
|
||||
return xfer_handle
|
||||
|
||||
# Non-prepped path: used for state transfers (SWA/NSA) via maybe_send_extra.
|
||||
@@ -2356,9 +2377,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
|
||||
return xfer_handle
|
||||
|
||||
def send_kvcache(
|
||||
@@ -2534,9 +2553,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create mixed prepped transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post mixed prepped transfer")
|
||||
self._post_xfer(
|
||||
xfer_handle, "KVSender failed to post mixed prepped transfer"
|
||||
)
|
||||
handles.append(xfer_handle)
|
||||
return handles
|
||||
|
||||
@@ -2581,9 +2600,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create prepped slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post prepped slice transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post prepped slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def send_kvcache_staged(
|
||||
@@ -2692,9 +2709,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
f"(src=0x{staging_buffer.get_ptr():x}, dst=0x{dst_write_ptr:x}, "
|
||||
f"size={per_rank_bytes})"
|
||||
)
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise RuntimeError("[Staging] NIXL bulk transfer failed to post")
|
||||
self._post_xfer(xfer_handle, "[Staging] NIXL bulk transfer failed to post")
|
||||
return xfer_handle
|
||||
|
||||
def _try_create_staging_strategy(self, staging_buffer):
|
||||
@@ -2827,9 +2842,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_slot_state(
|
||||
@@ -2881,9 +2894,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("KVSender failed to create dsa_tail transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("KVSender failed to post dsa_tail transfer")
|
||||
self._post_xfer(xfer_handle, "KVSender failed to post dsa_tail transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_mamba_state(
|
||||
@@ -2937,9 +2948,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create Mamba state transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post Mamba state transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post Mamba state transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_mamba_state_slice(
|
||||
@@ -3067,9 +3076,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create Mamba state slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post Mamba state slice transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post Mamba state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_swa_state_slice(
|
||||
@@ -3124,9 +3131,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create SWA state slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post SWA state slice transfer")
|
||||
self._post_xfer(xfer_handle, "Failed to post SWA state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def maybe_send_extra(
|
||||
|
||||
Reference in New Issue
Block a user