diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index 5df253eed1..c52f5839a6 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -603,9 +603,21 @@ class MoriKVManager(CommonKVManager): self._staging_outstanding[room] -= 1 if self._staging_outstanding[room] <= 0: self._staging_outstanding.pop(room) - targets = self._deferred_ack_targets.pop(room, None) + if room not in self._deferred_ack_poisoned_rooms: + targets = self._deferred_ack_targets.pop(room, None) self._send_abort_acks(room, targets) + def _poison_failed_transfer( + self, room: int, statuses: List[TransferStatus] + ) -> None: + # Failed() makes no claim on what the write still puts into the decode's + # pages, so the room never acks and decode falls back to its timeout. + if self.enable_deferred_decode_kv_release and any( + status.Failed() for status in statuses + ): + with self._abort_ack_lock: + self.poison_deferred_ack_room(room) + def _process_transfer_chunk(self, kv_chunk: TransferKVChunk) -> bool: room = kv_chunk.room if self._should_skip_transfer(room): @@ -633,6 +645,7 @@ class MoriKVManager(CommonKVManager): if not is_quiescent: # The drainer now owns the decrement. return False + self._poison_failed_transfer(room, statuses) if self._should_skip_transfer(room): return True if failure_reason is not None: @@ -733,6 +746,7 @@ class MoriKVManager(CommonKVManager): for status in statuses: if status.InProgress(): status.Wait() + self._poison_failed_transfer(kv_chunk.room, statuses) self._mark_transfer_quiescent(kv_chunk) def _should_skip_transfer(self, room: int) -> bool: @@ -941,6 +955,8 @@ class MoriKVManager(CommonKVManager): return with self._abort_ack_lock: + if bootstrap_room in self._deferred_ack_poisoned_rooms: + return if self._staging_outstanding.get(bootstrap_room, 0) > 0: self.register_deferred_ack_target(bootstrap_room, ack_target) return diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index a107f0f1a4..1d1cf55396 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -175,6 +175,12 @@ class _StateLayoutMismatch(ValueError): pass +# A post NIXL rejected. NIXL may already have writes of the request on the wire +# and does not wait for them, so the room must never be acked. +class _NixlPostError(RuntimeError): + pass + + # Once one handle of a batch reports ERR, its siblings settle only when NIXL # notices their peer is gone, which for UCX means waiting out peer keepalive. # This worker serves other rooms, so bound that wait. @@ -979,8 +985,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): raise RuntimeError("Failed to create prepared state transfer") # The chunk owns the handle before posting, so a failed post settles too. yield xfer_handle - if self.agent.transfer(xfer_handle) == "ERR": - raise RuntimeError("Failed to post prepared state transfer") + self._post_xfer(xfer_handle, "Failed to post prepared state transfer") def _init_staging_prefill_ctx(self): from sglang.srt.disaggregation.common.staging_handler import ( @@ -1132,6 +1137,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): def check_status(self, bootstrap_room: int): return self.request_status.get(bootstrap_room, KVPoll.WaitingForInput) + def _post_xfer(self, xfer_handle: Any, error_msg: str) -> None: + # The NIXL Python API raises on a failed post rather than returning "ERR". + try: + state = self.agent.transfer(xfer_handle) + except Exception as e: + raise _NixlPostError(f"{error_msg}: {e}") from e + if state == "ERR": + raise _NixlPostError(error_msg) + def _await_handles( self, handles: List[Any], *, failure_seen: bool ) -> Tuple[bool, bool]: @@ -1991,7 +2005,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): except Exception as e: # Catch all exceptions to prevent silently killing this # worker thread, but still propagate via failure_exception(). - if isinstance(e, _NIXL_TRANSPORT_ERRORS): + if isinstance(e, _NIXL_TRANSPORT_ERRORS) or isinstance( + e.__cause__, _NIXL_TRANSPORT_ERRORS + ): logger.warning(f"NIXL transport error for room {room}: {e}") else: logger.exception( @@ -2005,6 +2021,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): if not settle_timed_out: notify, _ = self._await_handles(handles, failure_seen=True) if notify: + if ( + isinstance(e, _NixlPostError) + and self.enable_deferred_decode_kv_release + ): + # The rejected request is not in `handles`, so its writes + # may still land; leave the room to decode's release timeout. + self.poison_deferred_ack_room(room) self.conclude_failure(bootstrap_room=room, failure_reason=str(e)) # Every handle settled => the writes are done, but the # normal-path decrement was never reached, so without this @@ -2200,9 +2223,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("KVSender failed to create prepped transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("KVSender failed to post prepped transfer") + self._post_xfer(xfer_handle, "KVSender failed to post prepped transfer") return xfer_handle # Non-prepped path: used for state transfers (SWA/NSA) via maybe_send_extra. @@ -2356,9 +2377,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("KVSender failed to create transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("KVSender failed to post transfer") + self._post_xfer(xfer_handle, "KVSender failed to post transfer") return xfer_handle def send_kvcache( @@ -2534,9 +2553,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("KVSender failed to create mixed prepped transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("KVSender failed to post mixed prepped transfer") + self._post_xfer( + xfer_handle, "KVSender failed to post mixed prepped transfer" + ) handles.append(xfer_handle) return handles @@ -2581,9 +2600,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("KVSender failed to create prepped slice transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("KVSender failed to post prepped slice transfer") + self._post_xfer(xfer_handle, "KVSender failed to post prepped slice transfer") return xfer_handle def send_kvcache_staged( @@ -2692,9 +2709,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): f"(src=0x{staging_buffer.get_ptr():x}, dst=0x{dst_write_ptr:x}, " f"size={per_rank_bytes})" ) - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise RuntimeError("[Staging] NIXL bulk transfer failed to post") + self._post_xfer(xfer_handle, "[Staging] NIXL bulk transfer failed to post") return xfer_handle def _try_create_staging_strategy(self, staging_buffer): @@ -2827,9 +2842,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("KVSender failed to create transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("KVSender failed to post transfer") + self._post_xfer(xfer_handle, "KVSender failed to post transfer") return xfer_handle def _send_slot_state( @@ -2881,9 +2894,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("KVSender failed to create dsa_tail transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("KVSender failed to post dsa_tail transfer") + self._post_xfer(xfer_handle, "KVSender failed to post dsa_tail transfer") return xfer_handle def _send_mamba_state( @@ -2937,9 +2948,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("Failed to create Mamba state transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("Failed to post Mamba state transfer") + self._post_xfer(xfer_handle, "Failed to post Mamba state transfer") return xfer_handle def _send_mamba_state_slice( @@ -3067,9 +3076,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("Failed to create Mamba state slice transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("Failed to post Mamba state slice transfer") + self._post_xfer(xfer_handle, "Failed to post Mamba state slice transfer") return xfer_handle def _send_swa_state_slice( @@ -3124,9 +3131,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) if not xfer_handle: raise Exception("Failed to create SWA state slice transfer") - state = self.agent.transfer(xfer_handle) - if state == "ERR": - raise Exception("Failed to post SWA state slice transfer") + self._post_xfer(xfer_handle, "Failed to post SWA state slice transfer") return xfer_handle def maybe_send_extra( diff --git a/test/registered/unit/disaggregation/test_mori_deferred_kv_release.py b/test/registered/unit/disaggregation/test_mori_deferred_kv_release.py index 232690bad2..8a49286bf8 100644 --- a/test/registered/unit/disaggregation/test_mori_deferred_kv_release.py +++ b/test/registered/unit/disaggregation/test_mori_deferred_kv_release.py @@ -136,6 +136,7 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase): manager.request_status[11] = KVPoll.Transferring status = MagicMock() status.InProgress.return_value = False + status.Failed.return_value = False manager._submit_kv_transfer = MagicMock(return_value=[status]) wait_results = iter((StatusCode.IN_PROGRESS, StatusCode.SUCCESS)) @@ -170,6 +171,7 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase): manager.engine.wait_all.return_value = StatusCode.IN_PROGRESS status = MagicMock() status.InProgress.return_value = True + status.Failed.return_value = False manager._submit_kv_transfer = MagicMock(return_value=[status]) manager.conclude_failure = MagicMock() chunk = _chunk() @@ -201,6 +203,45 @@ class TestMoriAbortAck(DeferredAbortNotificationScenarios, CustomTestCase): self.assertNotIn(11, manager._staging_outstanding) manager.conclude_failure.assert_called_once() + def test_failed_status_never_acks(self): + """A write that ended Failed() may still land in decode's pages, so + neither the pending ABORT nor a later one may be ACKed.""" + for via_drainer in (False, True): + with self.subTest(via_drainer=via_drainer): + manager = _manager() + manager.request_status[11] = KVPoll.Transferring + manager.conclude_failure = MagicMock() + status = MagicMock() + status.InProgress.return_value = via_drainer + status.Failed.return_value = True + manager._submit_kv_transfer = MagicMock(return_value=[status]) + manager.engine = MagicMock() + manager.engine.wait_all.return_value = ( + StatusCode.IN_PROGRESS if via_drainer else StatusCode.FAILED + ) + manager._transfer_timeout_ms = 1 if via_drainer else 0 + chunk = _chunk() + manager._mark_transfer_started(chunk) + + with patch( + "sglang.srt.disaggregation.mori.conn.time.perf_counter", + side_effect=(0.0, 0.002), + ): + is_quiescent = manager._process_transfer_chunk(chunk) + manager._handle_abort_message(_abort_message()) + if via_drainer: + self.assertFalse(is_quiescent) + queued_chunk, queued_statuses = manager._drain_queue.get_nowait() + manager._drain_transfer_statuses(queued_chunk, queued_statuses) + else: + self.assertTrue(is_quiescent) + manager._mark_transfer_quiescent(chunk) + manager._handle_abort_message(_abort_message()) + + self.assertEqual(manager._sent, []) + self.assertIn(11, manager._deferred_ack_poisoned_rooms) + self.assertNotIn(11, manager._staging_outstanding) + def test_room_cleared_after_drain_handoff_keeps_single_owner(self): manager = _manager() manager.request_status[11] = KVPoll.Transferring diff --git a/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py b/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py index 1bcb278b12..868883685b 100644 --- a/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py +++ b/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py @@ -21,6 +21,7 @@ from test_deferred_decode_kv_release import ( from sglang.srt.disaggregation.base.conn import KVPoll from sglang.srt.disaggregation.common.conn import AckTarget from sglang.srt.disaggregation.common.utils import TransferKVChunk +from sglang.srt.disaggregation.nixl import conn as nixl_conn from sglang.srt.disaggregation.nixl.conn import NixlKVManager from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -153,6 +154,80 @@ class TestNixlAbortNotification( ) self.assertEqual(manager._deferred_ack_targets, {}) + def _prepare_worker(self, manager, num_peers: int) -> None: + """Wire transfer_worker to MLA decode peers over the prepped path.""" + manager.enable_staging = False + manager._staging_ctx = None + manager._staging_outstanding = defaultdict(int) + manager.exceptions = {} + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.attn_tp_size = 1 + manager.transfer_source_rank = 0 + manager.src_mem_kind = "VRAM" + manager._num_slots_src = 4 + manager.kv_args = SimpleNamespace(kv_data_ptrs=[0x1000], kv_item_lens=[16]) + manager.prep_handles = {"": "src-prep"} + manager.decode_kv_args_table = {} + manager.transfer_infos = {ROOM: {}} + for i in range(num_peers): + agent_name = f"decode-{i}" + manager.prep_handles[agent_name] = f"dst-prep-{i}" + manager.decode_kv_args_table[agent_name] = SimpleNamespace( + decode_tp_size=1, + requires_dcp_relayout=False, + kv_xfer_segments=None, + dst_homogeneous_mem_kind="VRAM", + dst_kv_ptrs=[0x2000], + dst_num_slots=None, + gpu_id=0, + ) + manager.transfer_infos[ROOM][agent_name] = SimpleNamespace( + room=ROOM, + is_dummy=False, + agent_name=agent_name, + dst_kv_indices=np.array([1], dtype=np.int32), + ) + manager.conclude_failure = MagicMock( + side_effect=lambda **kw: manager.update_status( + kw["bootstrap_room"], KVPoll.Failed + ) + ) + manager._reload_invalidated_peers = MagicMock() + manager.agent = MagicMock() + manager.agent.make_prepped_xfer.side_effect = lambda *args: object() + + def test_rejected_post_never_acks(self): + """NIXL raises when it rejects a post and does not wait for writes it + already put on the wire, so after the first decode peer's write is DONE + and the second peer's post raises, neither an ABORT that arrived + mid-transfer nor a later one may be ACKed.""" + manager = _manager(status=KVPoll.Transferring) + self._prepare_worker(manager, num_peers=2) + posts = iter(("PROC", nixl_conn._NIXL_TRANSPORT_ERRORS[0]("NIXL_ERR_BACKEND"))) + + def post(handle): + result = next(posts) + if isinstance(result, Exception): + self._dispatch_abort(manager) + raise result + return result + + manager.agent.transfer.side_effect = post + manager.agent.check_xfer_state.return_value = "DONE" + queue = SimpleNamespace( + get=MagicMock(side_effect=(_chunk(), KeyboardInterrupt())) + ) + + with self.assertRaises(KeyboardInterrupt): + manager.transfer_worker(queue) + self._dispatch_abort(manager) + + self.assertEqual(manager.agent.transfer.call_count, 2) + manager.conclude_failure.assert_called_once() + self.assertEqual(manager._staging_outstanding[ROOM], 0) + self.assertEqual(manager._sent, []) + if __name__ == "__main__": unittest.main()