diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 45cb4c20bd..24ed9a2205 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -910,6 +910,10 @@ class CommonKVManager(BaseKVManager): (outstanding == 0). pop() makes it fire at most once.""" if self._staging_outstanding.get(room, 0) > 0: return + if room in self._deferred_ack_poisoned_rooms: + # A target registered concurrently with the poison must not fire. + self._deferred_ack_targets.pop(room, None) + return self._send_abort_acks(room, self._deferred_ack_targets.pop(room, None)) def _send_abort_acks( diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index e8a268aa55..a07cfc5139 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2292,6 +2292,16 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): return self._transfer_data(req.mooncake_session_id, transfer_blocks) + def _poison_unsettled_send(self, room: int) -> None: + """Keep the drain ack from firing after a submitted transfer failed. + + batch_transfer_sync returns -1 on its wall-clock timeout with the batch + still running (freeBatchID refuses a busy batch and the handle is + dropped), so a failed send may still be writing decode's pages; decode + falls back to its release timeout instead.""" + if self.enable_deferred_decode_kv_release: + self.poison_deferred_ack_room(room) + def transfer_worker( self, queue: FastQueue, @@ -2546,6 +2556,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): logger.error( f"Session {req.mooncake_session_id} failed." ) + self._poison_unsettled_send(kv_chunk.room) self.conclude_failure( bootstrap_room=kv_chunk.room, failure_reason=( @@ -2563,6 +2574,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): executor, target_rank_registration_info, ) + # Before the Failed check: an ABORT during the + # send also marks the room Failed. + if state_rc != 0: + self._poison_unsettled_send(kv_chunk.room) if ( self.request_status.get( kv_chunk.room, KVPoll.Failed @@ -2594,6 +2609,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): kv_chunk.prefill_aux_index, target_rank_registration_info.dst_aux_ptrs, ) + if ret != 0: + # The aux write targets decode's metadata + # slot, which the deferred release also frees. + self._poison_unsettled_send(req.room) polls.append(True if ret == 0 else False) dst_ranks_infos.append((req.endpoint, req.dst_port)) diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index 585ccea782..1c9673e4b6 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -414,6 +414,25 @@ class TestDeferredAckTargets(CustomTestCase): self.assertEqual(mgr._sent, [(8, target)]) self.assertNotIn(8, mgr._deferred_ack_targets) + def test_target_registered_during_poison_never_acks(self): + """A target that races past the poisoned check must not ack later.""" + mgr = _make_prefill_manager() + mgr._staging_outstanding[7] = 1 + + class _PoisonOnRead(dict): + def get(self, *args): + # The worker poisons between the check and the store. + mgr.poison_deferred_ack_room(7) + return dict.get(self, *args) + + mgr._deferred_ack_targets = _PoisonOnRead() + mgr.register_deferred_ack_target(7, AckTarget("10.0.0.1", 5000, 9)) + mgr._staging_outstanding[7] = 0 + mgr._maybe_ack_drained_abort(7) + + self.assertEqual(mgr._sent, []) + self.assertNotIn(7, mgr._deferred_ack_targets) + def test_unregistered_room_is_noop(self): mgr = _make_prefill_manager() diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index fd3dad1af6..35d0ca91f1 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -594,6 +594,7 @@ class TestDisaggregationWire(unittest.TestCase): manager.transfer_infos = {} manager._staging_outstanding = {42: outstanding} manager._deferred_ack_targets = {42: {(target.ip, target.port): target}} + manager._deferred_ack_poisoned_rooms = set() with patch.object(manager, "_send_abort_ack") as ack: sender.clear() if outstanding: diff --git a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py index 705156ec57..e39324a45d 100644 --- a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py +++ b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py @@ -4,12 +4,14 @@ The bootstrap thread records the ACK target while a transfer is outstanding. Only the transfer worker can send that ACK after its write has drained. """ +import threading import unittest from collections import defaultdict from unittest.mock import MagicMock, patch import numpy as np from test_deferred_decode_kv_release import ( + ABORT_GENERATION, DeferredAbortNotificationScenarios, TaggedAbortNotificationScenarios, WorkerFailureAbortScenarios, @@ -20,9 +22,11 @@ from sglang.srt.disaggregation.common.conn import ( ABORT_ACK_TAG, ABORT_TAG, AbortNotification, + AckTarget, + CommonKVManager, ) from sglang.srt.disaggregation.common.utils import TransferKVChunk -from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager +from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager, TransferInfo from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -64,6 +68,18 @@ def _chunk() -> TransferKVChunk: ) +def _last_chunk() -> TransferKVChunk: + return TransferKVChunk( + room=ROOM, + prefill_kv_indices=np.array([1], dtype=np.int32), + index_slice=slice(0, 1), + is_last_chunk=True, + chunk_id=0, + prefill_aux_index=0, + state_indices=[[1]], + ) + + class TestMooncakeAbortNotification( DeferredAbortNotificationScenarios, TaggedAbortNotificationScenarios, @@ -131,6 +147,137 @@ class TestMooncakeAbortNotification( self.assertEqual(manager._deferred_ack_targets, {}) self.assertEqual(manager._sent, []) + def _prepare_worker(self, manager, num_dsts: int = 1) -> None: + """Wire transfer_worker to num_dsts decode ranks that all need the room.""" + manager.enable_trace = False + manager.enable_staging = False + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.bootstrap_port = 8998 + manager.attn_tp_rank = manager.pp_rank = manager.attn_cp_rank = 0 + manager.pp_size = manager.attn_cp_size = 1 + manager._staging_outstanding = defaultdict(int) + manager.session_lock = threading.Lock() + manager.session_failures = defaultdict(int) + manager.failed_sessions = set() + manager.state_layout_rejections = {} + manager.req_to_decode_prefix_len = {} + manager.kv_args = MagicMock(kv_data_ptrs=[0x1000]) + manager._get_dsa_cache_transfer_skip_flags = lambda info: (False, False) + manager.decode_kv_args_table = {} + manager.transfer_infos = {ROOM: {}} + for i in range(num_dsts): + session = f"decode-session-{i}" + manager.decode_kv_args_table[session] = MagicMock( + requires_dcp_relayout=False + ) + manager.transfer_infos[ROOM][(DECODE_IP, DECODE_PORT + i)] = TransferInfo( + room=ROOM, + endpoint=DECODE_IP, + dst_port=DECODE_PORT + i, + mooncake_session_id=session, + dst_kv_indices=np.array([1], dtype=np.int32), + dst_aux_index=0, + dst_state_indices=[[1]], + required_dst_info_num=num_dsts, + is_dummy=False, + ) + manager.send_kvcache = MagicMock(return_value=0) + manager.maybe_send_extra = MagicMock(return_value=0) + manager.send_aux = MagicMock(return_value=0) + manager.conclude_failure = MagicMock( + side_effect=lambda **kw: manager.update_status( + kw["bootstrap_room"], KVPoll.Failed + ) + ) + manager.conclude_transfer = MagicMock( + side_effect=lambda **kw: manager.update_status( + kw["bootstrap_room"], kw["status"] + ) + ) + + def _run_worker(self, manager, chunk: TransferKVChunk) -> None: + queue = MagicMock() + queue.get.side_effect = [chunk, None] + manager.transfer_worker(queue, MagicMock()) + self.assertEqual(manager.request_status[ROOM], KVPoll.Failed) + self.assertEqual(manager._staging_outstanding.get(ROOM, 0), 0) + + def _abort_on_call(self, manager): + """A failed send whose ABORT lands while its writes may be in flight.""" + + def send(*args, **kwargs): + self._dispatch_abort(manager) + return -1 + + return send + + def _run_worker_with_failed_send(self, manager) -> None: + """Run one chunk through transfer_worker with send_kvcache returning -1.""" + self._prepare_worker(manager) + manager.send_kvcache.return_value = -1 + self._run_worker(manager, _chunk()) + manager.send_kvcache.assert_called_once() + manager.conclude_failure.assert_called_once() + + def test_failed_send_never_acks_drained(self): + # batch_transfer_sync can return -1 with the batch still writing, so a + # failed send must leave decode to its release timeout, whether the + # ABORT arrived mid-send or only after the Failed status. + target = AckTarget(DECODE_IP, DECODE_PORT, ABORT_GENERATION) + for name, abort_before_send in (("mid-send", True), ("after", False)): + with self.subTest(name=name): + manager = _manager() + if abort_before_send: + manager._deferred_ack_targets[ROOM] = { + (target.ip, target.port): target + } + + self._run_worker_with_failed_send(manager) + self.assertEqual(manager._sent, []) + self.assertNotIn(ROOM, manager._deferred_ack_targets) + + self._dispatch_abort(manager) + self.assertEqual(manager._sent, []) + self.assertNotIn(ROOM, manager._deferred_ack_targets) + + # A new lifecycle of the same bootstrap_room acks again. + manager.request_status.pop(ROOM, None) + CommonKVManager.update_status(manager, ROOM, KVPoll.Bootstrapping) + self._dispatch_abort(manager) + self.assertEqual(manager._sent, [(ROOM, target)]) + + def test_abort_during_failed_state_send_never_acks(self): + """A state send that fails after an ABORT marked the room Failed must + still withhold the drained ACK.""" + manager = _manager() + self._prepare_worker(manager) + manager.maybe_send_extra.side_effect = self._abort_on_call(manager) + + self._run_worker(manager, _last_chunk()) + + self.assertEqual(manager._sent, []) + self.assertIn(ROOM, manager._deferred_ack_poisoned_rooms) + + def test_failed_aux_to_one_decode_rank_never_acks(self): + """An aux send that fails for the first decode rank must withhold the + ACK even when the second rank stops the room before its own aux send.""" + for name, abort_during_aux in (("mid-send", True), ("dead-session", False)): + with self.subTest(name=name): + manager = _manager() + self._prepare_worker(manager, num_dsts=2) + if abort_during_aux: + manager.send_aux.side_effect = self._abort_on_call(manager) + else: + manager.send_aux.return_value = -1 + manager.failed_sessions.add("decode-session-1") + + self._run_worker(manager, _last_chunk()) + self._dispatch_abort(manager) + + manager.send_aux.assert_called_once() + self.assertEqual(manager._sent, []) + def test_feature_off_sends_legacy_ack(self): manager = _manager(enabled=False)