diff --git a/python/sglang/srt/disaggregation/common/utils.py b/python/sglang/srt/disaggregation/common/utils.py index 95991d8744..dab81e2d7c 100644 --- a/python/sglang/srt/disaggregation/common/utils.py +++ b/python/sglang/srt/disaggregation/common/utils.py @@ -32,7 +32,7 @@ class TransferKVChunk: # Set when the staging worker first counts this chunk toward the per-room # outstanding count; stays set across re-enqueue on a watermark defer. staging_counted: bool = False - # Mori early-send: CUDA event to synchronize before RDMA (optional). + # Early-send: GPU event to synchronize before reading KV (optional). wait_event: Optional[object] = None diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index b75f56c2b5..3833b5738c 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2415,6 +2415,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): self._staging_outstanding[kv_chunk.room] += 1 kv_chunk.staging_counted = True + # Under overlap scheduling an early cached-prefix send can + # precede completion of the previous prefill forward. Wait on + # the worker, not the scheduler, before reading source KV. + # A room that already failed skips the wait; the status check + # below then drops the chunk, including on an abort during it. + if kv_chunk.wait_event is not None: + if ( + kv_chunk.room in self.request_status + and self.check_status(kv_chunk.room) != KVPoll.Failed + ): + kv_chunk.wait_event.synchronize() + kv_chunk.wait_event = None + if ( kv_chunk.room not in self.request_status or self.check_status(kv_chunk.room) == KVPoll.Failed @@ -2965,6 +2978,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): state_indices: Optional[List] = None, num_kv_tokens: Optional[int] = None, trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None, + wait_event: Optional[object] = None, ): assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) @@ -3004,6 +3018,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): state_indices=state_indices, num_kv_tokens=num_kv_tokens, trace_ctx=trace_ctx, + wait_event=wait_event, ) ) @@ -3110,6 +3125,8 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): if should_skip: return + wait_event = getattr(self, "_early_send_wait_event", None) + self._early_send_wait_event = None if not is_last_chunk: self.kv_mgr.add_transfer_request( self.bootstrap_room, @@ -3118,6 +3135,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): False, num_kv_tokens=num_kv_tokens, trace_ctx=self.trace_ctx.copy_for_thread(), + wait_event=wait_event, ) else: self.kv_mgr.add_transfer_request( @@ -3129,6 +3147,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): state_indices=state_indices, num_kv_tokens=num_kv_tokens, trace_ctx=self.trace_ctx.copy_for_thread(), + wait_event=wait_event, ) self._record_transfer_indices(kv_indices, state_indices) diff --git a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py index d22af73993..3139f1cd89 100644 --- a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py +++ b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py @@ -1,14 +1,18 @@ import concurrent.futures import ctypes import unittest -from threading import Event +from collections import defaultdict +from queue import SimpleQueue +from threading import Event, Lock from types import SimpleNamespace from unittest.mock import MagicMock, call, patch import numpy as np -from sglang.srt.disaggregation.base.conn import StateType -from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager +from sglang.srt.disaggregation.base.conn import KVPoll, StateType +from sglang.srt.disaggregation.common.utils import TransferKVChunk +from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager, MooncakeKVSender +from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -407,5 +411,136 @@ class TestDcpPackLifetime(CustomTestCase): self.assertEqual(observed, [11]) +class TestMooncakeEarlySend(unittest.TestCase): + def test_sender_queues_event_once_without_waiting_on_scheduler(self): + for last in (False, True): + with self.subTest(last=last): + ready = MagicMock() + manager = MagicMock() + sender = SimpleNamespace( + kv_mgr=manager, + bootstrap_room=1, + aux_index=0, + trace_ctx=MagicMock(), + _early_send_wait_event=ready, + _prepare_send_indices=lambda ids, state: ( + ids, + slice(0, 1), + last, + False, + ), + _record_transfer_indices=MagicMock(), + ) + MooncakeKVSender.send(sender, [0]) + self.assertIs( + manager.add_transfer_request.call_args.kwargs["wait_event"], ready + ) + ready.synchronize.assert_not_called() + MooncakeKVSender.send(sender, [1]) + self.assertIsNone( + manager.add_transfer_request.call_args.kwargs["wait_event"] + ) + + def test_manager_preserves_event_in_queued_chunk(self): + queue = SimpleQueue() + ready = MagicMock() + manager = SimpleNamespace( + disaggregation_mode=DisaggregationMode.PREFILL, + request_status={1: KVPoll.WaitingForInput}, + check_status=lambda room: KVPoll.WaitingForInput, + transfer_infos={1: {"127.0.0.1:80": object()}}, + transfer_queues=[queue], + ) + MooncakeKVManager.add_transfer_request( + manager, 1, [0], slice(0, 1), False, wait_event=ready + ) + self.assertIs(queue.get_nowait().wait_event, ready) + ready.synchronize.assert_not_called() + + def _worker(self, event): + chunk = TransferKVChunk( + 1, [0], slice(0, 1), False, None, None, wait_event=event + ) + queue = SimpleQueue() + queue.put(chunk) + queue.put(None) + peer = SimpleNamespace( + is_dummy=False, + mooncake_session_id="peer", + dst_device_kv_indices=None, + dst_kv_indices=[10], + ) + registration = SimpleNamespace( + requires_dcp_relayout=False, + dst_kv_ptrs=[1000], + dst_kv_layer_ids=[0], + dst_kv_item_len=16, + dst_attn_tp_size=1, + ) + manager = SimpleNamespace( + enable_trace=False, + enable_staging=False, + enable_deferred_decode_kv_release=True, + _staging_outstanding=defaultdict(int), + request_status={1: KVPoll.WaitingForInput}, + transfer_infos={1: {"peer": peer}}, + _prefill_unique_rank=lambda: 0, + session_lock=Lock(), + failed_sessions=set(), + # Read under session_lock alongside failed_sessions on every + # non-dummy req, and invoked by the worker's except path. + state_layout_rejections={}, + poison_deferred_ack_room=MagicMock(), + decode_kv_args_table={"peer": registration}, + _get_dsa_cache_transfer_skip_flags=lambda reg: (False, False), + kv_args=SimpleNamespace(kv_data_ptrs=[1]), + is_mla_backend=True, + send_kvcache=MagicMock(return_value=0), + _maybe_ack_drained_abort=MagicMock(), + bootstrap_port=8998, + ) + manager.check_status = lambda room: manager.request_status[room] + return manager, queue + + def test_worker_waits_before_reading_kv(self): + ready = MagicMock() + manager, queue = self._worker(ready) + + def transfer(*args, **kwargs): + ready.synchronize.assert_called_once_with() + return 0 + + manager.send_kvcache.side_effect = transfer + MooncakeKVManager.transfer_worker(manager, queue, MagicMock()) + manager.send_kvcache.assert_called_once() + + def test_abort_during_wait_prevents_transfer_and_releases_ack(self): + ready = MagicMock() + manager, queue = self._worker(ready) + ready.synchronize.side_effect = lambda: manager.request_status.update( + {1: KVPoll.Failed} + ) + MooncakeKVManager.transfer_worker(manager, queue, MagicMock()) + ready.synchronize.assert_called_once_with() + manager.send_kvcache.assert_not_called() + manager._maybe_ack_drained_abort.assert_called_once_with(1) + self.assertNotIn(1, manager._staging_outstanding) + + def test_failed_room_skips_wait(self): + ready = MagicMock() + manager, queue = self._worker(ready) + manager.request_status[1] = KVPoll.Failed + MooncakeKVManager.transfer_worker(manager, queue, MagicMock()) + ready.synchronize.assert_not_called() + manager.send_kvcache.assert_not_called() + manager._maybe_ack_drained_abort.assert_called_once_with(1) + self.assertNotIn(1, manager._staging_outstanding) + + def test_worker_preserves_no_event_path(self): + manager, queue = self._worker(None) + MooncakeKVManager.transfer_worker(manager, queue, MagicMock()) + manager.send_kvcache.assert_called_once() + + if __name__ == "__main__": unittest.main()