From 8dc7f98005dce2f9b297840d95045fc4e8cf9de7 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Wed, 21 Oct 2026 20:11:16 -0700 Subject: [PATCH] [Perf] pd: Send prefill KV chunks without a host sync on the scheduler thread (#45492) Co-authored-by: alphabetc1 <2508695655@qq.com> Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> --- .../sglang/srt/disaggregation/common/utils.py | 2 + .../srt/disaggregation/mooncake/conn.py | 13 ++++++ python/sglang/srt/disaggregation/prefill.py | 43 +++++++++++++++---- .../test_mooncake_transfer_batching.py | 25 ++++++++++- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/python/sglang/srt/disaggregation/common/utils.py b/python/sglang/srt/disaggregation/common/utils.py index dab81e2d7c..9a21ff624f 100644 --- a/python/sglang/srt/disaggregation/common/utils.py +++ b/python/sglang/srt/disaggregation/common/utils.py @@ -34,6 +34,8 @@ class TransferKVChunk: staging_counted: bool = False # Early-send: GPU event to synchronize before reading KV (optional). wait_event: Optional[object] = None + # Must complete before reading asynchronously copied host indices. + indices_event: Optional[object] = None def pack_list_of_buffers(buffers: List[bytes]) -> bytes: diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index cfac8907fc..e40bf982a2 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -12,6 +12,7 @@ from typing import Dict, List, Optional, Set, Tuple, Union import numpy as np import numpy.typing as npt +import torch import zmq from prometheus_client import Counter @@ -2505,6 +2506,11 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ): kv_chunk.wait_event.synchronize() kv_chunk.wait_event = None + if kv_chunk.indices_event is not None: + kv_chunk.indices_event.synchronize() + kv_chunk.indices_event = None + if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor): + kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy() if ( kv_chunk.room not in self.request_status @@ -3072,6 +3078,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): num_kv_tokens: Optional[int] = None, trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None, wait_event: Optional[object] = None, + indices_event: Optional[object] = None, ): assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) @@ -3112,6 +3119,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): num_kv_tokens=num_kv_tokens, trace_ctx=trace_ctx, wait_event=wait_event, + indices_event=indices_event, ) ) @@ -3237,12 +3245,15 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): self.init_time = time.time() self._init_trace_ctx() + supports_host_tensor_indices = True + @mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND) def send( self, kv_indices: npt.NDArray[np.int32], state_indices: Optional[List] = None, num_kv_tokens: Optional[int] = None, + indices_event: Optional[object] = None, ): kv_indices, index_slice, is_last_chunk, should_skip = ( self._prepare_send_indices(kv_indices, state_indices) @@ -3261,6 +3272,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): num_kv_tokens=num_kv_tokens, trace_ctx=self.trace_ctx.copy_for_thread(), wait_event=wait_event, + indices_event=indices_event, ) else: self.kv_mgr.add_transfer_request( @@ -3273,6 +3285,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): num_kv_tokens=num_kv_tokens, trace_ctx=self.trace_ctx.copy_for_thread(), wait_event=wait_event, + indices_event=indices_event, ) self._record_transfer_indices(kv_indices, state_indices) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 5f85be3815..96b95faa0b 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1697,18 +1697,43 @@ class SchedulerDisaggregationPrefillMixin: raw_kv_indices ) ) - page_indices = kv_to_page_indices(kv_indices, page_size) - segment_is_last = last_chunk and is_final_segment - if not req.disagg_kv_sender.should_send_kv_chunk( - len(page_indices), segment_is_last + sender = req.disagg_kv_sender + indices_event = None + if getattr(sender, "supports_host_tensor_indices", False) and not ( + sender.kv_mgr.enable_all_cp_ranks_for_transfer + and not get_parallel().enable_dsa_cache_layer_split ): + # The schedule stream waits on the in-flight forward; move the + # D2H wait to the transfer worker so scheduling can continue. + page_indices_dev = kv_indices[::page_size] // page_size + page_indices = torch.empty( + page_indices_dev.shape, + dtype=page_indices_dev.dtype, + device="cpu", + pin_memory=True, + ) + page_indices.copy_(page_indices_dev, non_blocking=True) + indices_event = torch.cuda.Event() + indices_event.record(torch.cuda.current_stream()) + else: + page_indices = kv_to_page_indices(kv_indices, page_size) + segment_is_last = last_chunk and is_final_segment + if not sender.should_send_kv_chunk(len(page_indices), segment_is_last): continue send_state_indices = state_indices if segment_is_last else None - req.disagg_kv_sender.send( - page_indices, - send_state_indices, - num_kv_tokens=seg_end - seg_start, - ) + if indices_event is not None: + sender.send( + page_indices, + send_state_indices, + num_kv_tokens=seg_end - seg_start, + indices_event=indices_event, + ) + else: + sender.send( + page_indices, + send_state_indices, + num_kv_tokens=seg_end - seg_start, + ) req.start_send_idx = end_idx # A last chunk needs no entry: every `last_chunk=True` call site has # already put the request on `disagg_prefill_inflight_queue`. diff --git a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py index 0c8723a9cd..49479f6581 100644 --- a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py +++ b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py @@ -8,6 +8,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, call, patch import numpy as np +import torch from sglang.srt.disaggregation.base.conn import KVPoll, StateType from sglang.srt.disaggregation.common.utils import TransferKVChunk @@ -411,7 +412,7 @@ class TestDcpPackLifetime(CustomTestCase): self.assertEqual(observed, [11]) -class TestMooncakeEarlySend(unittest.TestCase): +class TestMooncakeEarlySend(CustomTestCase): def test_sender_queues_event_once_without_waiting_on_scheduler(self): for last in (False, True): with self.subTest(last=last): @@ -514,6 +515,28 @@ class TestMooncakeEarlySend(unittest.TestCase): MooncakeKVManager.transfer_worker(manager, queue, MagicMock()) manager.send_kvcache.assert_called_once() + def test_worker_sends_indices_after_copy_completes(self): + manager, queue = self._worker(None) + chunk = queue.get_nowait() + host_indices = torch.full((1,), -1, dtype=torch.int32) + chunk.prefill_kv_indices = host_indices + chunk.indices_event = MagicMock() + chunk.indices_event.synchronize.side_effect = lambda: host_indices.fill_(7) + queue.get_nowait() # Replace the stop sentinel after the updated chunk. + queue.put(chunk) + queue.put(None) + sent_indices = [] + + def transfer(session, indices, *args, **kwargs): + self.assertIsInstance(indices, np.ndarray) + sent_indices.extend(indices.tolist()) + return 0 + + manager.send_kvcache.side_effect = transfer + MooncakeKVManager.transfer_worker(manager, queue, MagicMock()) + + self.assertEqual(sent_indices, [7]) + def test_abort_during_wait_prevents_transfer_and_releases_ack(self): ready = MagicMock() manager, queue = self._worker(ready)