[Perf] pd: Send prefill KV chunks without a host sync on the scheduler thread (#45492)

Co-authored-by: alphabetc1 <2508695655@qq.com>
Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
Khoa Pham
2026-10-21 20:11:16 -07:00
committed by GitHub
co-authored by alphabetc1 Shuwen Wang
parent 3468a65b51
commit 8dc7f98005
4 changed files with 73 additions and 10 deletions
@@ -34,6 +34,8 @@ class TransferKVChunk:
staging_counted: bool = False
# Early-send: GPU event to synchronize before reading KV (optional).
wait_event: Optional[object] = None
# Must complete before reading asynchronously copied host indices.
indices_event: Optional[object] = None
def pack_list_of_buffers(buffers: List[bytes]) -> bytes:
@@ -12,6 +12,7 @@ from typing import Dict, List, Optional, Set, Tuple, Union
import numpy as np
import numpy.typing as npt
import torch
import zmq
from prometheus_client import Counter
@@ -2505,6 +2506,11 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
):
kv_chunk.wait_event.synchronize()
kv_chunk.wait_event = None
if kv_chunk.indices_event is not None:
kv_chunk.indices_event.synchronize()
kv_chunk.indices_event = None
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
if (
kv_chunk.room not in self.request_status
@@ -3072,6 +3078,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
num_kv_tokens: Optional[int] = None,
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
wait_event: Optional[object] = None,
indices_event: Optional[object] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -3112,6 +3119,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
num_kv_tokens=num_kv_tokens,
trace_ctx=trace_ctx,
wait_event=wait_event,
indices_event=indices_event,
)
)
@@ -3237,12 +3245,15 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
self.init_time = time.time()
self._init_trace_ctx()
supports_host_tensor_indices = True
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
def send(
self,
kv_indices: npt.NDArray[np.int32],
state_indices: Optional[List] = None,
num_kv_tokens: Optional[int] = None,
indices_event: Optional[object] = None,
):
kv_indices, index_slice, is_last_chunk, should_skip = (
self._prepare_send_indices(kv_indices, state_indices)
@@ -3261,6 +3272,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
indices_event=indices_event,
)
else:
self.kv_mgr.add_transfer_request(
@@ -3273,6 +3285,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
indices_event=indices_event,
)
self._record_transfer_indices(kv_indices, state_indices)
+34 -9
View File
@@ -1697,18 +1697,43 @@ class SchedulerDisaggregationPrefillMixin:
raw_kv_indices
)
)
page_indices = kv_to_page_indices(kv_indices, page_size)
segment_is_last = last_chunk and is_final_segment
if not req.disagg_kv_sender.should_send_kv_chunk(
len(page_indices), segment_is_last
sender = req.disagg_kv_sender
indices_event = None
if getattr(sender, "supports_host_tensor_indices", False) and not (
sender.kv_mgr.enable_all_cp_ranks_for_transfer
and not get_parallel().enable_dsa_cache_layer_split
):
# The schedule stream waits on the in-flight forward; move the
# D2H wait to the transfer worker so scheduling can continue.
page_indices_dev = kv_indices[::page_size] // page_size
page_indices = torch.empty(
page_indices_dev.shape,
dtype=page_indices_dev.dtype,
device="cpu",
pin_memory=True,
)
page_indices.copy_(page_indices_dev, non_blocking=True)
indices_event = torch.cuda.Event()
indices_event.record(torch.cuda.current_stream())
else:
page_indices = kv_to_page_indices(kv_indices, page_size)
segment_is_last = last_chunk and is_final_segment
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
continue
send_state_indices = state_indices if segment_is_last else None
req.disagg_kv_sender.send(
page_indices,
send_state_indices,
num_kv_tokens=seg_end - seg_start,
)
if indices_event is not None:
sender.send(
page_indices,
send_state_indices,
num_kv_tokens=seg_end - seg_start,
indices_event=indices_event,
)
else:
sender.send(
page_indices,
send_state_indices,
num_kv_tokens=seg_end - seg_start,
)
req.start_send_idx = end_idx
# A last chunk needs no entry: every `last_chunk=True` call site has
# already put the request on `disagg_prefill_inflight_queue`.
@@ -8,6 +8,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, call, patch
import numpy as np
import torch
from sglang.srt.disaggregation.base.conn import KVPoll, StateType
from sglang.srt.disaggregation.common.utils import TransferKVChunk
@@ -411,7 +412,7 @@ class TestDcpPackLifetime(CustomTestCase):
self.assertEqual(observed, [11])
class TestMooncakeEarlySend(unittest.TestCase):
class TestMooncakeEarlySend(CustomTestCase):
def test_sender_queues_event_once_without_waiting_on_scheduler(self):
for last in (False, True):
with self.subTest(last=last):
@@ -514,6 +515,28 @@ class TestMooncakeEarlySend(unittest.TestCase):
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
manager.send_kvcache.assert_called_once()
def test_worker_sends_indices_after_copy_completes(self):
manager, queue = self._worker(None)
chunk = queue.get_nowait()
host_indices = torch.full((1,), -1, dtype=torch.int32)
chunk.prefill_kv_indices = host_indices
chunk.indices_event = MagicMock()
chunk.indices_event.synchronize.side_effect = lambda: host_indices.fill_(7)
queue.get_nowait() # Replace the stop sentinel after the updated chunk.
queue.put(chunk)
queue.put(None)
sent_indices = []
def transfer(session, indices, *args, **kwargs):
self.assertIsInstance(indices, np.ndarray)
sent_indices.extend(indices.tolist())
return 0
manager.send_kvcache.side_effect = transfer
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
self.assertEqual(sent_indices, [7])
def test_abort_during_wait_prevents_transfer_and_releases_ack(self):
ready = MagicMock()
manager, queue = self._worker(ready)