[Perf] pd: Send prefill KV chunks without a host sync on the scheduler thread (#45492)
Co-authored-by: alphabetc1 <2508695655@qq.com> Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
co-authored by
alphabetc1
Shuwen Wang
parent
3468a65b51
commit
8dc7f98005
@@ -34,6 +34,8 @@ class TransferKVChunk:
|
||||
staging_counted: bool = False
|
||||
# Early-send: GPU event to synchronize before reading KV (optional).
|
||||
wait_event: Optional[object] = None
|
||||
# Must complete before reading asynchronously copied host indices.
|
||||
indices_event: Optional[object] = None
|
||||
|
||||
|
||||
def pack_list_of_buffers(buffers: List[bytes]) -> bytes:
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Dict, List, Optional, Set, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
import zmq
|
||||
from prometheus_client import Counter
|
||||
|
||||
@@ -2505,6 +2506,11 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
):
|
||||
kv_chunk.wait_event.synchronize()
|
||||
kv_chunk.wait_event = None
|
||||
if kv_chunk.indices_event is not None:
|
||||
kv_chunk.indices_event.synchronize()
|
||||
kv_chunk.indices_event = None
|
||||
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
|
||||
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
|
||||
|
||||
if (
|
||||
kv_chunk.room not in self.request_status
|
||||
@@ -3072,6 +3078,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
wait_event: Optional[object] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -3112,6 +3119,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=trace_ctx,
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3237,12 +3245,15 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
self.init_time = time.time()
|
||||
self._init_trace_ctx()
|
||||
|
||||
supports_host_tensor_indices = True
|
||||
|
||||
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
|
||||
def send(
|
||||
self,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
):
|
||||
kv_indices, index_slice, is_last_chunk, should_skip = (
|
||||
self._prepare_send_indices(kv_indices, state_indices)
|
||||
@@ -3261,6 +3272,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
)
|
||||
else:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
@@ -3273,6 +3285,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
|
||||
@@ -1697,18 +1697,43 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
raw_kv_indices
|
||||
)
|
||||
)
|
||||
page_indices = kv_to_page_indices(kv_indices, page_size)
|
||||
segment_is_last = last_chunk and is_final_segment
|
||||
if not req.disagg_kv_sender.should_send_kv_chunk(
|
||||
len(page_indices), segment_is_last
|
||||
sender = req.disagg_kv_sender
|
||||
indices_event = None
|
||||
if getattr(sender, "supports_host_tensor_indices", False) and not (
|
||||
sender.kv_mgr.enable_all_cp_ranks_for_transfer
|
||||
and not get_parallel().enable_dsa_cache_layer_split
|
||||
):
|
||||
# The schedule stream waits on the in-flight forward; move the
|
||||
# D2H wait to the transfer worker so scheduling can continue.
|
||||
page_indices_dev = kv_indices[::page_size] // page_size
|
||||
page_indices = torch.empty(
|
||||
page_indices_dev.shape,
|
||||
dtype=page_indices_dev.dtype,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
page_indices.copy_(page_indices_dev, non_blocking=True)
|
||||
indices_event = torch.cuda.Event()
|
||||
indices_event.record(torch.cuda.current_stream())
|
||||
else:
|
||||
page_indices = kv_to_page_indices(kv_indices, page_size)
|
||||
segment_is_last = last_chunk and is_final_segment
|
||||
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
|
||||
continue
|
||||
send_state_indices = state_indices if segment_is_last else None
|
||||
req.disagg_kv_sender.send(
|
||||
page_indices,
|
||||
send_state_indices,
|
||||
num_kv_tokens=seg_end - seg_start,
|
||||
)
|
||||
if indices_event is not None:
|
||||
sender.send(
|
||||
page_indices,
|
||||
send_state_indices,
|
||||
num_kv_tokens=seg_end - seg_start,
|
||||
indices_event=indices_event,
|
||||
)
|
||||
else:
|
||||
sender.send(
|
||||
page_indices,
|
||||
send_state_indices,
|
||||
num_kv_tokens=seg_end - seg_start,
|
||||
)
|
||||
req.start_send_idx = end_idx
|
||||
# A last chunk needs no entry: every `last_chunk=True` call site has
|
||||
# already put the request on `disagg_prefill_inflight_queue`.
|
||||
|
||||
@@ -8,6 +8,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.disaggregation.base.conn import KVPoll, StateType
|
||||
from sglang.srt.disaggregation.common.utils import TransferKVChunk
|
||||
@@ -411,7 +412,7 @@ class TestDcpPackLifetime(CustomTestCase):
|
||||
self.assertEqual(observed, [11])
|
||||
|
||||
|
||||
class TestMooncakeEarlySend(unittest.TestCase):
|
||||
class TestMooncakeEarlySend(CustomTestCase):
|
||||
def test_sender_queues_event_once_without_waiting_on_scheduler(self):
|
||||
for last in (False, True):
|
||||
with self.subTest(last=last):
|
||||
@@ -514,6 +515,28 @@ class TestMooncakeEarlySend(unittest.TestCase):
|
||||
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
|
||||
manager.send_kvcache.assert_called_once()
|
||||
|
||||
def test_worker_sends_indices_after_copy_completes(self):
|
||||
manager, queue = self._worker(None)
|
||||
chunk = queue.get_nowait()
|
||||
host_indices = torch.full((1,), -1, dtype=torch.int32)
|
||||
chunk.prefill_kv_indices = host_indices
|
||||
chunk.indices_event = MagicMock()
|
||||
chunk.indices_event.synchronize.side_effect = lambda: host_indices.fill_(7)
|
||||
queue.get_nowait() # Replace the stop sentinel after the updated chunk.
|
||||
queue.put(chunk)
|
||||
queue.put(None)
|
||||
sent_indices = []
|
||||
|
||||
def transfer(session, indices, *args, **kwargs):
|
||||
self.assertIsInstance(indices, np.ndarray)
|
||||
sent_indices.extend(indices.tolist())
|
||||
return 0
|
||||
|
||||
manager.send_kvcache.side_effect = transfer
|
||||
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
|
||||
|
||||
self.assertEqual(sent_indices, [7])
|
||||
|
||||
def test_abort_during_wait_prevents_transfer_and_releases_ack(self):
|
||||
ready = MagicMock()
|
||||
manager, queue = self._worker(ready)
|
||||
|
||||
Reference in New Issue
Block a user