prefill: async SWA state-payload D2H (first version)
Implements the v2 plan in handoff/prefill-state-payload-async-review-20261106.md. - mem_cache/common.py: add kv_to_page_indices_pinned() (pinned CPU buffer, non-blocking copy; no synchronous .cpu()). - prefill.py: gate via should_async_state_payload() (last chunk, independent supports_async_state_indices, kv_shard_size==1, CP guard); only the _swa_payload request_window-is-None hot branch uses the pinned helper. Final segment asserts async state has an indices_event. Other state builders (mamba/full/DSA) stay synchronous. - mooncake/conn.py: MooncakeKVSender advertises supports_async_state_indices; the worker materializes pinned state tensors to numpy only after indices_event.synchronize(), and fails loudly if a tensor arrives without an event. - tests: guard/event/materialize/pinned-body unit tests + CUDA ordering test (skipped unless torch+CUDA+sglang are available). No scheduler/transfer semantics change; rollback is the helper switch or reverting this commit. 49 tests pass, 1 skipped locally.
This commit is contained in:
@@ -357,6 +357,28 @@ def _uses_flat_entry_layout(
|
||||
return mla_backend or force_flat or has_layer_ids or is_single_region_swa
|
||||
|
||||
|
||||
def materialize_state_indices(state_indices, indices_event):
|
||||
"""Convert pinned state tensors to numpy after the event has completed.
|
||||
|
||||
``indices_event`` must already be synchronized by the caller. A tensor
|
||||
without an event is a configuration/implementation error: reading it would
|
||||
race the scheduler's non-blocking D2H copy.
|
||||
"""
|
||||
if not state_indices:
|
||||
return state_indices
|
||||
import torch
|
||||
|
||||
if not any(isinstance(t, torch.Tensor) for t in state_indices):
|
||||
return state_indices
|
||||
if indices_event is None:
|
||||
raise RuntimeError(
|
||||
"async state indices require an indices_event before materialization"
|
||||
)
|
||||
return [
|
||||
t.numpy() if isinstance(t, torch.Tensor) else t for t in state_indices
|
||||
]
|
||||
|
||||
|
||||
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
sender_trace_module = "mooncake"
|
||||
|
||||
@@ -2871,11 +2893,18 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
kv_chunk.wait_event = None
|
||||
_ind_t0 = time.monotonic_ns() if _obs else 0
|
||||
_ind_wall0 = time.time_ns() if _obs else 0
|
||||
if kv_chunk.indices_event is not None:
|
||||
kv_chunk.indices_event.synchronize()
|
||||
indices_event = kv_chunk.indices_event
|
||||
if indices_event is not None:
|
||||
indices_event.synchronize()
|
||||
kv_chunk.indices_event = None
|
||||
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
|
||||
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
|
||||
# First async-state version: the pinned SWA state tensor is
|
||||
# materialized only after the event above. Raises if a tensor
|
||||
# arrives without one (would race the scheduler's D2H).
|
||||
kv_chunk.state_indices = materialize_state_indices(
|
||||
kv_chunk.state_indices, indices_event
|
||||
)
|
||||
if _ind_t0:
|
||||
# synchronize + host indices availability, per design §5.2.
|
||||
_indices_wait_ns += time.monotonic_ns() - _ind_t0
|
||||
@@ -3757,6 +3786,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
class MooncakeKVSender(CommonKVSender):
|
||||
supports_host_tensor_indices = True
|
||||
# First async state-payload version (SWA-only): the scheduler copies the
|
||||
# SWA state indices to a pinned buffer and the worker materializes them
|
||||
# after kv_chunk.indices_event.
|
||||
supports_async_state_indices = True
|
||||
|
||||
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
|
||||
def send(
|
||||
|
||||
@@ -77,6 +77,7 @@ from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
||||
from sglang.srt.mem_cache.common import (
|
||||
checkpoint_kv_cache,
|
||||
kv_to_page_indices,
|
||||
kv_to_page_indices_pinned,
|
||||
kv_to_page_num,
|
||||
release_kv_cache,
|
||||
suffix_page_indices,
|
||||
@@ -564,6 +565,40 @@ class PrefillBootstrapQueue:
|
||||
self.kv_manager.register_buffer_to_engine()
|
||||
|
||||
|
||||
def should_async_state_payload(
|
||||
*,
|
||||
last_chunk: bool,
|
||||
supports_async_state_indices: bool,
|
||||
kv_shard_size: int,
|
||||
enable_all_cp_ranks_for_transfer: bool,
|
||||
enable_dsa_cache_layer_split: bool,
|
||||
) -> bool:
|
||||
"""Gate for the first async state-payload version (SWA-only).
|
||||
|
||||
``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old
|
||||
synchronous path: their indices are filtered before the transfer event,
|
||||
and deferring that to the worker would widen the pre-event shard-filter
|
||||
risk tracked by #45492.
|
||||
"""
|
||||
if not last_chunk or not supports_async_state_indices:
|
||||
return False
|
||||
if kv_shard_size != 1:
|
||||
return False
|
||||
if enable_all_cp_ranks_for_transfer and not enable_dsa_cache_layer_split:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def require_async_state_event(
|
||||
async_state: bool, segment_is_last: bool, indices_event
|
||||
) -> None:
|
||||
"""Fail loudly if an async state payload has no event to protect its copy."""
|
||||
if async_state and segment_is_last and indices_event is None:
|
||||
raise RuntimeError(
|
||||
"async state payload requires an indices_event for the final segment"
|
||||
)
|
||||
|
||||
|
||||
class SchedulerDisaggregationPrefillMixin:
|
||||
"""
|
||||
Mixin for Scheduler to handle disaggregation prefill
|
||||
@@ -1782,6 +1817,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
return
|
||||
|
||||
# First async state-payload version: SWA only, with the shard/CP guards
|
||||
# in should_async_state_payload(). Other builders stay synchronous.
|
||||
sender = req.disagg_kv_sender
|
||||
async_state = should_async_state_payload(
|
||||
last_chunk=last_chunk,
|
||||
supports_async_state_indices=getattr(
|
||||
sender, "supports_async_state_indices", False
|
||||
),
|
||||
kv_shard_size=getattr(sender.kv_mgr, "kv_shard_size", 1),
|
||||
enable_all_cp_ranks_for_transfer=getattr(
|
||||
sender.kv_mgr, "enable_all_cp_ranks_for_transfer", False
|
||||
),
|
||||
enable_dsa_cache_layer_split=get_parallel().enable_dsa_cache_layer_split,
|
||||
)
|
||||
|
||||
state_indices: Optional[List] = None
|
||||
_state_tok = (
|
||||
batch_telemetry.detail_begin("kv_state_payload." + send_kind)
|
||||
@@ -1833,6 +1883,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
window_kv_indices_full
|
||||
)
|
||||
)
|
||||
if async_state:
|
||||
return kv_to_page_indices_pinned(
|
||||
window_kv_indices_swa, page_size
|
||||
)
|
||||
return kv_to_page_indices(window_kv_indices_swa, page_size)
|
||||
|
||||
def _full_kv_pages_payload():
|
||||
@@ -1973,7 +2027,6 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
raw_kv_indices
|
||||
)
|
||||
)
|
||||
sender = req.disagg_kv_sender
|
||||
indices_event = None
|
||||
if getattr(sender, "supports_host_tensor_indices", False) and not (
|
||||
sender.kv_mgr.enable_all_cp_ranks_for_transfer
|
||||
@@ -2001,6 +2054,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
segment_is_last = last_chunk and is_final_segment
|
||||
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
|
||||
continue
|
||||
require_async_state_event(
|
||||
async_state, segment_is_last, indices_event
|
||||
)
|
||||
send_state_indices = state_indices if segment_is_last else None
|
||||
_send_t0 = (
|
||||
time.monotonic_ns()
|
||||
|
||||
@@ -47,6 +47,22 @@ def kv_to_page_indices(kv_indices: torch.Tensor, page_size: int) -> np.ndarray:
|
||||
return (kv_indices[::page_size] // page_size).cpu().numpy()
|
||||
|
||||
|
||||
def kv_to_page_indices_pinned(
|
||||
kv_indices: torch.Tensor, page_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Same values/dtype as :func:`kv_to_page_indices`, in a pinned CPU tensor.
|
||||
|
||||
The device-to-host copy is non-blocking on the current stream. The caller
|
||||
must record a CUDA event on that stream after all such copies and hand it
|
||||
to the transfer worker, which materializes the tensor only after the event
|
||||
(see ``materialize_state_indices``).
|
||||
"""
|
||||
src = kv_indices[::page_size] // page_size
|
||||
out = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
|
||||
out.copy_(src, non_blocking=True)
|
||||
return out
|
||||
|
||||
|
||||
def suffix_page_indices(
|
||||
req_to_token: torch.Tensor,
|
||||
req_pool_idx: int,
|
||||
|
||||
Reference in New Issue
Block a user