prefill: async SWA state-payload D2H (first version)

Implements the v2 plan in handoff/prefill-state-payload-async-review-20261106.md.

- mem_cache/common.py: add kv_to_page_indices_pinned() (pinned CPU buffer,
  non-blocking copy; no synchronous .cpu()).
- prefill.py: gate via should_async_state_payload() (last chunk, independent
  supports_async_state_indices, kv_shard_size==1, CP guard); only the
  _swa_payload request_window-is-None hot branch uses the pinned helper.
  Final segment asserts async state has an indices_event. Other state
  builders (mamba/full/DSA) stay synchronous.
- mooncake/conn.py: MooncakeKVSender advertises
  supports_async_state_indices; the worker materializes pinned state tensors
  to numpy only after indices_event.synchronize(), and fails loudly if a
  tensor arrives without an event.
- tests: guard/event/materialize/pinned-body unit tests + CUDA ordering test
  (skipped unless torch+CUDA+sglang are available).

No scheduler/transfer semantics change; rollback is the helper switch or
reverting this commit. 49 tests pass, 1 skipped locally.
This commit is contained in:
ymk
2026-11-06 23:00:22 +08:00
parent fe9c6574fe
commit 3b0baa3455
4 changed files with 373 additions and 3 deletions
@@ -357,6 +357,28 @@ def _uses_flat_entry_layout(
return mla_backend or force_flat or has_layer_ids or is_single_region_swa
def materialize_state_indices(state_indices, indices_event):
"""Convert pinned state tensors to numpy after the event has completed.
``indices_event`` must already be synchronized by the caller. A tensor
without an event is a configuration/implementation error: reading it would
race the scheduler's non-blocking D2H copy.
"""
if not state_indices:
return state_indices
import torch
if not any(isinstance(t, torch.Tensor) for t in state_indices):
return state_indices
if indices_event is None:
raise RuntimeError(
"async state indices require an indices_event before materialization"
)
return [
t.numpy() if isinstance(t, torch.Tensor) else t for t in state_indices
]
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
sender_trace_module = "mooncake"
@@ -2871,11 +2893,18 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
kv_chunk.wait_event = None
_ind_t0 = time.monotonic_ns() if _obs else 0
_ind_wall0 = time.time_ns() if _obs else 0
if kv_chunk.indices_event is not None:
kv_chunk.indices_event.synchronize()
indices_event = kv_chunk.indices_event
if indices_event is not None:
indices_event.synchronize()
kv_chunk.indices_event = None
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
# First async-state version: the pinned SWA state tensor is
# materialized only after the event above. Raises if a tensor
# arrives without one (would race the scheduler's D2H).
kv_chunk.state_indices = materialize_state_indices(
kv_chunk.state_indices, indices_event
)
if _ind_t0:
# synchronize + host indices availability, per design §5.2.
_indices_wait_ns += time.monotonic_ns() - _ind_t0
@@ -3757,6 +3786,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
class MooncakeKVSender(CommonKVSender):
supports_host_tensor_indices = True
# First async state-payload version (SWA-only): the scheduler copies the
# SWA state indices to a pinned buffer and the worker materializes them
# after kv_chunk.indices_event.
supports_async_state_indices = True
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
def send(
+57 -1
View File
@@ -77,6 +77,7 @@ from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
from sglang.srt.mem_cache.common import (
checkpoint_kv_cache,
kv_to_page_indices,
kv_to_page_indices_pinned,
kv_to_page_num,
release_kv_cache,
suffix_page_indices,
@@ -564,6 +565,40 @@ class PrefillBootstrapQueue:
self.kv_manager.register_buffer_to_engine()
def should_async_state_payload(
*,
last_chunk: bool,
supports_async_state_indices: bool,
kv_shard_size: int,
enable_all_cp_ranks_for_transfer: bool,
enable_dsa_cache_layer_split: bool,
) -> bool:
"""Gate for the first async state-payload version (SWA-only).
``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old
synchronous path: their indices are filtered before the transfer event,
and deferring that to the worker would widen the pre-event shard-filter
risk tracked by #45492.
"""
if not last_chunk or not supports_async_state_indices:
return False
if kv_shard_size != 1:
return False
if enable_all_cp_ranks_for_transfer and not enable_dsa_cache_layer_split:
return False
return True
def require_async_state_event(
async_state: bool, segment_is_last: bool, indices_event
) -> None:
"""Fail loudly if an async state payload has no event to protect its copy."""
if async_state and segment_is_last and indices_event is None:
raise RuntimeError(
"async state payload requires an indices_event for the final segment"
)
class SchedulerDisaggregationPrefillMixin:
"""
Mixin for Scheduler to handle disaggregation prefill
@@ -1782,6 +1817,21 @@ class SchedulerDisaggregationPrefillMixin:
)
return
# First async state-payload version: SWA only, with the shard/CP guards
# in should_async_state_payload(). Other builders stay synchronous.
sender = req.disagg_kv_sender
async_state = should_async_state_payload(
last_chunk=last_chunk,
supports_async_state_indices=getattr(
sender, "supports_async_state_indices", False
),
kv_shard_size=getattr(sender.kv_mgr, "kv_shard_size", 1),
enable_all_cp_ranks_for_transfer=getattr(
sender.kv_mgr, "enable_all_cp_ranks_for_transfer", False
),
enable_dsa_cache_layer_split=get_parallel().enable_dsa_cache_layer_split,
)
state_indices: Optional[List] = None
_state_tok = (
batch_telemetry.detail_begin("kv_state_payload." + send_kind)
@@ -1833,6 +1883,10 @@ class SchedulerDisaggregationPrefillMixin:
window_kv_indices_full
)
)
if async_state:
return kv_to_page_indices_pinned(
window_kv_indices_swa, page_size
)
return kv_to_page_indices(window_kv_indices_swa, page_size)
def _full_kv_pages_payload():
@@ -1973,7 +2027,6 @@ class SchedulerDisaggregationPrefillMixin:
raw_kv_indices
)
)
sender = req.disagg_kv_sender
indices_event = None
if getattr(sender, "supports_host_tensor_indices", False) and not (
sender.kv_mgr.enable_all_cp_ranks_for_transfer
@@ -2001,6 +2054,9 @@ class SchedulerDisaggregationPrefillMixin:
segment_is_last = last_chunk and is_final_segment
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
continue
require_async_state_event(
async_state, segment_is_last, indices_event
)
send_state_indices = state_indices if segment_is_last else None
_send_t0 = (
time.monotonic_ns()
+16
View File
@@ -47,6 +47,22 @@ def kv_to_page_indices(kv_indices: torch.Tensor, page_size: int) -> np.ndarray:
return (kv_indices[::page_size] // page_size).cpu().numpy()
def kv_to_page_indices_pinned(
kv_indices: torch.Tensor, page_size: int
) -> torch.Tensor:
"""Same values/dtype as :func:`kv_to_page_indices`, in a pinned CPU tensor.
The device-to-host copy is non-blocking on the current stream. The caller
must record a CUDA event on that stream after all such copies and hand it
to the transfer worker, which materializes the tensor only after the event
(see ``materialize_state_indices``).
"""
src = kv_indices[::page_size] // page_size
out = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
out.copy_(src, non_blocking=True)
return out
def suffix_page_indices(
req_to_token: torch.Tensor,
req_pool_idx: int,