prefill: async SWA state-payload D2H (first version)

Implements the v2 plan in handoff/prefill-state-payload-async-review-20261106.md.

- mem_cache/common.py: add kv_to_page_indices_pinned() (pinned CPU buffer,
  non-blocking copy; no synchronous .cpu()).
- prefill.py: gate via should_async_state_payload() (last chunk, independent
  supports_async_state_indices, kv_shard_size==1, CP guard); only the
  _swa_payload request_window-is-None hot branch uses the pinned helper.
  Final segment asserts async state has an indices_event. Other state
  builders (mamba/full/DSA) stay synchronous.
- mooncake/conn.py: MooncakeKVSender advertises
  supports_async_state_indices; the worker materializes pinned state tensors
  to numpy only after indices_event.synchronize(), and fails loudly if a
  tensor arrives without an event.
- tests: guard/event/materialize/pinned-body unit tests + CUDA ordering test
  (skipped unless torch+CUDA+sglang are available).

No scheduler/transfer semantics change; rollback is the helper switch or
reverting this commit. 49 tests pass, 1 skipped locally.
This commit is contained in:
ymk
2026-11-06 23:00:22 +08:00
parent fe9c6574fe
commit 3b0baa3455
4 changed files with 373 additions and 3 deletions
@@ -357,6 +357,28 @@ def _uses_flat_entry_layout(
return mla_backend or force_flat or has_layer_ids or is_single_region_swa
def materialize_state_indices(state_indices, indices_event):
"""Convert pinned state tensors to numpy after the event has completed.
``indices_event`` must already be synchronized by the caller. A tensor
without an event is a configuration/implementation error: reading it would
race the scheduler's non-blocking D2H copy.
"""
if not state_indices:
return state_indices
import torch
if not any(isinstance(t, torch.Tensor) for t in state_indices):
return state_indices
if indices_event is None:
raise RuntimeError(
"async state indices require an indices_event before materialization"
)
return [
t.numpy() if isinstance(t, torch.Tensor) else t for t in state_indices
]
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
sender_trace_module = "mooncake"
@@ -2871,11 +2893,18 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
kv_chunk.wait_event = None
_ind_t0 = time.monotonic_ns() if _obs else 0
_ind_wall0 = time.time_ns() if _obs else 0
if kv_chunk.indices_event is not None:
kv_chunk.indices_event.synchronize()
indices_event = kv_chunk.indices_event
if indices_event is not None:
indices_event.synchronize()
kv_chunk.indices_event = None
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
# First async-state version: the pinned SWA state tensor is
# materialized only after the event above. Raises if a tensor
# arrives without one (would race the scheduler's D2H).
kv_chunk.state_indices = materialize_state_indices(
kv_chunk.state_indices, indices_event
)
if _ind_t0:
# synchronize + host indices availability, per design §5.2.
_indices_wait_ns += time.monotonic_ns() - _ind_t0
@@ -3757,6 +3786,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
class MooncakeKVSender(CommonKVSender):
supports_host_tensor_indices = True
# First async state-payload version (SWA-only): the scheduler copies the
# SWA state indices to a pinned buffer and the worker materializes them
# after kv_chunk.indices_event.
supports_async_state_indices = True
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
def send(
+57 -1
View File
@@ -77,6 +77,7 @@ from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
from sglang.srt.mem_cache.common import (
checkpoint_kv_cache,
kv_to_page_indices,
kv_to_page_indices_pinned,
kv_to_page_num,
release_kv_cache,
suffix_page_indices,
@@ -564,6 +565,40 @@ class PrefillBootstrapQueue:
self.kv_manager.register_buffer_to_engine()
def should_async_state_payload(
*,
last_chunk: bool,
supports_async_state_indices: bool,
kv_shard_size: int,
enable_all_cp_ranks_for_transfer: bool,
enable_dsa_cache_layer_split: bool,
) -> bool:
"""Gate for the first async state-payload version (SWA-only).
``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old
synchronous path: their indices are filtered before the transfer event,
and deferring that to the worker would widen the pre-event shard-filter
risk tracked by #45492.
"""
if not last_chunk or not supports_async_state_indices:
return False
if kv_shard_size != 1:
return False
if enable_all_cp_ranks_for_transfer and not enable_dsa_cache_layer_split:
return False
return True
def require_async_state_event(
async_state: bool, segment_is_last: bool, indices_event
) -> None:
"""Fail loudly if an async state payload has no event to protect its copy."""
if async_state and segment_is_last and indices_event is None:
raise RuntimeError(
"async state payload requires an indices_event for the final segment"
)
class SchedulerDisaggregationPrefillMixin:
"""
Mixin for Scheduler to handle disaggregation prefill
@@ -1782,6 +1817,21 @@ class SchedulerDisaggregationPrefillMixin:
)
return
# First async state-payload version: SWA only, with the shard/CP guards
# in should_async_state_payload(). Other builders stay synchronous.
sender = req.disagg_kv_sender
async_state = should_async_state_payload(
last_chunk=last_chunk,
supports_async_state_indices=getattr(
sender, "supports_async_state_indices", False
),
kv_shard_size=getattr(sender.kv_mgr, "kv_shard_size", 1),
enable_all_cp_ranks_for_transfer=getattr(
sender.kv_mgr, "enable_all_cp_ranks_for_transfer", False
),
enable_dsa_cache_layer_split=get_parallel().enable_dsa_cache_layer_split,
)
state_indices: Optional[List] = None
_state_tok = (
batch_telemetry.detail_begin("kv_state_payload." + send_kind)
@@ -1833,6 +1883,10 @@ class SchedulerDisaggregationPrefillMixin:
window_kv_indices_full
)
)
if async_state:
return kv_to_page_indices_pinned(
window_kv_indices_swa, page_size
)
return kv_to_page_indices(window_kv_indices_swa, page_size)
def _full_kv_pages_payload():
@@ -1973,7 +2027,6 @@ class SchedulerDisaggregationPrefillMixin:
raw_kv_indices
)
)
sender = req.disagg_kv_sender
indices_event = None
if getattr(sender, "supports_host_tensor_indices", False) and not (
sender.kv_mgr.enable_all_cp_ranks_for_transfer
@@ -2001,6 +2054,9 @@ class SchedulerDisaggregationPrefillMixin:
segment_is_last = last_chunk and is_final_segment
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
continue
require_async_state_event(
async_state, segment_is_last, indices_event
)
send_state_indices = state_indices if segment_is_last else None
_send_t0 = (
time.monotonic_ns()
+16
View File
@@ -47,6 +47,22 @@ def kv_to_page_indices(kv_indices: torch.Tensor, page_size: int) -> np.ndarray:
return (kv_indices[::page_size] // page_size).cpu().numpy()
def kv_to_page_indices_pinned(
kv_indices: torch.Tensor, page_size: int
) -> torch.Tensor:
"""Same values/dtype as :func:`kv_to_page_indices`, in a pinned CPU tensor.
The device-to-host copy is non-blocking on the current stream. The caller
must record a CUDA event on that stream after all such copies and hand it
to the transfer worker, which materializes the tensor only after the event
(see ``materialize_state_indices``).
"""
src = kv_indices[::page_size] // page_size
out = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
out.copy_(src, non_blocking=True)
return out
def suffix_page_indices(
req_to_token: torch.Tensor,
req_pool_idx: int,
@@ -0,0 +1,265 @@
"""Unit tests for the SWA-only async state-payload first version.
Covers the v2 checklist that can run without a live server:
* should_async_state_payload guards (last chunk / capability / shard / CP);
* require_async_state_event (empty last chunk / multi segment invariant);
* worker materialize_state_indices (event ordering, no-event failure,
non-tensor and None entries preserved);
* kv_to_page_indices_pinned body: pinned CPU buffer, non-blocking copy,
same dtype/shape, no synchronous .cpu();
* a CUDA ordering test (skipped unless torch + CUDA + sglang are importable)
proving the helper returns before a preceding long kernel finishes and
the event-synchronized values are correct.
The functions are extracted from the real source with ``ast`` so the test does
not need to import the heavy SGLang modules. Full DSV4 component / abort /
multi-segment integration cases run in the server harness on b300.
"""
import ast
import pathlib
import sys
import time
import types
import unittest
from unittest import mock
import numpy as np
def _repo_root():
return pathlib.Path(__file__).resolve().parents[4]
def _load_function(rel_path, name, extra_ns=None):
path = _repo_root() / rel_path
tree = ast.parse(path.read_text(encoding="utf-8"))
fn = next(
node
for node in tree.body
if isinstance(node, ast.FunctionDef) and node.name == name
)
namespace = dict(extra_ns or {})
module = ast.Module(body=[fn], type_ignores=[])
exec(compile(module, str(path), "exec"), namespace)
return namespace[name]
def _load_pinned_helper():
"""Execute the helper body from the real source with a fake torch."""
path = _repo_root() / "python/sglang/srt/mem_cache/common.py"
tree = ast.parse(path.read_text(encoding="utf-8"))
fn = next(
node
for node in tree.body
if isinstance(node, ast.FunctionDef) and node.name == "kv_to_page_indices_pinned"
)
calls = {"empty": [], "copy": []}
class _FakeTensor:
def __init__(self, values, dtype="torch.int32", shape=None):
self.values = list(values)
self.dtype = dtype
self.shape = (len(self.values),) if shape is None else shape
def __getitem__(self, sl):
return _FakeTensor(self.values[sl], self.dtype)
def __floordiv__(self, value):
return _FakeTensor([v // value for v in self.values], self.dtype)
def cpu(self):
raise AssertionError("kv_to_page_indices_pinned must not call .cpu()")
def numpy(self):
raise AssertionError(
"kv_to_page_indices_pinned must not call .numpy()"
)
def copy_(self, src, non_blocking=False):
calls["copy"].append((src, non_blocking))
self.values = list(src.values)
class _FakeTorch:
Tensor = _FakeTensor
@staticmethod
def empty(shape, dtype, device, pin_memory):
calls["empty"].append(
{"shape": shape, "dtype": dtype, "device": device, "pin": pin_memory}
)
return _FakeTensor([], dtype=dtype, shape=shape)
namespace = {"torch": _FakeTorch}
module = ast.Module(body=[fn], type_ignores=[])
exec(compile(module, str(path), "exec"), namespace)
return namespace["kv_to_page_indices_pinned"], calls, _FakeTensor
class AsyncStateGateTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.mod = types.SimpleNamespace()
base = "python/sglang/srt/disaggregation/prefill.py"
cls.mod.should_async_state_payload = _load_function(
base, "should_async_state_payload"
)
cls.mod.require_async_state_event = _load_function(
base, "require_async_state_event"
)
def test_guards(self):
mod = self.mod
base = dict(
last_chunk=True,
supports_async_state_indices=True,
kv_shard_size=1,
enable_all_cp_ranks_for_transfer=False,
enable_dsa_cache_layer_split=False,
)
self.assertTrue(mod.should_async_state_payload(**base))
self.assertFalse(
mod.should_async_state_payload(**{**base, "last_chunk": False})
)
self.assertFalse(
mod.should_async_state_payload(
**{**base, "supports_async_state_indices": False}
)
)
self.assertFalse(
mod.should_async_state_payload(**{**base, "kv_shard_size": 2})
)
self.assertFalse(
mod.should_async_state_payload(
**{**base, "enable_all_cp_ranks_for_transfer": True}
)
)
self.assertTrue(
mod.should_async_state_payload(
**{
**base,
"enable_all_cp_ranks_for_transfer": True,
"enable_dsa_cache_layer_split": True,
}
)
)
def test_final_event_required(self):
mod = self.mod
mod.require_async_state_event(True, True, object()) # no raise
mod.require_async_state_event(False, True, None) # sync path
mod.require_async_state_event(True, False, None) # non-final segment
with self.assertRaises(RuntimeError):
mod.require_async_state_event(True, True, None)
class MaterializeStateIndicesTest(unittest.TestCase):
class _FakeTensor:
def __init__(self, values):
self.values = values
def numpy(self):
return np.array(self.values, dtype=np.int32)
@classmethod
def setUpClass(cls):
cls.fn = staticmethod(
_load_function(
"python/sglang/srt/disaggregation/mooncake/conn.py",
"materialize_state_indices",
)
)
def _fake_torch(self):
module = types.ModuleType("torch")
module.Tensor = self._FakeTensor
return module
def test_event_required_for_tensor(self):
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
with self.assertRaises(RuntimeError):
self.fn([self._FakeTensor([1, 2])], None)
def test_materializes_after_event_and_preserves_others(self):
np_indices = np.array([7, 8], dtype=np.int32)
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
out = self.fn([self._FakeTensor([1, 2]), None, np_indices], object())
self.assertIsInstance(out[0], np.ndarray)
np.testing.assert_array_equal(out[0], np.array([1, 2], dtype=np.int32))
self.assertIsNone(out[1])
self.assertIs(out[2], np_indices)
def test_no_tensor_needs_no_event(self):
np_indices = np.array([1], dtype=np.int32)
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
out = self.fn([None, np_indices], None)
self.assertIsNone(out[0])
self.assertIs(out[1], np_indices)
self.assertIsNone(self.fn(None, None))
class PinnedHelperBodyTest(unittest.TestCase):
def test_pinned_nonblocking_no_cpu_sync(self):
helper, calls, fake_tensor = _load_pinned_helper()
src = fake_tensor(range(16), dtype="torch.int32")
out = helper(src, page_size=4)
self.assertEqual(out.values, [0, 1, 2, 3])
self.assertEqual(out.dtype, "torch.int32")
self.assertEqual(out.shape, (4,))
self.assertEqual(len(calls["empty"]), 1)
self.assertEqual(calls["empty"][0]["device"], "cpu")
self.assertTrue(calls["empty"][0]["pin"])
self.assertEqual(len(calls["copy"]), 1)
self.assertTrue(calls["copy"][0][1])
class CudaOrderingTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
try:
import torch
if not torch.cuda.is_available():
raise unittest.SkipTest("CUDA required")
import sglang.srt.mem_cache.common # noqa: F401
except unittest.SkipTest:
raise
except Exception as exc: # noqa: BLE001
raise unittest.SkipTest(f"torch/CUDA/sglang unavailable: {exc}")
def test_pinned_copy_returns_before_kernel_and_event_protects_values(self):
import torch # noqa: F401
from sglang.srt.mem_cache.common import (
kv_to_page_indices,
kv_to_page_indices_pinned,
)
page_size = 8
src = torch.arange(4096, device="cuda", dtype=torch.int32)
# Warm up pinned allocation/CUDA context.
_ = kv_to_page_indices_pinned(src, page_size)
torch.cuda.synchronize()
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
torch.cuda._sleep(2_000_000_000)
t0 = time.monotonic()
out = kv_to_page_indices_pinned(src, page_size)
enqueue_s = time.monotonic() - t0
event = torch.cuda.Event()
event.record(stream)
t1 = time.monotonic()
event.synchronize()
wait_s = time.monotonic() - t1
# The helper returns from the enqueue without waiting for the kernel.
self.assertLess(enqueue_s, max(0.05, wait_s * 0.5))
# After the event, the pinned values match the synchronous helper.
np.testing.assert_array_equal(
out.numpy(), kv_to_page_indices(src, page_size)
)
if __name__ == "__main__":
unittest.main()