prefill: async SWA state-payload D2H (first version)
Implements the v2 plan in handoff/prefill-state-payload-async-review-20261106.md. - mem_cache/common.py: add kv_to_page_indices_pinned() (pinned CPU buffer, non-blocking copy; no synchronous .cpu()). - prefill.py: gate via should_async_state_payload() (last chunk, independent supports_async_state_indices, kv_shard_size==1, CP guard); only the _swa_payload request_window-is-None hot branch uses the pinned helper. Final segment asserts async state has an indices_event. Other state builders (mamba/full/DSA) stay synchronous. - mooncake/conn.py: MooncakeKVSender advertises supports_async_state_indices; the worker materializes pinned state tensors to numpy only after indices_event.synchronize(), and fails loudly if a tensor arrives without an event. - tests: guard/event/materialize/pinned-body unit tests + CUDA ordering test (skipped unless torch+CUDA+sglang are available). No scheduler/transfer semantics change; rollback is the helper switch or reverting this commit. 49 tests pass, 1 skipped locally.
This commit is contained in:
@@ -357,6 +357,28 @@ def _uses_flat_entry_layout(
|
||||
return mla_backend or force_flat or has_layer_ids or is_single_region_swa
|
||||
|
||||
|
||||
def materialize_state_indices(state_indices, indices_event):
|
||||
"""Convert pinned state tensors to numpy after the event has completed.
|
||||
|
||||
``indices_event`` must already be synchronized by the caller. A tensor
|
||||
without an event is a configuration/implementation error: reading it would
|
||||
race the scheduler's non-blocking D2H copy.
|
||||
"""
|
||||
if not state_indices:
|
||||
return state_indices
|
||||
import torch
|
||||
|
||||
if not any(isinstance(t, torch.Tensor) for t in state_indices):
|
||||
return state_indices
|
||||
if indices_event is None:
|
||||
raise RuntimeError(
|
||||
"async state indices require an indices_event before materialization"
|
||||
)
|
||||
return [
|
||||
t.numpy() if isinstance(t, torch.Tensor) else t for t in state_indices
|
||||
]
|
||||
|
||||
|
||||
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
sender_trace_module = "mooncake"
|
||||
|
||||
@@ -2871,11 +2893,18 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
kv_chunk.wait_event = None
|
||||
_ind_t0 = time.monotonic_ns() if _obs else 0
|
||||
_ind_wall0 = time.time_ns() if _obs else 0
|
||||
if kv_chunk.indices_event is not None:
|
||||
kv_chunk.indices_event.synchronize()
|
||||
indices_event = kv_chunk.indices_event
|
||||
if indices_event is not None:
|
||||
indices_event.synchronize()
|
||||
kv_chunk.indices_event = None
|
||||
if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor):
|
||||
kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy()
|
||||
# First async-state version: the pinned SWA state tensor is
|
||||
# materialized only after the event above. Raises if a tensor
|
||||
# arrives without one (would race the scheduler's D2H).
|
||||
kv_chunk.state_indices = materialize_state_indices(
|
||||
kv_chunk.state_indices, indices_event
|
||||
)
|
||||
if _ind_t0:
|
||||
# synchronize + host indices availability, per design §5.2.
|
||||
_indices_wait_ns += time.monotonic_ns() - _ind_t0
|
||||
@@ -3757,6 +3786,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
class MooncakeKVSender(CommonKVSender):
|
||||
supports_host_tensor_indices = True
|
||||
# First async state-payload version (SWA-only): the scheduler copies the
|
||||
# SWA state indices to a pinned buffer and the worker materializes them
|
||||
# after kv_chunk.indices_event.
|
||||
supports_async_state_indices = True
|
||||
|
||||
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
|
||||
def send(
|
||||
|
||||
@@ -77,6 +77,7 @@ from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
||||
from sglang.srt.mem_cache.common import (
|
||||
checkpoint_kv_cache,
|
||||
kv_to_page_indices,
|
||||
kv_to_page_indices_pinned,
|
||||
kv_to_page_num,
|
||||
release_kv_cache,
|
||||
suffix_page_indices,
|
||||
@@ -564,6 +565,40 @@ class PrefillBootstrapQueue:
|
||||
self.kv_manager.register_buffer_to_engine()
|
||||
|
||||
|
||||
def should_async_state_payload(
|
||||
*,
|
||||
last_chunk: bool,
|
||||
supports_async_state_indices: bool,
|
||||
kv_shard_size: int,
|
||||
enable_all_cp_ranks_for_transfer: bool,
|
||||
enable_dsa_cache_layer_split: bool,
|
||||
) -> bool:
|
||||
"""Gate for the first async state-payload version (SWA-only).
|
||||
|
||||
``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old
|
||||
synchronous path: their indices are filtered before the transfer event,
|
||||
and deferring that to the worker would widen the pre-event shard-filter
|
||||
risk tracked by #45492.
|
||||
"""
|
||||
if not last_chunk or not supports_async_state_indices:
|
||||
return False
|
||||
if kv_shard_size != 1:
|
||||
return False
|
||||
if enable_all_cp_ranks_for_transfer and not enable_dsa_cache_layer_split:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def require_async_state_event(
|
||||
async_state: bool, segment_is_last: bool, indices_event
|
||||
) -> None:
|
||||
"""Fail loudly if an async state payload has no event to protect its copy."""
|
||||
if async_state and segment_is_last and indices_event is None:
|
||||
raise RuntimeError(
|
||||
"async state payload requires an indices_event for the final segment"
|
||||
)
|
||||
|
||||
|
||||
class SchedulerDisaggregationPrefillMixin:
|
||||
"""
|
||||
Mixin for Scheduler to handle disaggregation prefill
|
||||
@@ -1782,6 +1817,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
return
|
||||
|
||||
# First async state-payload version: SWA only, with the shard/CP guards
|
||||
# in should_async_state_payload(). Other builders stay synchronous.
|
||||
sender = req.disagg_kv_sender
|
||||
async_state = should_async_state_payload(
|
||||
last_chunk=last_chunk,
|
||||
supports_async_state_indices=getattr(
|
||||
sender, "supports_async_state_indices", False
|
||||
),
|
||||
kv_shard_size=getattr(sender.kv_mgr, "kv_shard_size", 1),
|
||||
enable_all_cp_ranks_for_transfer=getattr(
|
||||
sender.kv_mgr, "enable_all_cp_ranks_for_transfer", False
|
||||
),
|
||||
enable_dsa_cache_layer_split=get_parallel().enable_dsa_cache_layer_split,
|
||||
)
|
||||
|
||||
state_indices: Optional[List] = None
|
||||
_state_tok = (
|
||||
batch_telemetry.detail_begin("kv_state_payload." + send_kind)
|
||||
@@ -1833,6 +1883,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
window_kv_indices_full
|
||||
)
|
||||
)
|
||||
if async_state:
|
||||
return kv_to_page_indices_pinned(
|
||||
window_kv_indices_swa, page_size
|
||||
)
|
||||
return kv_to_page_indices(window_kv_indices_swa, page_size)
|
||||
|
||||
def _full_kv_pages_payload():
|
||||
@@ -1973,7 +2027,6 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
raw_kv_indices
|
||||
)
|
||||
)
|
||||
sender = req.disagg_kv_sender
|
||||
indices_event = None
|
||||
if getattr(sender, "supports_host_tensor_indices", False) and not (
|
||||
sender.kv_mgr.enable_all_cp_ranks_for_transfer
|
||||
@@ -2001,6 +2054,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
segment_is_last = last_chunk and is_final_segment
|
||||
if not sender.should_send_kv_chunk(len(page_indices), segment_is_last):
|
||||
continue
|
||||
require_async_state_event(
|
||||
async_state, segment_is_last, indices_event
|
||||
)
|
||||
send_state_indices = state_indices if segment_is_last else None
|
||||
_send_t0 = (
|
||||
time.monotonic_ns()
|
||||
|
||||
@@ -47,6 +47,22 @@ def kv_to_page_indices(kv_indices: torch.Tensor, page_size: int) -> np.ndarray:
|
||||
return (kv_indices[::page_size] // page_size).cpu().numpy()
|
||||
|
||||
|
||||
def kv_to_page_indices_pinned(
|
||||
kv_indices: torch.Tensor, page_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Same values/dtype as :func:`kv_to_page_indices`, in a pinned CPU tensor.
|
||||
|
||||
The device-to-host copy is non-blocking on the current stream. The caller
|
||||
must record a CUDA event on that stream after all such copies and hand it
|
||||
to the transfer worker, which materializes the tensor only after the event
|
||||
(see ``materialize_state_indices``).
|
||||
"""
|
||||
src = kv_indices[::page_size] // page_size
|
||||
out = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
|
||||
out.copy_(src, non_blocking=True)
|
||||
return out
|
||||
|
||||
|
||||
def suffix_page_indices(
|
||||
req_to_token: torch.Tensor,
|
||||
req_pool_idx: int,
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Unit tests for the SWA-only async state-payload first version.
|
||||
|
||||
Covers the v2 checklist that can run without a live server:
|
||||
|
||||
* should_async_state_payload guards (last chunk / capability / shard / CP);
|
||||
* require_async_state_event (empty last chunk / multi segment invariant);
|
||||
* worker materialize_state_indices (event ordering, no-event failure,
|
||||
non-tensor and None entries preserved);
|
||||
* kv_to_page_indices_pinned body: pinned CPU buffer, non-blocking copy,
|
||||
same dtype/shape, no synchronous .cpu();
|
||||
* a CUDA ordering test (skipped unless torch + CUDA + sglang are importable)
|
||||
proving the helper returns before a preceding long kernel finishes and
|
||||
the event-synchronized values are correct.
|
||||
|
||||
The functions are extracted from the real source with ``ast`` so the test does
|
||||
not need to import the heavy SGLang modules. Full DSV4 component / abort /
|
||||
multi-segment integration cases run in the server harness on b300.
|
||||
"""
|
||||
import ast
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _repo_root():
|
||||
return pathlib.Path(__file__).resolve().parents[4]
|
||||
|
||||
|
||||
def _load_function(rel_path, name, extra_ns=None):
|
||||
path = _repo_root() / rel_path
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
fn = next(
|
||||
node
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.FunctionDef) and node.name == name
|
||||
)
|
||||
namespace = dict(extra_ns or {})
|
||||
module = ast.Module(body=[fn], type_ignores=[])
|
||||
exec(compile(module, str(path), "exec"), namespace)
|
||||
return namespace[name]
|
||||
|
||||
|
||||
def _load_pinned_helper():
|
||||
"""Execute the helper body from the real source with a fake torch."""
|
||||
path = _repo_root() / "python/sglang/srt/mem_cache/common.py"
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
fn = next(
|
||||
node
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.FunctionDef) and node.name == "kv_to_page_indices_pinned"
|
||||
)
|
||||
calls = {"empty": [], "copy": []}
|
||||
|
||||
class _FakeTensor:
|
||||
def __init__(self, values, dtype="torch.int32", shape=None):
|
||||
self.values = list(values)
|
||||
self.dtype = dtype
|
||||
self.shape = (len(self.values),) if shape is None else shape
|
||||
|
||||
def __getitem__(self, sl):
|
||||
return _FakeTensor(self.values[sl], self.dtype)
|
||||
|
||||
def __floordiv__(self, value):
|
||||
return _FakeTensor([v // value for v in self.values], self.dtype)
|
||||
|
||||
def cpu(self):
|
||||
raise AssertionError("kv_to_page_indices_pinned must not call .cpu()")
|
||||
|
||||
def numpy(self):
|
||||
raise AssertionError(
|
||||
"kv_to_page_indices_pinned must not call .numpy()"
|
||||
)
|
||||
|
||||
def copy_(self, src, non_blocking=False):
|
||||
calls["copy"].append((src, non_blocking))
|
||||
self.values = list(src.values)
|
||||
|
||||
class _FakeTorch:
|
||||
Tensor = _FakeTensor
|
||||
|
||||
@staticmethod
|
||||
def empty(shape, dtype, device, pin_memory):
|
||||
calls["empty"].append(
|
||||
{"shape": shape, "dtype": dtype, "device": device, "pin": pin_memory}
|
||||
)
|
||||
return _FakeTensor([], dtype=dtype, shape=shape)
|
||||
|
||||
namespace = {"torch": _FakeTorch}
|
||||
module = ast.Module(body=[fn], type_ignores=[])
|
||||
exec(compile(module, str(path), "exec"), namespace)
|
||||
return namespace["kv_to_page_indices_pinned"], calls, _FakeTensor
|
||||
|
||||
|
||||
class AsyncStateGateTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mod = types.SimpleNamespace()
|
||||
base = "python/sglang/srt/disaggregation/prefill.py"
|
||||
cls.mod.should_async_state_payload = _load_function(
|
||||
base, "should_async_state_payload"
|
||||
)
|
||||
cls.mod.require_async_state_event = _load_function(
|
||||
base, "require_async_state_event"
|
||||
)
|
||||
|
||||
def test_guards(self):
|
||||
mod = self.mod
|
||||
base = dict(
|
||||
last_chunk=True,
|
||||
supports_async_state_indices=True,
|
||||
kv_shard_size=1,
|
||||
enable_all_cp_ranks_for_transfer=False,
|
||||
enable_dsa_cache_layer_split=False,
|
||||
)
|
||||
self.assertTrue(mod.should_async_state_payload(**base))
|
||||
self.assertFalse(
|
||||
mod.should_async_state_payload(**{**base, "last_chunk": False})
|
||||
)
|
||||
self.assertFalse(
|
||||
mod.should_async_state_payload(
|
||||
**{**base, "supports_async_state_indices": False}
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
mod.should_async_state_payload(**{**base, "kv_shard_size": 2})
|
||||
)
|
||||
self.assertFalse(
|
||||
mod.should_async_state_payload(
|
||||
**{**base, "enable_all_cp_ranks_for_transfer": True}
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
mod.should_async_state_payload(
|
||||
**{
|
||||
**base,
|
||||
"enable_all_cp_ranks_for_transfer": True,
|
||||
"enable_dsa_cache_layer_split": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def test_final_event_required(self):
|
||||
mod = self.mod
|
||||
mod.require_async_state_event(True, True, object()) # no raise
|
||||
mod.require_async_state_event(False, True, None) # sync path
|
||||
mod.require_async_state_event(True, False, None) # non-final segment
|
||||
with self.assertRaises(RuntimeError):
|
||||
mod.require_async_state_event(True, True, None)
|
||||
|
||||
|
||||
class MaterializeStateIndicesTest(unittest.TestCase):
|
||||
class _FakeTensor:
|
||||
def __init__(self, values):
|
||||
self.values = values
|
||||
|
||||
def numpy(self):
|
||||
return np.array(self.values, dtype=np.int32)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fn = staticmethod(
|
||||
_load_function(
|
||||
"python/sglang/srt/disaggregation/mooncake/conn.py",
|
||||
"materialize_state_indices",
|
||||
)
|
||||
)
|
||||
|
||||
def _fake_torch(self):
|
||||
module = types.ModuleType("torch")
|
||||
module.Tensor = self._FakeTensor
|
||||
return module
|
||||
|
||||
def test_event_required_for_tensor(self):
|
||||
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.fn([self._FakeTensor([1, 2])], None)
|
||||
|
||||
def test_materializes_after_event_and_preserves_others(self):
|
||||
np_indices = np.array([7, 8], dtype=np.int32)
|
||||
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
|
||||
out = self.fn([self._FakeTensor([1, 2]), None, np_indices], object())
|
||||
self.assertIsInstance(out[0], np.ndarray)
|
||||
np.testing.assert_array_equal(out[0], np.array([1, 2], dtype=np.int32))
|
||||
self.assertIsNone(out[1])
|
||||
self.assertIs(out[2], np_indices)
|
||||
|
||||
def test_no_tensor_needs_no_event(self):
|
||||
np_indices = np.array([1], dtype=np.int32)
|
||||
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
|
||||
out = self.fn([None, np_indices], None)
|
||||
self.assertIsNone(out[0])
|
||||
self.assertIs(out[1], np_indices)
|
||||
self.assertIsNone(self.fn(None, None))
|
||||
|
||||
|
||||
class PinnedHelperBodyTest(unittest.TestCase):
|
||||
def test_pinned_nonblocking_no_cpu_sync(self):
|
||||
helper, calls, fake_tensor = _load_pinned_helper()
|
||||
src = fake_tensor(range(16), dtype="torch.int32")
|
||||
out = helper(src, page_size=4)
|
||||
self.assertEqual(out.values, [0, 1, 2, 3])
|
||||
self.assertEqual(out.dtype, "torch.int32")
|
||||
self.assertEqual(out.shape, (4,))
|
||||
self.assertEqual(len(calls["empty"]), 1)
|
||||
self.assertEqual(calls["empty"][0]["device"], "cpu")
|
||||
self.assertTrue(calls["empty"][0]["pin"])
|
||||
self.assertEqual(len(calls["copy"]), 1)
|
||||
self.assertTrue(calls["copy"][0][1])
|
||||
|
||||
|
||||
class CudaOrderingTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
try:
|
||||
import torch
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise unittest.SkipTest("CUDA required")
|
||||
import sglang.srt.mem_cache.common # noqa: F401
|
||||
except unittest.SkipTest:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise unittest.SkipTest(f"torch/CUDA/sglang unavailable: {exc}")
|
||||
|
||||
def test_pinned_copy_returns_before_kernel_and_event_protects_values(self):
|
||||
import torch # noqa: F401
|
||||
from sglang.srt.mem_cache.common import (
|
||||
kv_to_page_indices,
|
||||
kv_to_page_indices_pinned,
|
||||
)
|
||||
|
||||
page_size = 8
|
||||
src = torch.arange(4096, device="cuda", dtype=torch.int32)
|
||||
# Warm up pinned allocation/CUDA context.
|
||||
_ = kv_to_page_indices_pinned(src, page_size)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
torch.cuda._sleep(2_000_000_000)
|
||||
t0 = time.monotonic()
|
||||
out = kv_to_page_indices_pinned(src, page_size)
|
||||
enqueue_s = time.monotonic() - t0
|
||||
event = torch.cuda.Event()
|
||||
event.record(stream)
|
||||
|
||||
t1 = time.monotonic()
|
||||
event.synchronize()
|
||||
wait_s = time.monotonic() - t1
|
||||
|
||||
# The helper returns from the enqueue without waiting for the kernel.
|
||||
self.assertLess(enqueue_s, max(0.05, wait_s * 0.5))
|
||||
# After the event, the pinned values match the synchronous helper.
|
||||
np.testing.assert_array_equal(
|
||||
out.numpy(), kv_to_page_indices(src, page_size)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user