From 3b0baa345577a2f57539ece651c8b105f0d87d78 Mon Sep 17 00:00:00 2001 From: ymk Date: Fri, 6 Nov 2026 23:00:22 +0800 Subject: [PATCH] prefill: async SWA state-payload D2H (first version) Implements the v2 plan in handoff/prefill-state-payload-async-review-20261106.md. - mem_cache/common.py: add kv_to_page_indices_pinned() (pinned CPU buffer, non-blocking copy; no synchronous .cpu()). - prefill.py: gate via should_async_state_payload() (last chunk, independent supports_async_state_indices, kv_shard_size==1, CP guard); only the _swa_payload request_window-is-None hot branch uses the pinned helper. Final segment asserts async state has an indices_event. Other state builders (mamba/full/DSA) stay synchronous. - mooncake/conn.py: MooncakeKVSender advertises supports_async_state_indices; the worker materializes pinned state tensors to numpy only after indices_event.synchronize(), and fails loudly if a tensor arrives without an event. - tests: guard/event/materialize/pinned-body unit tests + CUDA ordering test (skipped unless torch+CUDA+sglang are available). No scheduler/transfer semantics change; rollback is the helper switch or reverting this commit. 49 tests pass, 1 skipped locally. --- .../srt/disaggregation/mooncake/conn.py | 37 ++- python/sglang/srt/disaggregation/prefill.py | 58 +++- python/sglang/srt/mem_cache/common.py | 16 ++ .../test_async_state_payload.py | 265 ++++++++++++++++++ 4 files changed, 373 insertions(+), 3 deletions(-) create mode 100644 test/registered/unit/disaggregation/test_async_state_payload.py diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 904529905b..50f3691107 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -357,6 +357,28 @@ def _uses_flat_entry_layout( return mla_backend or force_flat or has_layer_ids or is_single_region_swa +def materialize_state_indices(state_indices, indices_event): + """Convert pinned state tensors to numpy after the event has completed. + + ``indices_event`` must already be synchronized by the caller. A tensor + without an event is a configuration/implementation error: reading it would + race the scheduler's non-blocking D2H copy. + """ + if not state_indices: + return state_indices + import torch + + if not any(isinstance(t, torch.Tensor) for t in state_indices): + return state_indices + if indices_event is None: + raise RuntimeError( + "async state indices require an indices_event before materialization" + ) + return [ + t.numpy() if isinstance(t, torch.Tensor) else t for t in state_indices + ] + + class MooncakeKVManager(StagingManagerMixin, CommonKVManager): sender_trace_module = "mooncake" @@ -2871,11 +2893,18 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): kv_chunk.wait_event = None _ind_t0 = time.monotonic_ns() if _obs else 0 _ind_wall0 = time.time_ns() if _obs else 0 - if kv_chunk.indices_event is not None: - kv_chunk.indices_event.synchronize() + indices_event = kv_chunk.indices_event + if indices_event is not None: + indices_event.synchronize() kv_chunk.indices_event = None if isinstance(kv_chunk.prefill_kv_indices, torch.Tensor): kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices.numpy() + # First async-state version: the pinned SWA state tensor is + # materialized only after the event above. Raises if a tensor + # arrives without one (would race the scheduler's D2H). + kv_chunk.state_indices = materialize_state_indices( + kv_chunk.state_indices, indices_event + ) if _ind_t0: # synchronize + host indices availability, per design ยง5.2. _indices_wait_ns += time.monotonic_ns() - _ind_t0 @@ -3757,6 +3786,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): class MooncakeKVSender(CommonKVSender): supports_host_tensor_indices = True + # First async state-payload version (SWA-only): the scheduler copies the + # SWA state indices to a pinned buffer and the worker materializes them + # after kv_chunk.indices_event. + supports_async_state_indices = True @mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND) def send( diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index be47d02e7d..be9ada36bc 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -77,6 +77,7 @@ from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome from sglang.srt.mem_cache.common import ( checkpoint_kv_cache, kv_to_page_indices, + kv_to_page_indices_pinned, kv_to_page_num, release_kv_cache, suffix_page_indices, @@ -564,6 +565,40 @@ class PrefillBootstrapQueue: self.kv_manager.register_buffer_to_engine() +def should_async_state_payload( + *, + last_chunk: bool, + supports_async_state_indices: bool, + kv_shard_size: int, + enable_all_cp_ranks_for_transfer: bool, + enable_dsa_cache_layer_split: bool, +) -> bool: + """Gate for the first async state-payload version (SWA-only). + + ``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old + synchronous path: their indices are filtered before the transfer event, + and deferring that to the worker would widen the pre-event shard-filter + risk tracked by #45492. + """ + if not last_chunk or not supports_async_state_indices: + return False + if kv_shard_size != 1: + return False + if enable_all_cp_ranks_for_transfer and not enable_dsa_cache_layer_split: + return False + return True + + +def require_async_state_event( + async_state: bool, segment_is_last: bool, indices_event +) -> None: + """Fail loudly if an async state payload has no event to protect its copy.""" + if async_state and segment_is_last and indices_event is None: + raise RuntimeError( + "async state payload requires an indices_event for the final segment" + ) + + class SchedulerDisaggregationPrefillMixin: """ Mixin for Scheduler to handle disaggregation prefill @@ -1782,6 +1817,21 @@ class SchedulerDisaggregationPrefillMixin: ) return + # First async state-payload version: SWA only, with the shard/CP guards + # in should_async_state_payload(). Other builders stay synchronous. + sender = req.disagg_kv_sender + async_state = should_async_state_payload( + last_chunk=last_chunk, + supports_async_state_indices=getattr( + sender, "supports_async_state_indices", False + ), + kv_shard_size=getattr(sender.kv_mgr, "kv_shard_size", 1), + enable_all_cp_ranks_for_transfer=getattr( + sender.kv_mgr, "enable_all_cp_ranks_for_transfer", False + ), + enable_dsa_cache_layer_split=get_parallel().enable_dsa_cache_layer_split, + ) + state_indices: Optional[List] = None _state_tok = ( batch_telemetry.detail_begin("kv_state_payload." + send_kind) @@ -1833,6 +1883,10 @@ class SchedulerDisaggregationPrefillMixin: window_kv_indices_full ) ) + if async_state: + return kv_to_page_indices_pinned( + window_kv_indices_swa, page_size + ) return kv_to_page_indices(window_kv_indices_swa, page_size) def _full_kv_pages_payload(): @@ -1973,7 +2027,6 @@ class SchedulerDisaggregationPrefillMixin: raw_kv_indices ) ) - sender = req.disagg_kv_sender indices_event = None if getattr(sender, "supports_host_tensor_indices", False) and not ( sender.kv_mgr.enable_all_cp_ranks_for_transfer @@ -2001,6 +2054,9 @@ class SchedulerDisaggregationPrefillMixin: segment_is_last = last_chunk and is_final_segment if not sender.should_send_kv_chunk(len(page_indices), segment_is_last): continue + require_async_state_event( + async_state, segment_is_last, indices_event + ) send_state_indices = state_indices if segment_is_last else None _send_t0 = ( time.monotonic_ns() diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 7122c3fab6..e3e1937208 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -47,6 +47,22 @@ def kv_to_page_indices(kv_indices: torch.Tensor, page_size: int) -> np.ndarray: return (kv_indices[::page_size] // page_size).cpu().numpy() +def kv_to_page_indices_pinned( + kv_indices: torch.Tensor, page_size: int +) -> torch.Tensor: + """Same values/dtype as :func:`kv_to_page_indices`, in a pinned CPU tensor. + + The device-to-host copy is non-blocking on the current stream. The caller + must record a CUDA event on that stream after all such copies and hand it + to the transfer worker, which materializes the tensor only after the event + (see ``materialize_state_indices``). + """ + src = kv_indices[::page_size] // page_size + out = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True) + out.copy_(src, non_blocking=True) + return out + + def suffix_page_indices( req_to_token: torch.Tensor, req_pool_idx: int, diff --git a/test/registered/unit/disaggregation/test_async_state_payload.py b/test/registered/unit/disaggregation/test_async_state_payload.py new file mode 100644 index 0000000000..62378da110 --- /dev/null +++ b/test/registered/unit/disaggregation/test_async_state_payload.py @@ -0,0 +1,265 @@ +"""Unit tests for the SWA-only async state-payload first version. + +Covers the v2 checklist that can run without a live server: + + * should_async_state_payload guards (last chunk / capability / shard / CP); + * require_async_state_event (empty last chunk / multi segment invariant); + * worker materialize_state_indices (event ordering, no-event failure, + non-tensor and None entries preserved); + * kv_to_page_indices_pinned body: pinned CPU buffer, non-blocking copy, + same dtype/shape, no synchronous .cpu(); + * a CUDA ordering test (skipped unless torch + CUDA + sglang are importable) + proving the helper returns before a preceding long kernel finishes and + the event-synchronized values are correct. + +The functions are extracted from the real source with ``ast`` so the test does +not need to import the heavy SGLang modules. Full DSV4 component / abort / +multi-segment integration cases run in the server harness on b300. +""" +import ast +import pathlib +import sys +import time +import types +import unittest +from unittest import mock + +import numpy as np + + +def _repo_root(): + return pathlib.Path(__file__).resolve().parents[4] + + +def _load_function(rel_path, name, extra_ns=None): + path = _repo_root() / rel_path + tree = ast.parse(path.read_text(encoding="utf-8")) + fn = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + namespace = dict(extra_ns or {}) + module = ast.Module(body=[fn], type_ignores=[]) + exec(compile(module, str(path), "exec"), namespace) + return namespace[name] + + +def _load_pinned_helper(): + """Execute the helper body from the real source with a fake torch.""" + path = _repo_root() / "python/sglang/srt/mem_cache/common.py" + tree = ast.parse(path.read_text(encoding="utf-8")) + fn = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "kv_to_page_indices_pinned" + ) + calls = {"empty": [], "copy": []} + + class _FakeTensor: + def __init__(self, values, dtype="torch.int32", shape=None): + self.values = list(values) + self.dtype = dtype + self.shape = (len(self.values),) if shape is None else shape + + def __getitem__(self, sl): + return _FakeTensor(self.values[sl], self.dtype) + + def __floordiv__(self, value): + return _FakeTensor([v // value for v in self.values], self.dtype) + + def cpu(self): + raise AssertionError("kv_to_page_indices_pinned must not call .cpu()") + + def numpy(self): + raise AssertionError( + "kv_to_page_indices_pinned must not call .numpy()" + ) + + def copy_(self, src, non_blocking=False): + calls["copy"].append((src, non_blocking)) + self.values = list(src.values) + + class _FakeTorch: + Tensor = _FakeTensor + + @staticmethod + def empty(shape, dtype, device, pin_memory): + calls["empty"].append( + {"shape": shape, "dtype": dtype, "device": device, "pin": pin_memory} + ) + return _FakeTensor([], dtype=dtype, shape=shape) + + namespace = {"torch": _FakeTorch} + module = ast.Module(body=[fn], type_ignores=[]) + exec(compile(module, str(path), "exec"), namespace) + return namespace["kv_to_page_indices_pinned"], calls, _FakeTensor + + +class AsyncStateGateTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mod = types.SimpleNamespace() + base = "python/sglang/srt/disaggregation/prefill.py" + cls.mod.should_async_state_payload = _load_function( + base, "should_async_state_payload" + ) + cls.mod.require_async_state_event = _load_function( + base, "require_async_state_event" + ) + + def test_guards(self): + mod = self.mod + base = dict( + last_chunk=True, + supports_async_state_indices=True, + kv_shard_size=1, + enable_all_cp_ranks_for_transfer=False, + enable_dsa_cache_layer_split=False, + ) + self.assertTrue(mod.should_async_state_payload(**base)) + self.assertFalse( + mod.should_async_state_payload(**{**base, "last_chunk": False}) + ) + self.assertFalse( + mod.should_async_state_payload( + **{**base, "supports_async_state_indices": False} + ) + ) + self.assertFalse( + mod.should_async_state_payload(**{**base, "kv_shard_size": 2}) + ) + self.assertFalse( + mod.should_async_state_payload( + **{**base, "enable_all_cp_ranks_for_transfer": True} + ) + ) + self.assertTrue( + mod.should_async_state_payload( + **{ + **base, + "enable_all_cp_ranks_for_transfer": True, + "enable_dsa_cache_layer_split": True, + } + ) + ) + + def test_final_event_required(self): + mod = self.mod + mod.require_async_state_event(True, True, object()) # no raise + mod.require_async_state_event(False, True, None) # sync path + mod.require_async_state_event(True, False, None) # non-final segment + with self.assertRaises(RuntimeError): + mod.require_async_state_event(True, True, None) + + +class MaterializeStateIndicesTest(unittest.TestCase): + class _FakeTensor: + def __init__(self, values): + self.values = values + + def numpy(self): + return np.array(self.values, dtype=np.int32) + + @classmethod + def setUpClass(cls): + cls.fn = staticmethod( + _load_function( + "python/sglang/srt/disaggregation/mooncake/conn.py", + "materialize_state_indices", + ) + ) + + def _fake_torch(self): + module = types.ModuleType("torch") + module.Tensor = self._FakeTensor + return module + + def test_event_required_for_tensor(self): + with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}): + with self.assertRaises(RuntimeError): + self.fn([self._FakeTensor([1, 2])], None) + + def test_materializes_after_event_and_preserves_others(self): + np_indices = np.array([7, 8], dtype=np.int32) + with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}): + out = self.fn([self._FakeTensor([1, 2]), None, np_indices], object()) + self.assertIsInstance(out[0], np.ndarray) + np.testing.assert_array_equal(out[0], np.array([1, 2], dtype=np.int32)) + self.assertIsNone(out[1]) + self.assertIs(out[2], np_indices) + + def test_no_tensor_needs_no_event(self): + np_indices = np.array([1], dtype=np.int32) + with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}): + out = self.fn([None, np_indices], None) + self.assertIsNone(out[0]) + self.assertIs(out[1], np_indices) + self.assertIsNone(self.fn(None, None)) + + +class PinnedHelperBodyTest(unittest.TestCase): + def test_pinned_nonblocking_no_cpu_sync(self): + helper, calls, fake_tensor = _load_pinned_helper() + src = fake_tensor(range(16), dtype="torch.int32") + out = helper(src, page_size=4) + self.assertEqual(out.values, [0, 1, 2, 3]) + self.assertEqual(out.dtype, "torch.int32") + self.assertEqual(out.shape, (4,)) + self.assertEqual(len(calls["empty"]), 1) + self.assertEqual(calls["empty"][0]["device"], "cpu") + self.assertTrue(calls["empty"][0]["pin"]) + self.assertEqual(len(calls["copy"]), 1) + self.assertTrue(calls["copy"][0][1]) + + +class CudaOrderingTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + import torch + + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA required") + import sglang.srt.mem_cache.common # noqa: F401 + except unittest.SkipTest: + raise + except Exception as exc: # noqa: BLE001 + raise unittest.SkipTest(f"torch/CUDA/sglang unavailable: {exc}") + + def test_pinned_copy_returns_before_kernel_and_event_protects_values(self): + import torch # noqa: F401 + from sglang.srt.mem_cache.common import ( + kv_to_page_indices, + kv_to_page_indices_pinned, + ) + + page_size = 8 + src = torch.arange(4096, device="cuda", dtype=torch.int32) + # Warm up pinned allocation/CUDA context. + _ = kv_to_page_indices_pinned(src, page_size) + torch.cuda.synchronize() + + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + torch.cuda._sleep(2_000_000_000) + t0 = time.monotonic() + out = kv_to_page_indices_pinned(src, page_size) + enqueue_s = time.monotonic() - t0 + event = torch.cuda.Event() + event.record(stream) + + t1 = time.monotonic() + event.synchronize() + wait_s = time.monotonic() - t1 + + # The helper returns from the enqueue without waiting for the kernel. + self.assertLess(enqueue_s, max(0.05, wait_s * 0.5)) + # After the event, the pinned values match the synchronous helper. + np.testing.assert_array_equal( + out.numpy(), kv_to_page_indices(src, page_size) + ) + + +if __name__ == "__main__": + unittest.main()