From 7b3f34cbc03a93e448854024314fea49727ba396 Mon Sep 17 00:00:00 2001 From: ymk Date: Sat, 7 Nov 2026 00:22:57 +0800 Subject: [PATCH] test: cover async SWA state-payload call chain; clarify CP gate Review follow-up for the v2 async state payload: - P2: new tests use CustomTestCase (with a local fallback when sglang is not importable), matching the disaggregation/Mooncake test convention. - P3: CP gate docstring now says all-CP-rank transfer *without DSA cache layer split* keeps the synchronous path. - Add real call-chain coverage: * _send_kv_chunk -> sender.send -> add_transfer_request -> transfer_worker -> materialize_state_indices -> maybe_send_extra, with DSV4's four components (DSV4_REQUEST_STATE, REQUEST_ROW, SWA, SWA); * the worker materializes pinned tensors only after indices_event and fails loudly when a tensor has no event; * a second materialization without an event (staging retry) is a no-op; * real CUDA ordering: the pinned helper returns before a long kernel and the event-protected values match kv_to_page_indices. - Verified on b300 (P container, overlay source): 9 passed in test_async_state_payload.py, 2 passed in the new TestMooncakeAsyncStateMaterialization class; the old batching suite has 11 pre-existing failures that also fail on the image source. The test-only batching suite requires the detail envs to be off when run in the live pod (the inherited SGLANG_TRACE_MOONCAKE_DETAIL=1 exposes an existing test fixture omission); noted, not changed here. --- python/sglang/srt/disaggregation/prefill.py | 8 +- .../test_async_state_payload.py | 247 +++++++++++++++++- .../test_mooncake_transfer_batching.py | 103 ++++++++ 3 files changed, 350 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index be9ada36bc..ddafd46e18 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -575,10 +575,10 @@ def should_async_state_payload( ) -> bool: """Gate for the first async state-payload version (SWA-only). - ``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old - synchronous path: their indices are filtered before the transfer event, - and deferring that to the worker would widen the pre-event shard-filter - risk tracked by #45492. + ``kv_shard_size > 1`` and all-CP-rank transfer **without DSA cache layer + split** keep the old synchronous path: their indices are filtered before + the transfer event, and deferring that to the worker would widen the + pre-event shard-filter risk tracked by #45492. """ if not last_chunk or not supports_async_state_indices: return False diff --git a/test/registered/unit/disaggregation/test_async_state_payload.py b/test/registered/unit/disaggregation/test_async_state_payload.py index 62378da110..053d04f5c8 100644 --- a/test/registered/unit/disaggregation/test_async_state_payload.py +++ b/test/registered/unit/disaggregation/test_async_state_payload.py @@ -26,6 +26,11 @@ from unittest import mock import numpy as np +try: + from sglang.test.test_utils import CustomTestCase +except Exception: # local helper-only runs without sglang installed + CustomTestCase = unittest.TestCase + def _repo_root(): return pathlib.Path(__file__).resolve().parents[4] @@ -96,7 +101,7 @@ def _load_pinned_helper(): return namespace["kv_to_page_indices_pinned"], calls, _FakeTensor -class AsyncStateGateTest(unittest.TestCase): +class AsyncStateGateTest(CustomTestCase): @classmethod def setUpClass(cls): cls.mod = types.SimpleNamespace() @@ -153,7 +158,7 @@ class AsyncStateGateTest(unittest.TestCase): mod.require_async_state_event(True, True, None) -class MaterializeStateIndicesTest(unittest.TestCase): +class MaterializeStateIndicesTest(CustomTestCase): class _FakeTensor: def __init__(self, values): self.values = values @@ -189,6 +194,14 @@ class MaterializeStateIndicesTest(unittest.TestCase): self.assertIsNone(out[1]) self.assertIs(out[2], np_indices) + def test_second_materialization_without_event_is_safe(self): + """Staging retry: first pass materializes; second pass has no event.""" + with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}): + first = self.fn([self._FakeTensor([1, 2])], object()) + second = self.fn(first, None) + self.assertIs(second, first) + np.testing.assert_array_equal(first[0], np.array([1, 2], dtype=np.int32)) + def test_no_tensor_needs_no_event(self): np_indices = np.array([1], dtype=np.int32) with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}): @@ -198,7 +211,7 @@ class MaterializeStateIndicesTest(unittest.TestCase): self.assertIsNone(self.fn(None, None)) -class PinnedHelperBodyTest(unittest.TestCase): +class PinnedHelperBodyTest(CustomTestCase): def test_pinned_nonblocking_no_cpu_sync(self): helper, calls, fake_tensor = _load_pinned_helper() src = fake_tensor(range(16), dtype="torch.int32") @@ -213,7 +226,233 @@ class PinnedHelperBodyTest(unittest.TestCase): self.assertTrue(calls["copy"][0][1]) -class CudaOrderingTest(unittest.TestCase): +class CudaSendChainTest(CustomTestCase): + """Real _send_kv_chunk -> sender.send -> transfer_worker call chain.""" + + @classmethod + def setUpClass(cls): + try: + import torch + + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA required") + from sglang.srt.disaggregation.prefill import ( # noqa: F401 + SchedulerDisaggregationPrefillMixin, + ) + except unittest.SkipTest: + raise + except Exception as exc: # noqa: BLE001 + raise unittest.SkipTest(f"torch/CUDA/sglang unavailable: {exc}") + + def test_send_chunk_queues_async_state_and_worker_materializes(self): + import types + + import torch + from sglang.srt.disaggregation.base.conn import KVPoll, StateType + from sglang.srt.disaggregation.common.utils import TransferKVChunk + from sglang.srt.disaggregation.mooncake.conn import ( + MooncakeKVManager, + MooncakeKVSender, + ) + from sglang.srt.disaggregation.prefill import ( + SchedulerDisaggregationPrefillMixin, + ) + from sglang.srt.observability import batch_telemetry # noqa: F401 + + seq_len = 32 + page_size = 8 + req_to_token = torch.arange(0, seq_len, device="cuda", dtype=torch.int32).view( + 1, seq_len + ) + allocator = types.SimpleNamespace( + page_size=page_size, + get_kvcache=lambda: types.SimpleNamespace( + request_window=None, + sliding_window=seq_len, + request_state_transfer_indices=lambda req_idx, length: np.array( + [req_idx], dtype=np.int32 + ), + ), + translate_swa_indices_for_transfer=lambda x: x, + translate_kv_indices_for_transfer=lambda x: x, + ) + req_pool = types.SimpleNamespace(req_to_token=req_to_token) + captured = [] + + def add_transfer_request( + room, + kv_indices, + index_slice, + is_last_chunk, + aux_index=None, + state_indices=None, + num_kv_tokens=None, + wait_event=None, + indices_event=None, + trace_ctx=None, + obs=None, + ): + chunk = TransferKVChunk( + room, + kv_indices, + index_slice, + is_last_chunk, + aux_index, + state_indices, + num_kv_tokens, + ) + chunk.indices_event = indices_event + captured.append(chunk) + + fake_mgr = types.SimpleNamespace( + kv_shard_size=1, + is_dummy_cp_rank=False, + enable_all_cp_ranks_for_transfer=False, + _dcp_pack_max_tokens=None, + prepare_send_state_indices=lambda state: state, + add_transfer_request=add_transfer_request, + ) + sender = object.__new__(MooncakeKVSender) + sender.kv_mgr = fake_mgr + sender.bootstrap_room = 1 + sender.aux_index = 0 + sender.conclude_state = None + sender.trace_ctx = mock.MagicMock() + sender._early_send_wait_event = None + sender._transfer_start_time = None + sender._transfer_num_kv_indices = 0 + sender._transfer_num_state_indices = 0 + sender.num_kv_indices = seq_len // page_size + sender.curr_idx = 0 + sender._send_kind = "last" + + fake_sched = types.SimpleNamespace( + token_to_kv_pool_allocator=allocator, + req_to_token_pool=req_pool, + disagg_metadata_buffers=types.SimpleNamespace( + set_buf=lambda req: None + ), + disagg_prefill_bootstrap_queue=types.SimpleNamespace( + kv_manager=types.SimpleNamespace( + kv_args=types.SimpleNamespace( + state_types=[ + StateType.DSV4_REQUEST_STATE, + StateType.REQUEST_ROW, + StateType.SWA, + StateType.SWA, + ] + ) + ) + ), + disagg_prefill_pending_chunk_rids=set(), + sliding_window_size=seq_len, + enable_staging=False, + ) + req = types.SimpleNamespace( + rid="r1", + kv=types.SimpleNamespace(req_pool_idx=0), + start_send_idx=0, + extend_end=seq_len, + origin_input_ids=list(range(seq_len)), + disagg_decode_prefix_len=0, + pending_bootstrap=False, + disagg_kv_sender=sender, + metadata_buffer_index=0, + ) + + parallel = types.SimpleNamespace( + enable_dsa_cache_layer_split=False, num_dp_ranks=4 + ) + with mock.patch( + "sglang.srt.disaggregation.prefill.get_parallel", lambda: parallel + ), mock.patch( + "sglang.srt.disaggregation.common.conn.get_parallel", lambda: parallel + ): + SchedulerDisaggregationPrefillMixin._send_kv_chunk( + fake_sched, req, last_chunk=True + ) + + self.assertEqual(len(captured), 1) + chunk = captured[0] + self.assertTrue(chunk.is_last_chunk) + self.assertIsNotNone(chunk.indices_event) + self.assertEqual(len(chunk.state_indices), 4) + # DSV4_REQUEST_STATE / REQUEST_ROW stay host arrays; both SWA entries + # are pinned CPU tensors before the worker materializes them. + self.assertIsInstance(chunk.state_indices[0], np.ndarray) + self.assertIsInstance(chunk.state_indices[1], np.ndarray) + self.assertIsInstance(chunk.state_indices[2], torch.Tensor) + self.assertIsInstance(chunk.state_indices[3], torch.Tensor) + self.assertFalse(chunk.state_indices[2].is_cuda) + self.assertFalse(chunk.state_indices[3].is_cuda) + + # Now run the real Mooncake worker on the chunk produced above. + peer = types.SimpleNamespace( + is_dummy=False, + mooncake_session_id="peer", + dst_device_kv_indices=None, + dst_kv_indices=[0, 1, 2, 3], + required_dst_info_num=1, + room=1, + endpoint="127.0.0.1", + dst_port=8998, + dst_aux_ptrs=[], + ) + manager = types.SimpleNamespace( + enable_trace=False, + enable_staging=False, + enable_deferred_decode_kv_release=False, + _staging_outstanding=__import__("collections").defaultdict(int), + request_status={1: KVPoll.WaitingForInput}, + transfer_infos={1: {"peer": peer}}, + _prefill_unique_rank=lambda: 0, + session_lock=__import__("threading").Lock(), + failed_sessions=set(), + state_layout_rejections={}, + poison_deferred_ack_room=lambda room: None, + decode_kv_args_table={ + "peer": types.SimpleNamespace( + requires_dcp_relayout=False, + dst_kv_ptrs=[1000], + dst_kv_layer_ids=[0], + dst_kv_item_len=16, + dst_attn_tp_size=1, + dst_aux_ptrs=[], + ) + }, + _get_dsa_cache_transfer_skip_flags=lambda reg: (False, False), + kv_args=types.SimpleNamespace( + kv_data_ptrs=[1], state_types=[StateType.SWA] + ), + is_mla_backend=True, + send_kvcache=lambda *a, **kw: 0, + send_aux=lambda *a, **kw: 0, + conclude_transfer=lambda *a, **kw: None, + _maybe_ack_drained_abort=lambda room: None, + bootstrap_port=8998, + ) + manager.check_status = lambda room: manager.request_status[room] + + seen = {} + + def maybe_send_extra(req, indices, executor, registration): + seen["types"] = [type(x) for x in indices] + seen["swa_values"] = [indices[2].tolist(), indices[3].tolist()] + return 0 + + manager.maybe_send_extra = maybe_send_extra + + queue = __import__("queue").SimpleQueue() + queue.put(chunk) + queue.put(None) + MooncakeKVManager.transfer_worker(manager, queue, mock.MagicMock()) + + self.assertEqual(seen["types"], [np.ndarray] * 4) + self.assertEqual(seen["swa_values"], [[0, 1, 2, 3], [0, 1, 2, 3]]) + torch.cuda.synchronize() + + +class CudaOrderingTest(CustomTestCase): @classmethod def setUpClass(cls): try: diff --git a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py index 16508287c2..b026f177c9 100644 --- a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py +++ b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py @@ -580,6 +580,109 @@ class TestMooncakeEarlySend(CustomTestCase): manager.send_kvcache.assert_called_once() +class TestMooncakeAsyncStateMaterialization(CustomTestCase): + """Real transfer_worker path for pinned state tensors (v2 async state).""" + + def _manager(self): + peer = SimpleNamespace( + is_dummy=False, + mooncake_session_id="peer", + dst_device_kv_indices=None, + dst_kv_indices=[10], + required_dst_info_num=1, + room=1, + endpoint="127.0.0.1", + dst_port=8998, + dst_aux_ptrs=[], + ) + manager = SimpleNamespace( + enable_trace=False, + enable_staging=False, + enable_deferred_decode_kv_release=False, + _staging_outstanding=defaultdict(int), + request_status={1: KVPoll.WaitingForInput}, + transfer_infos={1: {"peer": peer}}, + _prefill_unique_rank=lambda: 0, + session_lock=Lock(), + failed_sessions=set(), + state_layout_rejections={}, + poison_deferred_ack_room=MagicMock(), + decode_kv_args_table={ + "peer": SimpleNamespace( + requires_dcp_relayout=False, + dst_kv_ptrs=[1000], + dst_kv_layer_ids=[0], + dst_kv_item_len=16, + dst_attn_tp_size=1, + dst_aux_ptrs=[], + ) + }, + _get_dsa_cache_transfer_skip_flags=lambda reg: (False, False), + kv_args=SimpleNamespace(kv_data_ptrs=[1], state_types=[StateType.SWA]), + is_mla_backend=True, + send_kvcache=MagicMock(return_value=0), + send_aux=MagicMock(return_value=0), + maybe_send_extra=MagicMock(return_value=0), + conclude_transfer=MagicMock(), + _maybe_ack_drained_abort=MagicMock(), + bootstrap_port=8998, + ) + manager.check_status = lambda room: manager.request_status[room] + return manager + + def _run(self, manager, chunk): + queue = SimpleQueue() + queue.put(chunk) + queue.put(None) + MooncakeKVManager.transfer_worker(manager, queue, MagicMock()) + + def test_worker_materializes_state_after_event(self): + manager = self._manager() + state = torch.tensor([0], dtype=torch.int32) + event = MagicMock() + order = [] + + def sync(): + order.append("sync") + state.fill_(7) + + event.synchronize.side_effect = sync + chunk = TransferKVChunk( + 1, np.array([0], dtype=np.int32), slice(0, 1), True, 0, [state] + ) + chunk.indices_event = event + + def maybe_send_extra(req, indices, executor, registration): + order.append("state") + self.assertIsInstance(indices[0], np.ndarray) + self.assertEqual(indices[0].tolist(), [7]) + return 0 + + manager.maybe_send_extra.side_effect = maybe_send_extra + self._run(manager, chunk) + + self.assertEqual(order, ["sync", "state"]) + self.assertIsInstance(chunk.state_indices[0], np.ndarray) + manager.send_aux.assert_called_once() + + def test_worker_rejects_state_tensor_without_event(self): + manager = self._manager() + chunk = TransferKVChunk( + 1, + np.array([0], dtype=np.int32), + slice(0, 1), + True, + 0, + [torch.tensor([0], dtype=torch.int32)], + ) + with self.assertRaises(RuntimeError) as ctx: + self._run(manager, chunk) + self.assertIn( + "async state indices require an indices_event", str(ctx.exception) + ) + manager.maybe_send_extra.assert_not_called() + + class TestMooncakeStagingDeviceType(unittest.TestCase): """Staging init must receive the runtime's device, not a hardcoded "cuda".