test: cover async SWA state-payload call chain; clarify CP gate

Review follow-up for the v2 async state payload:

- P2: new tests use CustomTestCase (with a local fallback when sglang is not
  importable), matching the disaggregation/Mooncake test convention.
- P3: CP gate docstring now says all-CP-rank transfer *without DSA cache
  layer split* keeps the synchronous path.
- Add real call-chain coverage:
  * _send_kv_chunk -> sender.send -> add_transfer_request -> transfer_worker
    -> materialize_state_indices -> maybe_send_extra, with DSV4's four
    components (DSV4_REQUEST_STATE, REQUEST_ROW, SWA, SWA);
  * the worker materializes pinned tensors only after indices_event and
    fails loudly when a tensor has no event;
  * a second materialization without an event (staging retry) is a no-op;
  * real CUDA ordering: the pinned helper returns before a long kernel and
    the event-protected values match kv_to_page_indices.
- Verified on b300 (P container, overlay source): 9 passed in
  test_async_state_payload.py, 2 passed in the new
  TestMooncakeAsyncStateMaterialization class; the old batching suite has
  11 pre-existing failures that also fail on the image source.

The test-only batching suite requires the detail envs to be off when run in
the live pod (the inherited SGLANG_TRACE_MOONCAKE_DETAIL=1 exposes an existing
test fixture omission); noted, not changed here.
This commit is contained in:
ymk
2026-11-07 00:22:57 +08:00
parent 3b0baa3455
commit 7b3f34cbc0
3 changed files with 350 additions and 8 deletions
+4 -4
View File
@@ -575,10 +575,10 @@ def should_async_state_payload(
) -> bool:
"""Gate for the first async state-payload version (SWA-only).
``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old
synchronous path: their indices are filtered before the transfer event,
and deferring that to the worker would widen the pre-event shard-filter
risk tracked by #45492.
``kv_shard_size > 1`` and all-CP-rank transfer **without DSA cache layer
split** keep the old synchronous path: their indices are filtered before
the transfer event, and deferring that to the worker would widen the
pre-event shard-filter risk tracked by #45492.
"""
if not last_chunk or not supports_async_state_indices:
return False
@@ -26,6 +26,11 @@ from unittest import mock
import numpy as np
try:
from sglang.test.test_utils import CustomTestCase
except Exception: # local helper-only runs without sglang installed
CustomTestCase = unittest.TestCase
def _repo_root():
return pathlib.Path(__file__).resolve().parents[4]
@@ -96,7 +101,7 @@ def _load_pinned_helper():
return namespace["kv_to_page_indices_pinned"], calls, _FakeTensor
class AsyncStateGateTest(unittest.TestCase):
class AsyncStateGateTest(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.mod = types.SimpleNamespace()
@@ -153,7 +158,7 @@ class AsyncStateGateTest(unittest.TestCase):
mod.require_async_state_event(True, True, None)
class MaterializeStateIndicesTest(unittest.TestCase):
class MaterializeStateIndicesTest(CustomTestCase):
class _FakeTensor:
def __init__(self, values):
self.values = values
@@ -189,6 +194,14 @@ class MaterializeStateIndicesTest(unittest.TestCase):
self.assertIsNone(out[1])
self.assertIs(out[2], np_indices)
def test_second_materialization_without_event_is_safe(self):
"""Staging retry: first pass materializes; second pass has no event."""
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
first = self.fn([self._FakeTensor([1, 2])], object())
second = self.fn(first, None)
self.assertIs(second, first)
np.testing.assert_array_equal(first[0], np.array([1, 2], dtype=np.int32))
def test_no_tensor_needs_no_event(self):
np_indices = np.array([1], dtype=np.int32)
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
@@ -198,7 +211,7 @@ class MaterializeStateIndicesTest(unittest.TestCase):
self.assertIsNone(self.fn(None, None))
class PinnedHelperBodyTest(unittest.TestCase):
class PinnedHelperBodyTest(CustomTestCase):
def test_pinned_nonblocking_no_cpu_sync(self):
helper, calls, fake_tensor = _load_pinned_helper()
src = fake_tensor(range(16), dtype="torch.int32")
@@ -213,7 +226,233 @@ class PinnedHelperBodyTest(unittest.TestCase):
self.assertTrue(calls["copy"][0][1])
class CudaOrderingTest(unittest.TestCase):
class CudaSendChainTest(CustomTestCase):
"""Real _send_kv_chunk -> sender.send -> transfer_worker call chain."""
@classmethod
def setUpClass(cls):
try:
import torch
if not torch.cuda.is_available():
raise unittest.SkipTest("CUDA required")
from sglang.srt.disaggregation.prefill import ( # noqa: F401
SchedulerDisaggregationPrefillMixin,
)
except unittest.SkipTest:
raise
except Exception as exc: # noqa: BLE001
raise unittest.SkipTest(f"torch/CUDA/sglang unavailable: {exc}")
def test_send_chunk_queues_async_state_and_worker_materializes(self):
import types
import torch
from sglang.srt.disaggregation.base.conn import KVPoll, StateType
from sglang.srt.disaggregation.common.utils import TransferKVChunk
from sglang.srt.disaggregation.mooncake.conn import (
MooncakeKVManager,
MooncakeKVSender,
)
from sglang.srt.disaggregation.prefill import (
SchedulerDisaggregationPrefillMixin,
)
from sglang.srt.observability import batch_telemetry # noqa: F401
seq_len = 32
page_size = 8
req_to_token = torch.arange(0, seq_len, device="cuda", dtype=torch.int32).view(
1, seq_len
)
allocator = types.SimpleNamespace(
page_size=page_size,
get_kvcache=lambda: types.SimpleNamespace(
request_window=None,
sliding_window=seq_len,
request_state_transfer_indices=lambda req_idx, length: np.array(
[req_idx], dtype=np.int32
),
),
translate_swa_indices_for_transfer=lambda x: x,
translate_kv_indices_for_transfer=lambda x: x,
)
req_pool = types.SimpleNamespace(req_to_token=req_to_token)
captured = []
def add_transfer_request(
room,
kv_indices,
index_slice,
is_last_chunk,
aux_index=None,
state_indices=None,
num_kv_tokens=None,
wait_event=None,
indices_event=None,
trace_ctx=None,
obs=None,
):
chunk = TransferKVChunk(
room,
kv_indices,
index_slice,
is_last_chunk,
aux_index,
state_indices,
num_kv_tokens,
)
chunk.indices_event = indices_event
captured.append(chunk)
fake_mgr = types.SimpleNamespace(
kv_shard_size=1,
is_dummy_cp_rank=False,
enable_all_cp_ranks_for_transfer=False,
_dcp_pack_max_tokens=None,
prepare_send_state_indices=lambda state: state,
add_transfer_request=add_transfer_request,
)
sender = object.__new__(MooncakeKVSender)
sender.kv_mgr = fake_mgr
sender.bootstrap_room = 1
sender.aux_index = 0
sender.conclude_state = None
sender.trace_ctx = mock.MagicMock()
sender._early_send_wait_event = None
sender._transfer_start_time = None
sender._transfer_num_kv_indices = 0
sender._transfer_num_state_indices = 0
sender.num_kv_indices = seq_len // page_size
sender.curr_idx = 0
sender._send_kind = "last"
fake_sched = types.SimpleNamespace(
token_to_kv_pool_allocator=allocator,
req_to_token_pool=req_pool,
disagg_metadata_buffers=types.SimpleNamespace(
set_buf=lambda req: None
),
disagg_prefill_bootstrap_queue=types.SimpleNamespace(
kv_manager=types.SimpleNamespace(
kv_args=types.SimpleNamespace(
state_types=[
StateType.DSV4_REQUEST_STATE,
StateType.REQUEST_ROW,
StateType.SWA,
StateType.SWA,
]
)
)
),
disagg_prefill_pending_chunk_rids=set(),
sliding_window_size=seq_len,
enable_staging=False,
)
req = types.SimpleNamespace(
rid="r1",
kv=types.SimpleNamespace(req_pool_idx=0),
start_send_idx=0,
extend_end=seq_len,
origin_input_ids=list(range(seq_len)),
disagg_decode_prefix_len=0,
pending_bootstrap=False,
disagg_kv_sender=sender,
metadata_buffer_index=0,
)
parallel = types.SimpleNamespace(
enable_dsa_cache_layer_split=False, num_dp_ranks=4
)
with mock.patch(
"sglang.srt.disaggregation.prefill.get_parallel", lambda: parallel
), mock.patch(
"sglang.srt.disaggregation.common.conn.get_parallel", lambda: parallel
):
SchedulerDisaggregationPrefillMixin._send_kv_chunk(
fake_sched, req, last_chunk=True
)
self.assertEqual(len(captured), 1)
chunk = captured[0]
self.assertTrue(chunk.is_last_chunk)
self.assertIsNotNone(chunk.indices_event)
self.assertEqual(len(chunk.state_indices), 4)
# DSV4_REQUEST_STATE / REQUEST_ROW stay host arrays; both SWA entries
# are pinned CPU tensors before the worker materializes them.
self.assertIsInstance(chunk.state_indices[0], np.ndarray)
self.assertIsInstance(chunk.state_indices[1], np.ndarray)
self.assertIsInstance(chunk.state_indices[2], torch.Tensor)
self.assertIsInstance(chunk.state_indices[3], torch.Tensor)
self.assertFalse(chunk.state_indices[2].is_cuda)
self.assertFalse(chunk.state_indices[3].is_cuda)
# Now run the real Mooncake worker on the chunk produced above.
peer = types.SimpleNamespace(
is_dummy=False,
mooncake_session_id="peer",
dst_device_kv_indices=None,
dst_kv_indices=[0, 1, 2, 3],
required_dst_info_num=1,
room=1,
endpoint="127.0.0.1",
dst_port=8998,
dst_aux_ptrs=[],
)
manager = types.SimpleNamespace(
enable_trace=False,
enable_staging=False,
enable_deferred_decode_kv_release=False,
_staging_outstanding=__import__("collections").defaultdict(int),
request_status={1: KVPoll.WaitingForInput},
transfer_infos={1: {"peer": peer}},
_prefill_unique_rank=lambda: 0,
session_lock=__import__("threading").Lock(),
failed_sessions=set(),
state_layout_rejections={},
poison_deferred_ack_room=lambda room: None,
decode_kv_args_table={
"peer": types.SimpleNamespace(
requires_dcp_relayout=False,
dst_kv_ptrs=[1000],
dst_kv_layer_ids=[0],
dst_kv_item_len=16,
dst_attn_tp_size=1,
dst_aux_ptrs=[],
)
},
_get_dsa_cache_transfer_skip_flags=lambda reg: (False, False),
kv_args=types.SimpleNamespace(
kv_data_ptrs=[1], state_types=[StateType.SWA]
),
is_mla_backend=True,
send_kvcache=lambda *a, **kw: 0,
send_aux=lambda *a, **kw: 0,
conclude_transfer=lambda *a, **kw: None,
_maybe_ack_drained_abort=lambda room: None,
bootstrap_port=8998,
)
manager.check_status = lambda room: manager.request_status[room]
seen = {}
def maybe_send_extra(req, indices, executor, registration):
seen["types"] = [type(x) for x in indices]
seen["swa_values"] = [indices[2].tolist(), indices[3].tolist()]
return 0
manager.maybe_send_extra = maybe_send_extra
queue = __import__("queue").SimpleQueue()
queue.put(chunk)
queue.put(None)
MooncakeKVManager.transfer_worker(manager, queue, mock.MagicMock())
self.assertEqual(seen["types"], [np.ndarray] * 4)
self.assertEqual(seen["swa_values"], [[0, 1, 2, 3], [0, 1, 2, 3]])
torch.cuda.synchronize()
class CudaOrderingTest(CustomTestCase):
@classmethod
def setUpClass(cls):
try:
@@ -580,6 +580,109 @@ class TestMooncakeEarlySend(CustomTestCase):
manager.send_kvcache.assert_called_once()
class TestMooncakeAsyncStateMaterialization(CustomTestCase):
"""Real transfer_worker path for pinned state tensors (v2 async state)."""
def _manager(self):
peer = SimpleNamespace(
is_dummy=False,
mooncake_session_id="peer",
dst_device_kv_indices=None,
dst_kv_indices=[10],
required_dst_info_num=1,
room=1,
endpoint="127.0.0.1",
dst_port=8998,
dst_aux_ptrs=[],
)
manager = SimpleNamespace(
enable_trace=False,
enable_staging=False,
enable_deferred_decode_kv_release=False,
_staging_outstanding=defaultdict(int),
request_status={1: KVPoll.WaitingForInput},
transfer_infos={1: {"peer": peer}},
_prefill_unique_rank=lambda: 0,
session_lock=Lock(),
failed_sessions=set(),
state_layout_rejections={},
poison_deferred_ack_room=MagicMock(),
decode_kv_args_table={
"peer": SimpleNamespace(
requires_dcp_relayout=False,
dst_kv_ptrs=[1000],
dst_kv_layer_ids=[0],
dst_kv_item_len=16,
dst_attn_tp_size=1,
dst_aux_ptrs=[],
)
},
_get_dsa_cache_transfer_skip_flags=lambda reg: (False, False),
kv_args=SimpleNamespace(kv_data_ptrs=[1], state_types=[StateType.SWA]),
is_mla_backend=True,
send_kvcache=MagicMock(return_value=0),
send_aux=MagicMock(return_value=0),
maybe_send_extra=MagicMock(return_value=0),
conclude_transfer=MagicMock(),
_maybe_ack_drained_abort=MagicMock(),
bootstrap_port=8998,
)
manager.check_status = lambda room: manager.request_status[room]
return manager
def _run(self, manager, chunk):
queue = SimpleQueue()
queue.put(chunk)
queue.put(None)
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
def test_worker_materializes_state_after_event(self):
manager = self._manager()
state = torch.tensor([0], dtype=torch.int32)
event = MagicMock()
order = []
def sync():
order.append("sync")
state.fill_(7)
event.synchronize.side_effect = sync
chunk = TransferKVChunk(
1, np.array([0], dtype=np.int32), slice(0, 1), True, 0, [state]
)
chunk.indices_event = event
def maybe_send_extra(req, indices, executor, registration):
order.append("state")
self.assertIsInstance(indices[0], np.ndarray)
self.assertEqual(indices[0].tolist(), [7])
return 0
manager.maybe_send_extra.side_effect = maybe_send_extra
self._run(manager, chunk)
self.assertEqual(order, ["sync", "state"])
self.assertIsInstance(chunk.state_indices[0], np.ndarray)
manager.send_aux.assert_called_once()
def test_worker_rejects_state_tensor_without_event(self):
manager = self._manager()
chunk = TransferKVChunk(
1,
np.array([0], dtype=np.int32),
slice(0, 1),
True,
0,
[torch.tensor([0], dtype=torch.int32)],
)
with self.assertRaises(RuntimeError) as ctx:
self._run(manager, chunk)
self.assertIn(
"async state indices require an indices_event", str(ctx.exception)
)
manager.maybe_send_extra.assert_not_called()
class TestMooncakeStagingDeviceType(unittest.TestCase):
"""Staging init must receive the runtime's device, not a hardcoded "cuda".