test: cover async SWA state-payload call chain; clarify CP gate
Review follow-up for the v2 async state payload:
- P2: new tests use CustomTestCase (with a local fallback when sglang is not
importable), matching the disaggregation/Mooncake test convention.
- P3: CP gate docstring now says all-CP-rank transfer *without DSA cache
layer split* keeps the synchronous path.
- Add real call-chain coverage:
* _send_kv_chunk -> sender.send -> add_transfer_request -> transfer_worker
-> materialize_state_indices -> maybe_send_extra, with DSV4's four
components (DSV4_REQUEST_STATE, REQUEST_ROW, SWA, SWA);
* the worker materializes pinned tensors only after indices_event and
fails loudly when a tensor has no event;
* a second materialization without an event (staging retry) is a no-op;
* real CUDA ordering: the pinned helper returns before a long kernel and
the event-protected values match kv_to_page_indices.
- Verified on b300 (P container, overlay source): 9 passed in
test_async_state_payload.py, 2 passed in the new
TestMooncakeAsyncStateMaterialization class; the old batching suite has
11 pre-existing failures that also fail on the image source.
The test-only batching suite requires the detail envs to be off when run in
the live pod (the inherited SGLANG_TRACE_MOONCAKE_DETAIL=1 exposes an existing
test fixture omission); noted, not changed here.
This commit is contained in:
@@ -575,10 +575,10 @@ def should_async_state_payload(
|
||||
) -> bool:
|
||||
"""Gate for the first async state-payload version (SWA-only).
|
||||
|
||||
``kv_shard_size > 1`` and the all-CP-rank transfer mode keep the old
|
||||
synchronous path: their indices are filtered before the transfer event,
|
||||
and deferring that to the worker would widen the pre-event shard-filter
|
||||
risk tracked by #45492.
|
||||
``kv_shard_size > 1`` and all-CP-rank transfer **without DSA cache layer
|
||||
split** keep the old synchronous path: their indices are filtered before
|
||||
the transfer event, and deferring that to the worker would widen the
|
||||
pre-event shard-filter risk tracked by #45492.
|
||||
"""
|
||||
if not last_chunk or not supports_async_state_indices:
|
||||
return False
|
||||
|
||||
@@ -26,6 +26,11 @@ from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
except Exception: # local helper-only runs without sglang installed
|
||||
CustomTestCase = unittest.TestCase
|
||||
|
||||
|
||||
def _repo_root():
|
||||
return pathlib.Path(__file__).resolve().parents[4]
|
||||
@@ -96,7 +101,7 @@ def _load_pinned_helper():
|
||||
return namespace["kv_to_page_indices_pinned"], calls, _FakeTensor
|
||||
|
||||
|
||||
class AsyncStateGateTest(unittest.TestCase):
|
||||
class AsyncStateGateTest(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mod = types.SimpleNamespace()
|
||||
@@ -153,7 +158,7 @@ class AsyncStateGateTest(unittest.TestCase):
|
||||
mod.require_async_state_event(True, True, None)
|
||||
|
||||
|
||||
class MaterializeStateIndicesTest(unittest.TestCase):
|
||||
class MaterializeStateIndicesTest(CustomTestCase):
|
||||
class _FakeTensor:
|
||||
def __init__(self, values):
|
||||
self.values = values
|
||||
@@ -189,6 +194,14 @@ class MaterializeStateIndicesTest(unittest.TestCase):
|
||||
self.assertIsNone(out[1])
|
||||
self.assertIs(out[2], np_indices)
|
||||
|
||||
def test_second_materialization_without_event_is_safe(self):
|
||||
"""Staging retry: first pass materializes; second pass has no event."""
|
||||
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
|
||||
first = self.fn([self._FakeTensor([1, 2])], object())
|
||||
second = self.fn(first, None)
|
||||
self.assertIs(second, first)
|
||||
np.testing.assert_array_equal(first[0], np.array([1, 2], dtype=np.int32))
|
||||
|
||||
def test_no_tensor_needs_no_event(self):
|
||||
np_indices = np.array([1], dtype=np.int32)
|
||||
with mock.patch.dict(sys.modules, {"torch": self._fake_torch()}):
|
||||
@@ -198,7 +211,7 @@ class MaterializeStateIndicesTest(unittest.TestCase):
|
||||
self.assertIsNone(self.fn(None, None))
|
||||
|
||||
|
||||
class PinnedHelperBodyTest(unittest.TestCase):
|
||||
class PinnedHelperBodyTest(CustomTestCase):
|
||||
def test_pinned_nonblocking_no_cpu_sync(self):
|
||||
helper, calls, fake_tensor = _load_pinned_helper()
|
||||
src = fake_tensor(range(16), dtype="torch.int32")
|
||||
@@ -213,7 +226,233 @@ class PinnedHelperBodyTest(unittest.TestCase):
|
||||
self.assertTrue(calls["copy"][0][1])
|
||||
|
||||
|
||||
class CudaOrderingTest(unittest.TestCase):
|
||||
class CudaSendChainTest(CustomTestCase):
|
||||
"""Real _send_kv_chunk -> sender.send -> transfer_worker call chain."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
try:
|
||||
import torch
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise unittest.SkipTest("CUDA required")
|
||||
from sglang.srt.disaggregation.prefill import ( # noqa: F401
|
||||
SchedulerDisaggregationPrefillMixin,
|
||||
)
|
||||
except unittest.SkipTest:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise unittest.SkipTest(f"torch/CUDA/sglang unavailable: {exc}")
|
||||
|
||||
def test_send_chunk_queues_async_state_and_worker_materializes(self):
|
||||
import types
|
||||
|
||||
import torch
|
||||
from sglang.srt.disaggregation.base.conn import KVPoll, StateType
|
||||
from sglang.srt.disaggregation.common.utils import TransferKVChunk
|
||||
from sglang.srt.disaggregation.mooncake.conn import (
|
||||
MooncakeKVManager,
|
||||
MooncakeKVSender,
|
||||
)
|
||||
from sglang.srt.disaggregation.prefill import (
|
||||
SchedulerDisaggregationPrefillMixin,
|
||||
)
|
||||
from sglang.srt.observability import batch_telemetry # noqa: F401
|
||||
|
||||
seq_len = 32
|
||||
page_size = 8
|
||||
req_to_token = torch.arange(0, seq_len, device="cuda", dtype=torch.int32).view(
|
||||
1, seq_len
|
||||
)
|
||||
allocator = types.SimpleNamespace(
|
||||
page_size=page_size,
|
||||
get_kvcache=lambda: types.SimpleNamespace(
|
||||
request_window=None,
|
||||
sliding_window=seq_len,
|
||||
request_state_transfer_indices=lambda req_idx, length: np.array(
|
||||
[req_idx], dtype=np.int32
|
||||
),
|
||||
),
|
||||
translate_swa_indices_for_transfer=lambda x: x,
|
||||
translate_kv_indices_for_transfer=lambda x: x,
|
||||
)
|
||||
req_pool = types.SimpleNamespace(req_to_token=req_to_token)
|
||||
captured = []
|
||||
|
||||
def add_transfer_request(
|
||||
room,
|
||||
kv_indices,
|
||||
index_slice,
|
||||
is_last_chunk,
|
||||
aux_index=None,
|
||||
state_indices=None,
|
||||
num_kv_tokens=None,
|
||||
wait_event=None,
|
||||
indices_event=None,
|
||||
trace_ctx=None,
|
||||
obs=None,
|
||||
):
|
||||
chunk = TransferKVChunk(
|
||||
room,
|
||||
kv_indices,
|
||||
index_slice,
|
||||
is_last_chunk,
|
||||
aux_index,
|
||||
state_indices,
|
||||
num_kv_tokens,
|
||||
)
|
||||
chunk.indices_event = indices_event
|
||||
captured.append(chunk)
|
||||
|
||||
fake_mgr = types.SimpleNamespace(
|
||||
kv_shard_size=1,
|
||||
is_dummy_cp_rank=False,
|
||||
enable_all_cp_ranks_for_transfer=False,
|
||||
_dcp_pack_max_tokens=None,
|
||||
prepare_send_state_indices=lambda state: state,
|
||||
add_transfer_request=add_transfer_request,
|
||||
)
|
||||
sender = object.__new__(MooncakeKVSender)
|
||||
sender.kv_mgr = fake_mgr
|
||||
sender.bootstrap_room = 1
|
||||
sender.aux_index = 0
|
||||
sender.conclude_state = None
|
||||
sender.trace_ctx = mock.MagicMock()
|
||||
sender._early_send_wait_event = None
|
||||
sender._transfer_start_time = None
|
||||
sender._transfer_num_kv_indices = 0
|
||||
sender._transfer_num_state_indices = 0
|
||||
sender.num_kv_indices = seq_len // page_size
|
||||
sender.curr_idx = 0
|
||||
sender._send_kind = "last"
|
||||
|
||||
fake_sched = types.SimpleNamespace(
|
||||
token_to_kv_pool_allocator=allocator,
|
||||
req_to_token_pool=req_pool,
|
||||
disagg_metadata_buffers=types.SimpleNamespace(
|
||||
set_buf=lambda req: None
|
||||
),
|
||||
disagg_prefill_bootstrap_queue=types.SimpleNamespace(
|
||||
kv_manager=types.SimpleNamespace(
|
||||
kv_args=types.SimpleNamespace(
|
||||
state_types=[
|
||||
StateType.DSV4_REQUEST_STATE,
|
||||
StateType.REQUEST_ROW,
|
||||
StateType.SWA,
|
||||
StateType.SWA,
|
||||
]
|
||||
)
|
||||
)
|
||||
),
|
||||
disagg_prefill_pending_chunk_rids=set(),
|
||||
sliding_window_size=seq_len,
|
||||
enable_staging=False,
|
||||
)
|
||||
req = types.SimpleNamespace(
|
||||
rid="r1",
|
||||
kv=types.SimpleNamespace(req_pool_idx=0),
|
||||
start_send_idx=0,
|
||||
extend_end=seq_len,
|
||||
origin_input_ids=list(range(seq_len)),
|
||||
disagg_decode_prefix_len=0,
|
||||
pending_bootstrap=False,
|
||||
disagg_kv_sender=sender,
|
||||
metadata_buffer_index=0,
|
||||
)
|
||||
|
||||
parallel = types.SimpleNamespace(
|
||||
enable_dsa_cache_layer_split=False, num_dp_ranks=4
|
||||
)
|
||||
with mock.patch(
|
||||
"sglang.srt.disaggregation.prefill.get_parallel", lambda: parallel
|
||||
), mock.patch(
|
||||
"sglang.srt.disaggregation.common.conn.get_parallel", lambda: parallel
|
||||
):
|
||||
SchedulerDisaggregationPrefillMixin._send_kv_chunk(
|
||||
fake_sched, req, last_chunk=True
|
||||
)
|
||||
|
||||
self.assertEqual(len(captured), 1)
|
||||
chunk = captured[0]
|
||||
self.assertTrue(chunk.is_last_chunk)
|
||||
self.assertIsNotNone(chunk.indices_event)
|
||||
self.assertEqual(len(chunk.state_indices), 4)
|
||||
# DSV4_REQUEST_STATE / REQUEST_ROW stay host arrays; both SWA entries
|
||||
# are pinned CPU tensors before the worker materializes them.
|
||||
self.assertIsInstance(chunk.state_indices[0], np.ndarray)
|
||||
self.assertIsInstance(chunk.state_indices[1], np.ndarray)
|
||||
self.assertIsInstance(chunk.state_indices[2], torch.Tensor)
|
||||
self.assertIsInstance(chunk.state_indices[3], torch.Tensor)
|
||||
self.assertFalse(chunk.state_indices[2].is_cuda)
|
||||
self.assertFalse(chunk.state_indices[3].is_cuda)
|
||||
|
||||
# Now run the real Mooncake worker on the chunk produced above.
|
||||
peer = types.SimpleNamespace(
|
||||
is_dummy=False,
|
||||
mooncake_session_id="peer",
|
||||
dst_device_kv_indices=None,
|
||||
dst_kv_indices=[0, 1, 2, 3],
|
||||
required_dst_info_num=1,
|
||||
room=1,
|
||||
endpoint="127.0.0.1",
|
||||
dst_port=8998,
|
||||
dst_aux_ptrs=[],
|
||||
)
|
||||
manager = types.SimpleNamespace(
|
||||
enable_trace=False,
|
||||
enable_staging=False,
|
||||
enable_deferred_decode_kv_release=False,
|
||||
_staging_outstanding=__import__("collections").defaultdict(int),
|
||||
request_status={1: KVPoll.WaitingForInput},
|
||||
transfer_infos={1: {"peer": peer}},
|
||||
_prefill_unique_rank=lambda: 0,
|
||||
session_lock=__import__("threading").Lock(),
|
||||
failed_sessions=set(),
|
||||
state_layout_rejections={},
|
||||
poison_deferred_ack_room=lambda room: None,
|
||||
decode_kv_args_table={
|
||||
"peer": types.SimpleNamespace(
|
||||
requires_dcp_relayout=False,
|
||||
dst_kv_ptrs=[1000],
|
||||
dst_kv_layer_ids=[0],
|
||||
dst_kv_item_len=16,
|
||||
dst_attn_tp_size=1,
|
||||
dst_aux_ptrs=[],
|
||||
)
|
||||
},
|
||||
_get_dsa_cache_transfer_skip_flags=lambda reg: (False, False),
|
||||
kv_args=types.SimpleNamespace(
|
||||
kv_data_ptrs=[1], state_types=[StateType.SWA]
|
||||
),
|
||||
is_mla_backend=True,
|
||||
send_kvcache=lambda *a, **kw: 0,
|
||||
send_aux=lambda *a, **kw: 0,
|
||||
conclude_transfer=lambda *a, **kw: None,
|
||||
_maybe_ack_drained_abort=lambda room: None,
|
||||
bootstrap_port=8998,
|
||||
)
|
||||
manager.check_status = lambda room: manager.request_status[room]
|
||||
|
||||
seen = {}
|
||||
|
||||
def maybe_send_extra(req, indices, executor, registration):
|
||||
seen["types"] = [type(x) for x in indices]
|
||||
seen["swa_values"] = [indices[2].tolist(), indices[3].tolist()]
|
||||
return 0
|
||||
|
||||
manager.maybe_send_extra = maybe_send_extra
|
||||
|
||||
queue = __import__("queue").SimpleQueue()
|
||||
queue.put(chunk)
|
||||
queue.put(None)
|
||||
MooncakeKVManager.transfer_worker(manager, queue, mock.MagicMock())
|
||||
|
||||
self.assertEqual(seen["types"], [np.ndarray] * 4)
|
||||
self.assertEqual(seen["swa_values"], [[0, 1, 2, 3], [0, 1, 2, 3]])
|
||||
torch.cuda.synchronize()
|
||||
|
||||
|
||||
class CudaOrderingTest(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
try:
|
||||
|
||||
@@ -580,6 +580,109 @@ class TestMooncakeEarlySend(CustomTestCase):
|
||||
manager.send_kvcache.assert_called_once()
|
||||
|
||||
|
||||
class TestMooncakeAsyncStateMaterialization(CustomTestCase):
|
||||
"""Real transfer_worker path for pinned state tensors (v2 async state)."""
|
||||
|
||||
def _manager(self):
|
||||
peer = SimpleNamespace(
|
||||
is_dummy=False,
|
||||
mooncake_session_id="peer",
|
||||
dst_device_kv_indices=None,
|
||||
dst_kv_indices=[10],
|
||||
required_dst_info_num=1,
|
||||
room=1,
|
||||
endpoint="127.0.0.1",
|
||||
dst_port=8998,
|
||||
dst_aux_ptrs=[],
|
||||
)
|
||||
manager = SimpleNamespace(
|
||||
enable_trace=False,
|
||||
enable_staging=False,
|
||||
enable_deferred_decode_kv_release=False,
|
||||
_staging_outstanding=defaultdict(int),
|
||||
request_status={1: KVPoll.WaitingForInput},
|
||||
transfer_infos={1: {"peer": peer}},
|
||||
_prefill_unique_rank=lambda: 0,
|
||||
session_lock=Lock(),
|
||||
failed_sessions=set(),
|
||||
state_layout_rejections={},
|
||||
poison_deferred_ack_room=MagicMock(),
|
||||
decode_kv_args_table={
|
||||
"peer": SimpleNamespace(
|
||||
requires_dcp_relayout=False,
|
||||
dst_kv_ptrs=[1000],
|
||||
dst_kv_layer_ids=[0],
|
||||
dst_kv_item_len=16,
|
||||
dst_attn_tp_size=1,
|
||||
dst_aux_ptrs=[],
|
||||
)
|
||||
},
|
||||
_get_dsa_cache_transfer_skip_flags=lambda reg: (False, False),
|
||||
kv_args=SimpleNamespace(kv_data_ptrs=[1], state_types=[StateType.SWA]),
|
||||
is_mla_backend=True,
|
||||
send_kvcache=MagicMock(return_value=0),
|
||||
send_aux=MagicMock(return_value=0),
|
||||
maybe_send_extra=MagicMock(return_value=0),
|
||||
conclude_transfer=MagicMock(),
|
||||
_maybe_ack_drained_abort=MagicMock(),
|
||||
bootstrap_port=8998,
|
||||
)
|
||||
manager.check_status = lambda room: manager.request_status[room]
|
||||
return manager
|
||||
|
||||
def _run(self, manager, chunk):
|
||||
queue = SimpleQueue()
|
||||
queue.put(chunk)
|
||||
queue.put(None)
|
||||
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
|
||||
|
||||
def test_worker_materializes_state_after_event(self):
|
||||
manager = self._manager()
|
||||
state = torch.tensor([0], dtype=torch.int32)
|
||||
event = MagicMock()
|
||||
order = []
|
||||
|
||||
def sync():
|
||||
order.append("sync")
|
||||
state.fill_(7)
|
||||
|
||||
event.synchronize.side_effect = sync
|
||||
chunk = TransferKVChunk(
|
||||
1, np.array([0], dtype=np.int32), slice(0, 1), True, 0, [state]
|
||||
)
|
||||
chunk.indices_event = event
|
||||
|
||||
def maybe_send_extra(req, indices, executor, registration):
|
||||
order.append("state")
|
||||
self.assertIsInstance(indices[0], np.ndarray)
|
||||
self.assertEqual(indices[0].tolist(), [7])
|
||||
return 0
|
||||
|
||||
manager.maybe_send_extra.side_effect = maybe_send_extra
|
||||
self._run(manager, chunk)
|
||||
|
||||
self.assertEqual(order, ["sync", "state"])
|
||||
self.assertIsInstance(chunk.state_indices[0], np.ndarray)
|
||||
manager.send_aux.assert_called_once()
|
||||
|
||||
def test_worker_rejects_state_tensor_without_event(self):
|
||||
manager = self._manager()
|
||||
chunk = TransferKVChunk(
|
||||
1,
|
||||
np.array([0], dtype=np.int32),
|
||||
slice(0, 1),
|
||||
True,
|
||||
0,
|
||||
[torch.tensor([0], dtype=torch.int32)],
|
||||
)
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
self._run(manager, chunk)
|
||||
self.assertIn(
|
||||
"async state indices require an indices_event", str(ctx.exception)
|
||||
)
|
||||
manager.maybe_send_extra.assert_not_called()
|
||||
|
||||
|
||||
class TestMooncakeStagingDeviceType(unittest.TestCase):
|
||||
"""Staging init must receive the runtime's device, not a hardcoded "cuda".
|
||||
|
||||
|
||||
Reference in New Issue
Block a user