[PD] Validate decode state layout once at registration (#42051)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Shangming Cai
2026-10-04 18:32:30 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent bc5a055dd7
commit 148da8e77d
3 changed files with 496 additions and 56 deletions
+241 -56
View File
@@ -8,7 +8,7 @@ import struct
import threading
import time
from collections import defaultdict
from typing import List, Optional, Set, Tuple, Union
from typing import Dict, List, Optional, Set, Tuple, Union
import numpy as np
import numpy.typing as npt
@@ -216,6 +216,27 @@ class KVArgsRegisterInfo:
)
def _uses_flat_entry_layout(
*,
mla_backend: bool,
force_flat: bool,
has_layer_ids: bool,
state_type: Optional[StateType],
num_src_entries: int,
num_dst_entries: int,
) -> bool:
"""One region per transfer entry, as opposed to plain MHA's K-then-V split.
Shared by the transfer and the registration-time stride check so both
judge the same entries. Unified SWA publishes one page-envelope region
even on an MHA backend.
"""
is_single_region_swa = (
state_type == StateType.SWA and num_src_entries == 1 and num_dst_entries == 1
)
return mla_backend or force_flat or has_layer_ids or is_single_region_swa
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
AUX_DATA_HEADER = b"AUX_DATA"
# Implements teardown() below, so runtime PD role switching is supported.
@@ -250,6 +271,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.session_failures = defaultdict(int)
self.failed_sessions = set()
# Separate from failed_sessions, which _run_one_probe_pass clears
# on a successful probe: an incompatible decode is still reachable,
# so only a re-registration may lift this.
self.state_layout_rejections: Dict[str, str] = {}
# Peers already cleared at registration; skip the per-transfer check.
self.state_strides_validated: Set[str] = set()
self.session_lock = threading.Lock()
self.start_prefill_thread()
# Determine the number of threads to use for kv sender
@@ -762,6 +789,167 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
)
def _state_stride_mismatch(
self,
*,
src_data_ptrs: list[int],
dst_data_ptrs: list[int],
item_lens: list[int],
dst_item_lens: list[int],
state_type: Optional[StateType] = None,
force_flat: bool = False,
src_layer_ids: Optional[List[int]] = None,
dst_layer_ids: Optional[List[int]] = None,
) -> Optional[str]:
"""First entry whose decode stride differs from prefill's, else None.
The destination is addressed as ``dst_ptr + dst_index * src_item_len``,
so mapped entries must agree on the stride or the write lands in the
wrong slot.
"""
has_layer_ids = bool(src_layer_ids or dst_layer_ids)
if not _uses_flat_entry_layout(
mla_backend=self.is_mla_backend or self.is_hybrid_mla_backend,
force_flat=force_flat,
has_layer_ids=has_layer_ids,
state_type=state_type,
num_src_entries=len(src_data_ptrs),
num_dst_entries=len(dst_data_ptrs),
):
# The MHA branch splits the entries into K/V and is not covered.
return None
if has_layer_ids:
pairs = build_transfer_entry_pairs(
src_layer_ids or [],
dst_layer_ids or [],
len(src_data_ptrs),
len(dst_data_ptrs),
allow_positional_fallback=self.pp_size == 1,
)
for i, j in pairs:
if j >= len(dst_item_lens):
return (
f"{state_type} decode published {len(dst_item_lens)} item "
f"lengths but paired entry dst[{j}] needs one"
)
if item_lens[i] != dst_item_lens[j]:
return (
f"{state_type} item length mismatch for paired "
f"entries src[{i}]={item_lens[i]} "
f"dst[{j}]={dst_item_lens[j]}"
)
return None
_, _, layers_current_pp_stage = self.get_mla_kv_ptrs_with_pp(
src_data_ptrs, dst_data_ptrs, state_type
)
# Lens are parallel to pointers, so the same slice gives each entry's
# decode stride.
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
item_lens, dst_item_lens, state_type
)
if len(mapped_dst_lens) < layers_current_pp_stage:
return (
f"{state_type} decode published {len(mapped_dst_lens)} item "
f"lengths for this stage's {layers_current_pp_stage} entries"
)
for layer_id in range(layers_current_pp_stage):
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
return (
f"{state_type} item length mismatch for positional "
f"entry {layer_id}: prefill={item_lens[layer_id]} "
f"decode={mapped_dst_lens[layer_id]}"
)
return None
def _publish_peer_registration(
self, mooncake_session_id: str, decode_kv_args: KVArgsRegisterInfo
) -> Optional[str]:
"""Record this peer's state-layout verdict, then publish its metadata.
Publishing last stops a transfer from reading the new layout while the
previous registration's verdict still stands. Returns the rejection
reason, if any.
"""
fully_checked, mismatch = self._validate_peer_state_layout(decode_kv_args)
with self.session_lock:
self.failed_sessions.discard(mooncake_session_id)
self.session_failures.pop(mooncake_session_id, None)
self.state_layout_rejections.pop(mooncake_session_id, None)
self.state_strides_validated.discard(mooncake_session_id)
if mismatch is not None:
self.state_layout_rejections[mooncake_session_id] = mismatch
elif fully_checked:
self.state_strides_validated.add(mooncake_session_id)
self.decode_kv_args_table[mooncake_session_id] = decode_kv_args
return mismatch
def _validate_peer_state_layout(
self, registration_info: KVArgsRegisterInfo
) -> Tuple[bool, Optional[str]]:
"""Check every generic state component against one decode peer.
Neither side changes its entry sizes after registering, so one verdict
covers the whole session. Returns ``(fully_checked, mismatch_reason)``:
a reason rejects the peer, ``fully_checked`` retires the per-transfer
check.
"""
state_types = getattr(self.kv_args, "state_types", [])
fully_checked = True
for i, st in enumerate(state_types):
if not self._is_generic_kvcache_state_type(st):
# Mamba, DSA tail and MiniMax send paths check their own layout.
continue
dst_data_ptrs = (
registration_info.dst_state_data_ptrs[i]
if i < len(registration_info.dst_state_data_ptrs)
else []
)
dst_item_lens = (
registration_info.dst_state_item_lens[i]
if i < len(registration_info.dst_state_item_lens)
else []
)
if not dst_data_ptrs or not dst_item_lens:
# Nothing published for this component; let the send path report it.
fully_checked = False
continue
try:
mismatch = self._state_stride_mismatch(
src_data_ptrs=self.kv_args.state_data_ptrs[i],
dst_data_ptrs=dst_data_ptrs,
item_lens=self.kv_args.state_item_lens[i],
dst_item_lens=dst_item_lens,
state_type=st,
force_flat=(
st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED)
or get_memory().enable_unified_memory
),
src_layer_ids=(
self.kv_args.state_layer_ids[i]
if i < len(self.kv_args.state_layer_ids)
else []
),
dst_layer_ids=(
registration_info.dst_state_layer_ids[i]
if i < len(registration_info.dst_state_layer_ids)
else []
),
)
except Exception as e:
# Pairing can reject a layout on its own; never let that kill
# the bootstrap thread. Fall back to the send path's check.
logger.warning(
"Could not pre-check %s layout for session %s: %s",
st,
registration_info.mooncake_session_id,
e,
)
fully_checked = False
continue
if mismatch is not None:
return False, mismatch
return fully_checked, None
def _send_kvcache_generic(
self,
mooncake_session_id: str,
@@ -803,22 +991,41 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
layers_params = None
# Skipped once registration has cleared this peer's layout.
if (
dst_item_lens is not None
and mooncake_session_id not in self.state_strides_validated
):
mismatch = self._state_stride_mismatch(
src_data_ptrs=src_data_ptrs,
dst_data_ptrs=dst_data_ptrs,
item_lens=item_lens,
dst_item_lens=dst_item_lens,
state_type=state_type,
force_flat=force_flat,
src_layer_ids=src_layer_ids,
dst_layer_ids=dst_layer_ids,
)
if mismatch is not None:
assert bootstrap_room is not None
logger.error(mismatch)
self.conclude_failure(
bootstrap_room=bootstrap_room,
failure_reason=mismatch,
)
return -1
# Decode pp size should be equal to prefill pp size or 1
# Published layer IDs give exact pairing; plain-MHA peers publish none
# and keep positional slicing.
has_layer_ids = bool(src_layer_ids or dst_layer_ids)
# Unified SWA publishes one page-envelope region even on an MHA backend.
is_single_region_swa = (
state_type == StateType.SWA
and len(src_data_ptrs) == 1
and len(dst_data_ptrs) == 1
)
if (
self.is_mla_backend
or self.is_hybrid_mla_backend
or force_flat
or has_layer_ids
or is_single_region_swa
if _uses_flat_entry_layout(
mla_backend=self.is_mla_backend or self.is_hybrid_mla_backend,
force_flat=force_flat,
has_layer_ids=has_layer_ids,
state_type=state_type,
num_src_entries=len(src_data_ptrs),
num_dst_entries=len(dst_data_ptrs),
):
# Layer IDs map PP-local buffers to global decode entries.
# Registrations without them retain the existing PP mapping.
@@ -830,21 +1037,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
len(dst_data_ptrs),
allow_positional_fallback=self.pp_size == 1,
)
if dst_item_lens is not None:
for i, j in pairs:
if item_lens[i] != dst_item_lens[j]:
assert bootstrap_room is not None
failure_reason = (
f"{state_type} item length mismatch for paired "
f"entries src[{i}]={item_lens[i]} "
f"dst[{j}]={dst_item_lens[j]}"
)
logger.error(failure_reason)
self.conclude_failure(
bootstrap_room=bootstrap_room,
failure_reason=failure_reason,
)
return -1
layers_params = [
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
]
@@ -854,24 +1046,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_data_ptrs, dst_data_ptrs, state_type
)
)
if dst_item_lens is not None:
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
item_lens, dst_item_lens, state_type
)
for layer_id in range(layers_current_pp_stage):
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
assert bootstrap_room is not None
failure_reason = (
f"{state_type} item length mismatch for positional "
f"entry {layer_id}: prefill={item_lens[layer_id]} "
f"decode={mapped_dst_lens[layer_id]}"
)
logger.error(failure_reason)
self.conclude_failure(
bootstrap_room=bootstrap_room,
failure_reason=failure_reason,
)
return -1
layers_params = [
(
src_kv_ptrs[layer_id],
@@ -2199,10 +2373,17 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if not req.is_dummy:
# Early exit if the request has failed
with self.session_lock:
if req.mooncake_session_id in self.failed_sessions:
layout_rejection = self.state_layout_rejections.get(
req.mooncake_session_id
)
if (
layout_rejection is not None
or req.mooncake_session_id in self.failed_sessions
):
self.conclude_failure(
bootstrap_room=kv_chunk.room,
failure_reason=(
failure_reason=layout_rejection
or (
"Decode instance could be dead, remote "
f"mooncake session {req.mooncake_session_id} "
"is not alive"
@@ -2594,15 +2775,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self._init_dcp_pack_buffers_once(
decode_kv_args.dst_dcp_size, include_draft=True
)
self.decode_kv_args_table[mooncake_session_id] = decode_kv_args
with self.session_lock:
if mooncake_session_id in self.failed_sessions:
self.failed_sessions.remove(mooncake_session_id)
if mooncake_session_id in self.session_failures:
del self.session_failures[mooncake_session_id]
logger.debug(
f"Register KVArgs from {mooncake_session_id} successfully"
layout_mismatch = self._publish_peer_registration(
mooncake_session_id, decode_kv_args
)
if layout_mismatch is not None:
logger.error(
"Rejecting state transfers to decode session %s: %s",
mooncake_session_id,
layout_mismatch,
)
else:
logger.debug(
f"Register KVArgs from {mooncake_session_id} successfully"
)
continue
else:
required_dst_info_num = int(waiting_req_bytes[7].decode("ascii"))
@@ -156,6 +156,8 @@ class TestDisaggregationWire(unittest.TestCase):
manager.failure_lock = threading.Lock()
manager.failed_sessions = set()
manager.session_failures = defaultdict(int)
manager.state_layout_rejections = {}
manager.state_strides_validated = set()
manager.failure_records = {}
manager._staging_outstanding = defaultdict(int)
manager.request_status = {
@@ -280,6 +282,256 @@ class TestDisaggregationWire(unittest.TestCase):
self.assertFalse(manager.transfer_infos)
self.assertFalse(manager.req_to_decode_prefix_len)
def test_mooncake_short_decode_item_lens_is_a_verdict_not_an_index_error(self):
"""A decode publishing fewer item lengths than entries used to raise
IndexError out of the stride check and take the transfer thread down;
it must come back as a rejection reason on both pairing paths."""
for layer_ids in ([], [3, 4]):
with self.subTest(layer_ids=layer_ids):
manager = object.__new__(MooncakeKVManager)
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.pp_size = 1
manager.kv_args = SimpleNamespace(
mla_compression_ratios=None, prefill_start_layer=0
)
reason = manager._state_stride_mismatch(
src_data_ptrs=[0x1000, 0x1100],
dst_data_ptrs=[0x2000, 0x2100],
item_lens=[32, 32],
dst_item_lens=[32],
state_type=StateType.SWA,
src_layer_ids=layer_ids,
dst_layer_ids=layer_ids,
)
self.assertIsNotNone(reason)
self.assertIn("item lengths", reason)
def test_mooncake_validates_peer_state_layout_at_registration(self):
"""The verdict is reachable at registration, not just at transfer time."""
for layer_ids in ([], [7]):
for dst_item_len in (32, 64):
with self.subTest(layer_ids=layer_ids, dst_item_len=dst_item_len):
manager = object.__new__(MooncakeKVManager)
manager.kv_args = SimpleNamespace(
state_types=[StateType.SWA, StateType.MAMBA],
state_data_ptrs=[[0x1000], [0x3000]],
state_item_lens=[[32], [16]],
state_layer_ids=[layer_ids, layer_ids],
)
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.pp_size = 1
registration_info = SimpleNamespace(
mooncake_session_id="decode",
dst_state_data_ptrs=[[0x2000], [0x4000]],
# Mamba disagrees too, but ships on its own path.
dst_state_item_lens=[[dst_item_len], [999]],
dst_state_layer_ids=[layer_ids, layer_ids],
)
with get_context().override_server_args(
enable_unified_memory=False
):
fully_checked, reason = manager._validate_peer_state_layout(
registration_info
)
if dst_item_len == 32:
self.assertTrue(fully_checked)
self.assertIsNone(reason)
else:
self.assertFalse(fully_checked)
self.assertEqual(
reason,
f"{StateType.SWA} item length mismatch for "
+ (
"paired entries src[0]=32 dst[0]=64"
if layer_ids
else "positional entry 0: prefill=32 decode=64"
),
)
def test_mooncake_registration_replaces_the_previous_layout_verdict(self):
"""Re-registering re-derives the verdict, and publishes the layout last."""
manager = object.__new__(MooncakeKVManager)
manager.kv_args = SimpleNamespace(
state_types=[StateType.SWA],
state_data_ptrs=[[0x1000]],
state_item_lens=[[32]],
state_layer_ids=[[]],
)
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.pp_size = 1
manager.session_lock = threading.Lock()
manager.failed_sessions = {"decode"}
manager.session_failures = defaultdict(int, {"decode": 2})
manager.state_layout_rejections = {}
manager.state_strides_validated = set()
manager.decode_kv_args_table = {}
def register(dst_item_len):
info = SimpleNamespace(
mooncake_session_id="decode",
dst_state_data_ptrs=[[0x2000]],
dst_state_item_lens=[[dst_item_len]],
dst_state_layer_ids=[[]],
)
with get_context().override_server_args(enable_unified_memory=False):
return info, manager._publish_peer_registration("decode", info)
good, mismatch = register(32)
self.assertIsNone(mismatch)
self.assertEqual(manager.state_strides_validated, {"decode"})
self.assertFalse(manager.state_layout_rejections)
self.assertIs(manager.decode_kv_args_table["decode"], good)
# A transport blacklist is cleared by registering, as before.
self.assertFalse(manager.failed_sessions)
self.assertFalse(manager.session_failures)
bad, mismatch = register(64)
self.assertIn("item length mismatch", mismatch)
# The stale clearance is gone before the new layout is reachable.
self.assertFalse(manager.state_strides_validated)
self.assertEqual(manager.state_layout_rejections, {"decode": mismatch})
self.assertIs(manager.decode_kv_args_table["decode"], bad)
def test_mooncake_unpublished_state_metadata_keeps_the_transfer_check(self):
"""An unpublished component cannot be judged; keep the transfer check."""
manager = object.__new__(MooncakeKVManager)
manager.kv_args = SimpleNamespace(
state_types=[StateType.SWA],
state_data_ptrs=[[0x1000]],
state_item_lens=[[32]],
state_layer_ids=[[]],
)
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.pp_size = 1
registration_info = SimpleNamespace(
mooncake_session_id="decode",
dst_state_data_ptrs=[[0x2000]],
dst_state_item_lens=[],
dst_state_layer_ids=[[]],
)
with get_context().override_server_args(enable_unified_memory=False):
fully_checked, reason = manager._validate_peer_state_layout(
registration_info
)
self.assertFalse(fully_checked)
self.assertIsNone(reason)
def test_mooncake_validated_session_skips_the_per_transfer_check(self):
"""A cleared session pays nothing per request.
Driven with sizes that would be rejected, so the transfer going through
proves the check was skipped rather than merely passing.
"""
manager = object.__new__(MooncakeKVManager)
manager.kv_args = SimpleNamespace(kv_data_ptrs=[])
manager.is_mla_backend = True
manager.pp_size = 1
manager.enable_custom_mem_pool = False
manager.max_transfer_batch_indices = 0
manager.state_strides_validated = {"decode"}
manager._transfer_data = Mock(return_value=0)
manager._state_stride_mismatch = Mock(
side_effect=AssertionError("must not re-check a validated session")
)
rc = manager._send_kvcache_generic(
mooncake_session_id="decode",
executor=None,
src_data_ptrs=[0x1000],
dst_data_ptrs=[0x2000],
item_lens=[32],
dst_item_lens=[64],
prefill_data_indices=np.array([2], dtype=np.int32),
dst_data_indices=np.array([5], dtype=np.int32),
state_type=StateType.SWA,
bootstrap_room=42,
)
self.assertEqual(rc, 0)
manager._state_stride_mismatch.assert_not_called()
manager._transfer_data.assert_called_once_with(
"decode", [(0x1000 + 2 * 32, 0x2000 + 5 * 32, 32)]
)
def test_mooncake_layout_rejected_session_fails_requests_without_writing(self):
"""A rejected peer fails every request and stays off the probe-clearable
transport blacklist."""
manager = object.__new__(MooncakeKVManager)
manager.kv_args = SimpleNamespace(kv_data_ptrs=[], state_types=[StateType.SWA])
manager.is_mla_backend = True
manager.is_hybrid_mla_backend = False
manager.attn_tp_size = manager.attn_cp_size = manager.pp_size = 1
manager.attn_tp_rank = manager.attn_cp_rank = manager.pp_rank = 0
manager.enable_custom_mem_pool = False
manager.max_transfer_batch_indices = 0
manager.enable_trace = manager.enable_staging = False
manager.bootstrap_port = 1234
manager.session_lock = threading.Lock()
manager.failure_lock = threading.Lock()
manager.failed_sessions = set()
manager.session_failures = defaultdict(int)
manager.failure_records = {}
manager._staging_outstanding = defaultdict(int)
manager.request_status = {42: KVPoll.Transferring}
manager.req_to_decode_prefix_len = {42: 0}
reason = f"{StateType.SWA} item length mismatch for positional entry 0"
manager.state_layout_rejections = {"decode:42": reason}
manager.state_strides_validated = set()
req = TransferInfo(
room=42,
endpoint="127.0.0.1",
dst_port=1234,
mooncake_session_id="decode:42",
dst_kv_indices=np.array([], dtype=np.int32),
dst_aux_index=5,
dst_state_indices=[[5]],
required_dst_info_num=1,
is_dummy=False,
)
manager.transfer_infos = {42: {"decode:42": req}}
manager.decode_kv_args_table = {}
manager._send_multipart_locked = Mock()
manager._transfer_data = Mock(return_value=0)
queue = FastQueue()
queue.put(
TransferKVChunk(
room=42,
prefill_kv_indices=np.array([], dtype=np.int32),
index_slice=slice(0, 0),
is_last_chunk=True,
prefill_aux_index=2,
state_indices=[[2]],
)
)
queue.put(None)
manager.transfer_worker(queue, executor=None)
self.assertEqual(manager.request_status[42], KVPoll.Failed)
self.assertEqual(manager.failure_records[42], reason)
manager._transfer_data.assert_not_called()
# Not a transport fault, so the probe loop must not clear it.
self.assertFalse(manager.failed_sessions)
self.assertFalse(manager.session_failures)
self.assertEqual(manager.state_layout_rejections, {"decode:42": reason})
def test_mooncake_probe_recovery_keeps_a_layout_rejection(self):
"""A probe clears the transport blacklist but not a layout rejection."""
manager = object.__new__(MooncakeKVManager)
manager.session_lock = threading.Lock()
manager.failed_sessions = {"decode"}
manager.session_failures = defaultdict(int, {"decode": 1})
reason = f"{StateType.SWA} item length mismatch for positional entry 0"
manager.state_layout_rejections = {"decode": reason}
manager.engine = SimpleNamespace(send_probe=Mock(return_value=0))
manager._run_one_probe_pass()
self.assertFalse(manager.failed_sessions)
self.assertFalse(manager.session_failures)
self.assertEqual(manager.state_layout_rejections, {"decode": reason})
def test_mooncake_state_stride_matches_pp_slice(self):
cases = [
([], [64, 128], [32, 64, 128, 256], [1, 2]),
@@ -301,6 +553,7 @@ class TestDisaggregationWire(unittest.TestCase):
)
manager.is_mla_backend = True
manager.pp_size = 2
manager.state_strides_validated = set()
manager.enable_custom_mem_pool = False
manager.max_transfer_batch_indices = 0
manager._transfer_data = Mock(return_value=0)
@@ -663,6 +663,8 @@ class TestAllocationIntegration(CustomTestCase):
mgr._staging_outstanding = defaultdict(int)
mgr.session_lock = threading.Lock()
mgr.failed_sessions = set()
mgr.state_layout_rejections = {}
mgr.state_strides_validated = set()
mgr.enable_deferred_decode_kv_release = True
mgr.decode_kv_args_table = {
"session": SimpleNamespace(requires_dcp_relayout=False, dst_aux_ptrs=[])