[PD] Validate decode state layout once at registration (#42051)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
bc5a055dd7
commit
148da8e77d
@@ -8,7 +8,7 @@ import struct
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from typing import List, Optional, Set, Tuple, Union
|
||||
from typing import Dict, List, Optional, Set, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -216,6 +216,27 @@ class KVArgsRegisterInfo:
|
||||
)
|
||||
|
||||
|
||||
def _uses_flat_entry_layout(
|
||||
*,
|
||||
mla_backend: bool,
|
||||
force_flat: bool,
|
||||
has_layer_ids: bool,
|
||||
state_type: Optional[StateType],
|
||||
num_src_entries: int,
|
||||
num_dst_entries: int,
|
||||
) -> bool:
|
||||
"""One region per transfer entry, as opposed to plain MHA's K-then-V split.
|
||||
|
||||
Shared by the transfer and the registration-time stride check so both
|
||||
judge the same entries. Unified SWA publishes one page-envelope region
|
||||
even on an MHA backend.
|
||||
"""
|
||||
is_single_region_swa = (
|
||||
state_type == StateType.SWA and num_src_entries == 1 and num_dst_entries == 1
|
||||
)
|
||||
return mla_backend or force_flat or has_layer_ids or is_single_region_swa
|
||||
|
||||
|
||||
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
AUX_DATA_HEADER = b"AUX_DATA"
|
||||
# Implements teardown() below, so runtime PD role switching is supported.
|
||||
@@ -250,6 +271,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
self.session_failures = defaultdict(int)
|
||||
self.failed_sessions = set()
|
||||
# Separate from failed_sessions, which _run_one_probe_pass clears
|
||||
# on a successful probe: an incompatible decode is still reachable,
|
||||
# so only a re-registration may lift this.
|
||||
self.state_layout_rejections: Dict[str, str] = {}
|
||||
# Peers already cleared at registration; skip the per-transfer check.
|
||||
self.state_strides_validated: Set[str] = set()
|
||||
self.session_lock = threading.Lock()
|
||||
self.start_prefill_thread()
|
||||
# Determine the number of threads to use for kv sender
|
||||
@@ -762,6 +789,167 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
|
||||
)
|
||||
|
||||
def _state_stride_mismatch(
|
||||
self,
|
||||
*,
|
||||
src_data_ptrs: list[int],
|
||||
dst_data_ptrs: list[int],
|
||||
item_lens: list[int],
|
||||
dst_item_lens: list[int],
|
||||
state_type: Optional[StateType] = None,
|
||||
force_flat: bool = False,
|
||||
src_layer_ids: Optional[List[int]] = None,
|
||||
dst_layer_ids: Optional[List[int]] = None,
|
||||
) -> Optional[str]:
|
||||
"""First entry whose decode stride differs from prefill's, else None.
|
||||
|
||||
The destination is addressed as ``dst_ptr + dst_index * src_item_len``,
|
||||
so mapped entries must agree on the stride or the write lands in the
|
||||
wrong slot.
|
||||
"""
|
||||
has_layer_ids = bool(src_layer_ids or dst_layer_ids)
|
||||
if not _uses_flat_entry_layout(
|
||||
mla_backend=self.is_mla_backend or self.is_hybrid_mla_backend,
|
||||
force_flat=force_flat,
|
||||
has_layer_ids=has_layer_ids,
|
||||
state_type=state_type,
|
||||
num_src_entries=len(src_data_ptrs),
|
||||
num_dst_entries=len(dst_data_ptrs),
|
||||
):
|
||||
# The MHA branch splits the entries into K/V and is not covered.
|
||||
return None
|
||||
if has_layer_ids:
|
||||
pairs = build_transfer_entry_pairs(
|
||||
src_layer_ids or [],
|
||||
dst_layer_ids or [],
|
||||
len(src_data_ptrs),
|
||||
len(dst_data_ptrs),
|
||||
allow_positional_fallback=self.pp_size == 1,
|
||||
)
|
||||
for i, j in pairs:
|
||||
if j >= len(dst_item_lens):
|
||||
return (
|
||||
f"{state_type} decode published {len(dst_item_lens)} item "
|
||||
f"lengths but paired entry dst[{j}] needs one"
|
||||
)
|
||||
if item_lens[i] != dst_item_lens[j]:
|
||||
return (
|
||||
f"{state_type} item length mismatch for paired "
|
||||
f"entries src[{i}]={item_lens[i]} "
|
||||
f"dst[{j}]={dst_item_lens[j]}"
|
||||
)
|
||||
return None
|
||||
_, _, layers_current_pp_stage = self.get_mla_kv_ptrs_with_pp(
|
||||
src_data_ptrs, dst_data_ptrs, state_type
|
||||
)
|
||||
# Lens are parallel to pointers, so the same slice gives each entry's
|
||||
# decode stride.
|
||||
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
|
||||
item_lens, dst_item_lens, state_type
|
||||
)
|
||||
if len(mapped_dst_lens) < layers_current_pp_stage:
|
||||
return (
|
||||
f"{state_type} decode published {len(mapped_dst_lens)} item "
|
||||
f"lengths for this stage's {layers_current_pp_stage} entries"
|
||||
)
|
||||
for layer_id in range(layers_current_pp_stage):
|
||||
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
|
||||
return (
|
||||
f"{state_type} item length mismatch for positional "
|
||||
f"entry {layer_id}: prefill={item_lens[layer_id]} "
|
||||
f"decode={mapped_dst_lens[layer_id]}"
|
||||
)
|
||||
return None
|
||||
|
||||
def _publish_peer_registration(
|
||||
self, mooncake_session_id: str, decode_kv_args: KVArgsRegisterInfo
|
||||
) -> Optional[str]:
|
||||
"""Record this peer's state-layout verdict, then publish its metadata.
|
||||
|
||||
Publishing last stops a transfer from reading the new layout while the
|
||||
previous registration's verdict still stands. Returns the rejection
|
||||
reason, if any.
|
||||
"""
|
||||
fully_checked, mismatch = self._validate_peer_state_layout(decode_kv_args)
|
||||
with self.session_lock:
|
||||
self.failed_sessions.discard(mooncake_session_id)
|
||||
self.session_failures.pop(mooncake_session_id, None)
|
||||
self.state_layout_rejections.pop(mooncake_session_id, None)
|
||||
self.state_strides_validated.discard(mooncake_session_id)
|
||||
if mismatch is not None:
|
||||
self.state_layout_rejections[mooncake_session_id] = mismatch
|
||||
elif fully_checked:
|
||||
self.state_strides_validated.add(mooncake_session_id)
|
||||
self.decode_kv_args_table[mooncake_session_id] = decode_kv_args
|
||||
return mismatch
|
||||
|
||||
def _validate_peer_state_layout(
|
||||
self, registration_info: KVArgsRegisterInfo
|
||||
) -> Tuple[bool, Optional[str]]:
|
||||
"""Check every generic state component against one decode peer.
|
||||
|
||||
Neither side changes its entry sizes after registering, so one verdict
|
||||
covers the whole session. Returns ``(fully_checked, mismatch_reason)``:
|
||||
a reason rejects the peer, ``fully_checked`` retires the per-transfer
|
||||
check.
|
||||
"""
|
||||
state_types = getattr(self.kv_args, "state_types", [])
|
||||
fully_checked = True
|
||||
for i, st in enumerate(state_types):
|
||||
if not self._is_generic_kvcache_state_type(st):
|
||||
# Mamba, DSA tail and MiniMax send paths check their own layout.
|
||||
continue
|
||||
dst_data_ptrs = (
|
||||
registration_info.dst_state_data_ptrs[i]
|
||||
if i < len(registration_info.dst_state_data_ptrs)
|
||||
else []
|
||||
)
|
||||
dst_item_lens = (
|
||||
registration_info.dst_state_item_lens[i]
|
||||
if i < len(registration_info.dst_state_item_lens)
|
||||
else []
|
||||
)
|
||||
if not dst_data_ptrs or not dst_item_lens:
|
||||
# Nothing published for this component; let the send path report it.
|
||||
fully_checked = False
|
||||
continue
|
||||
try:
|
||||
mismatch = self._state_stride_mismatch(
|
||||
src_data_ptrs=self.kv_args.state_data_ptrs[i],
|
||||
dst_data_ptrs=dst_data_ptrs,
|
||||
item_lens=self.kv_args.state_item_lens[i],
|
||||
dst_item_lens=dst_item_lens,
|
||||
state_type=st,
|
||||
force_flat=(
|
||||
st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED)
|
||||
or get_memory().enable_unified_memory
|
||||
),
|
||||
src_layer_ids=(
|
||||
self.kv_args.state_layer_ids[i]
|
||||
if i < len(self.kv_args.state_layer_ids)
|
||||
else []
|
||||
),
|
||||
dst_layer_ids=(
|
||||
registration_info.dst_state_layer_ids[i]
|
||||
if i < len(registration_info.dst_state_layer_ids)
|
||||
else []
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
# Pairing can reject a layout on its own; never let that kill
|
||||
# the bootstrap thread. Fall back to the send path's check.
|
||||
logger.warning(
|
||||
"Could not pre-check %s layout for session %s: %s",
|
||||
st,
|
||||
registration_info.mooncake_session_id,
|
||||
e,
|
||||
)
|
||||
fully_checked = False
|
||||
continue
|
||||
if mismatch is not None:
|
||||
return False, mismatch
|
||||
return fully_checked, None
|
||||
|
||||
def _send_kvcache_generic(
|
||||
self,
|
||||
mooncake_session_id: str,
|
||||
@@ -803,22 +991,41 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
layers_params = None
|
||||
|
||||
# Skipped once registration has cleared this peer's layout.
|
||||
if (
|
||||
dst_item_lens is not None
|
||||
and mooncake_session_id not in self.state_strides_validated
|
||||
):
|
||||
mismatch = self._state_stride_mismatch(
|
||||
src_data_ptrs=src_data_ptrs,
|
||||
dst_data_ptrs=dst_data_ptrs,
|
||||
item_lens=item_lens,
|
||||
dst_item_lens=dst_item_lens,
|
||||
state_type=state_type,
|
||||
force_flat=force_flat,
|
||||
src_layer_ids=src_layer_ids,
|
||||
dst_layer_ids=dst_layer_ids,
|
||||
)
|
||||
if mismatch is not None:
|
||||
assert bootstrap_room is not None
|
||||
logger.error(mismatch)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=mismatch,
|
||||
)
|
||||
return -1
|
||||
|
||||
# Decode pp size should be equal to prefill pp size or 1
|
||||
# Published layer IDs give exact pairing; plain-MHA peers publish none
|
||||
# and keep positional slicing.
|
||||
has_layer_ids = bool(src_layer_ids or dst_layer_ids)
|
||||
# Unified SWA publishes one page-envelope region even on an MHA backend.
|
||||
is_single_region_swa = (
|
||||
state_type == StateType.SWA
|
||||
and len(src_data_ptrs) == 1
|
||||
and len(dst_data_ptrs) == 1
|
||||
)
|
||||
if (
|
||||
self.is_mla_backend
|
||||
or self.is_hybrid_mla_backend
|
||||
or force_flat
|
||||
or has_layer_ids
|
||||
or is_single_region_swa
|
||||
if _uses_flat_entry_layout(
|
||||
mla_backend=self.is_mla_backend or self.is_hybrid_mla_backend,
|
||||
force_flat=force_flat,
|
||||
has_layer_ids=has_layer_ids,
|
||||
state_type=state_type,
|
||||
num_src_entries=len(src_data_ptrs),
|
||||
num_dst_entries=len(dst_data_ptrs),
|
||||
):
|
||||
# Layer IDs map PP-local buffers to global decode entries.
|
||||
# Registrations without them retain the existing PP mapping.
|
||||
@@ -830,21 +1037,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
len(dst_data_ptrs),
|
||||
allow_positional_fallback=self.pp_size == 1,
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
for i, j in pairs:
|
||||
if item_lens[i] != dst_item_lens[j]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for paired "
|
||||
f"entries src[{i}]={item_lens[i]} "
|
||||
f"dst[{j}]={dst_item_lens[j]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
|
||||
]
|
||||
@@ -854,24 +1046,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_data_ptrs, dst_data_ptrs, state_type
|
||||
)
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
|
||||
item_lens, dst_item_lens, state_type
|
||||
)
|
||||
for layer_id in range(layers_current_pp_stage):
|
||||
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for positional "
|
||||
f"entry {layer_id}: prefill={item_lens[layer_id]} "
|
||||
f"decode={mapped_dst_lens[layer_id]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(
|
||||
src_kv_ptrs[layer_id],
|
||||
@@ -2199,10 +2373,17 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if not req.is_dummy:
|
||||
# Early exit if the request has failed
|
||||
with self.session_lock:
|
||||
if req.mooncake_session_id in self.failed_sessions:
|
||||
layout_rejection = self.state_layout_rejections.get(
|
||||
req.mooncake_session_id
|
||||
)
|
||||
if (
|
||||
layout_rejection is not None
|
||||
or req.mooncake_session_id in self.failed_sessions
|
||||
):
|
||||
self.conclude_failure(
|
||||
bootstrap_room=kv_chunk.room,
|
||||
failure_reason=(
|
||||
failure_reason=layout_rejection
|
||||
or (
|
||||
"Decode instance could be dead, remote "
|
||||
f"mooncake session {req.mooncake_session_id} "
|
||||
"is not alive"
|
||||
@@ -2594,15 +2775,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._init_dcp_pack_buffers_once(
|
||||
decode_kv_args.dst_dcp_size, include_draft=True
|
||||
)
|
||||
self.decode_kv_args_table[mooncake_session_id] = decode_kv_args
|
||||
with self.session_lock:
|
||||
if mooncake_session_id in self.failed_sessions:
|
||||
self.failed_sessions.remove(mooncake_session_id)
|
||||
if mooncake_session_id in self.session_failures:
|
||||
del self.session_failures[mooncake_session_id]
|
||||
logger.debug(
|
||||
f"Register KVArgs from {mooncake_session_id} successfully"
|
||||
layout_mismatch = self._publish_peer_registration(
|
||||
mooncake_session_id, decode_kv_args
|
||||
)
|
||||
if layout_mismatch is not None:
|
||||
logger.error(
|
||||
"Rejecting state transfers to decode session %s: %s",
|
||||
mooncake_session_id,
|
||||
layout_mismatch,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
f"Register KVArgs from {mooncake_session_id} successfully"
|
||||
)
|
||||
continue
|
||||
else:
|
||||
required_dst_info_num = int(waiting_req_bytes[7].decode("ascii"))
|
||||
|
||||
@@ -156,6 +156,8 @@ class TestDisaggregationWire(unittest.TestCase):
|
||||
manager.failure_lock = threading.Lock()
|
||||
manager.failed_sessions = set()
|
||||
manager.session_failures = defaultdict(int)
|
||||
manager.state_layout_rejections = {}
|
||||
manager.state_strides_validated = set()
|
||||
manager.failure_records = {}
|
||||
manager._staging_outstanding = defaultdict(int)
|
||||
manager.request_status = {
|
||||
@@ -280,6 +282,256 @@ class TestDisaggregationWire(unittest.TestCase):
|
||||
self.assertFalse(manager.transfer_infos)
|
||||
self.assertFalse(manager.req_to_decode_prefix_len)
|
||||
|
||||
def test_mooncake_short_decode_item_lens_is_a_verdict_not_an_index_error(self):
|
||||
"""A decode publishing fewer item lengths than entries used to raise
|
||||
IndexError out of the stride check and take the transfer thread down;
|
||||
it must come back as a rejection reason on both pairing paths."""
|
||||
for layer_ids in ([], [3, 4]):
|
||||
with self.subTest(layer_ids=layer_ids):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.is_mla_backend = True
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.pp_size = 1
|
||||
manager.kv_args = SimpleNamespace(
|
||||
mla_compression_ratios=None, prefill_start_layer=0
|
||||
)
|
||||
reason = manager._state_stride_mismatch(
|
||||
src_data_ptrs=[0x1000, 0x1100],
|
||||
dst_data_ptrs=[0x2000, 0x2100],
|
||||
item_lens=[32, 32],
|
||||
dst_item_lens=[32],
|
||||
state_type=StateType.SWA,
|
||||
src_layer_ids=layer_ids,
|
||||
dst_layer_ids=layer_ids,
|
||||
)
|
||||
self.assertIsNotNone(reason)
|
||||
self.assertIn("item lengths", reason)
|
||||
|
||||
def test_mooncake_validates_peer_state_layout_at_registration(self):
|
||||
"""The verdict is reachable at registration, not just at transfer time."""
|
||||
for layer_ids in ([], [7]):
|
||||
for dst_item_len in (32, 64):
|
||||
with self.subTest(layer_ids=layer_ids, dst_item_len=dst_item_len):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.kv_args = SimpleNamespace(
|
||||
state_types=[StateType.SWA, StateType.MAMBA],
|
||||
state_data_ptrs=[[0x1000], [0x3000]],
|
||||
state_item_lens=[[32], [16]],
|
||||
state_layer_ids=[layer_ids, layer_ids],
|
||||
)
|
||||
manager.is_mla_backend = True
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.pp_size = 1
|
||||
registration_info = SimpleNamespace(
|
||||
mooncake_session_id="decode",
|
||||
dst_state_data_ptrs=[[0x2000], [0x4000]],
|
||||
# Mamba disagrees too, but ships on its own path.
|
||||
dst_state_item_lens=[[dst_item_len], [999]],
|
||||
dst_state_layer_ids=[layer_ids, layer_ids],
|
||||
)
|
||||
with get_context().override_server_args(
|
||||
enable_unified_memory=False
|
||||
):
|
||||
fully_checked, reason = manager._validate_peer_state_layout(
|
||||
registration_info
|
||||
)
|
||||
if dst_item_len == 32:
|
||||
self.assertTrue(fully_checked)
|
||||
self.assertIsNone(reason)
|
||||
else:
|
||||
self.assertFalse(fully_checked)
|
||||
self.assertEqual(
|
||||
reason,
|
||||
f"{StateType.SWA} item length mismatch for "
|
||||
+ (
|
||||
"paired entries src[0]=32 dst[0]=64"
|
||||
if layer_ids
|
||||
else "positional entry 0: prefill=32 decode=64"
|
||||
),
|
||||
)
|
||||
|
||||
def test_mooncake_registration_replaces_the_previous_layout_verdict(self):
|
||||
"""Re-registering re-derives the verdict, and publishes the layout last."""
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.kv_args = SimpleNamespace(
|
||||
state_types=[StateType.SWA],
|
||||
state_data_ptrs=[[0x1000]],
|
||||
state_item_lens=[[32]],
|
||||
state_layer_ids=[[]],
|
||||
)
|
||||
manager.is_mla_backend = True
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.pp_size = 1
|
||||
manager.session_lock = threading.Lock()
|
||||
manager.failed_sessions = {"decode"}
|
||||
manager.session_failures = defaultdict(int, {"decode": 2})
|
||||
manager.state_layout_rejections = {}
|
||||
manager.state_strides_validated = set()
|
||||
manager.decode_kv_args_table = {}
|
||||
|
||||
def register(dst_item_len):
|
||||
info = SimpleNamespace(
|
||||
mooncake_session_id="decode",
|
||||
dst_state_data_ptrs=[[0x2000]],
|
||||
dst_state_item_lens=[[dst_item_len]],
|
||||
dst_state_layer_ids=[[]],
|
||||
)
|
||||
with get_context().override_server_args(enable_unified_memory=False):
|
||||
return info, manager._publish_peer_registration("decode", info)
|
||||
|
||||
good, mismatch = register(32)
|
||||
self.assertIsNone(mismatch)
|
||||
self.assertEqual(manager.state_strides_validated, {"decode"})
|
||||
self.assertFalse(manager.state_layout_rejections)
|
||||
self.assertIs(manager.decode_kv_args_table["decode"], good)
|
||||
# A transport blacklist is cleared by registering, as before.
|
||||
self.assertFalse(manager.failed_sessions)
|
||||
self.assertFalse(manager.session_failures)
|
||||
|
||||
bad, mismatch = register(64)
|
||||
self.assertIn("item length mismatch", mismatch)
|
||||
# The stale clearance is gone before the new layout is reachable.
|
||||
self.assertFalse(manager.state_strides_validated)
|
||||
self.assertEqual(manager.state_layout_rejections, {"decode": mismatch})
|
||||
self.assertIs(manager.decode_kv_args_table["decode"], bad)
|
||||
|
||||
def test_mooncake_unpublished_state_metadata_keeps_the_transfer_check(self):
|
||||
"""An unpublished component cannot be judged; keep the transfer check."""
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.kv_args = SimpleNamespace(
|
||||
state_types=[StateType.SWA],
|
||||
state_data_ptrs=[[0x1000]],
|
||||
state_item_lens=[[32]],
|
||||
state_layer_ids=[[]],
|
||||
)
|
||||
manager.is_mla_backend = True
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.pp_size = 1
|
||||
registration_info = SimpleNamespace(
|
||||
mooncake_session_id="decode",
|
||||
dst_state_data_ptrs=[[0x2000]],
|
||||
dst_state_item_lens=[],
|
||||
dst_state_layer_ids=[[]],
|
||||
)
|
||||
with get_context().override_server_args(enable_unified_memory=False):
|
||||
fully_checked, reason = manager._validate_peer_state_layout(
|
||||
registration_info
|
||||
)
|
||||
self.assertFalse(fully_checked)
|
||||
self.assertIsNone(reason)
|
||||
|
||||
def test_mooncake_validated_session_skips_the_per_transfer_check(self):
|
||||
"""A cleared session pays nothing per request.
|
||||
|
||||
Driven with sizes that would be rejected, so the transfer going through
|
||||
proves the check was skipped rather than merely passing.
|
||||
"""
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.kv_args = SimpleNamespace(kv_data_ptrs=[])
|
||||
manager.is_mla_backend = True
|
||||
manager.pp_size = 1
|
||||
manager.enable_custom_mem_pool = False
|
||||
manager.max_transfer_batch_indices = 0
|
||||
manager.state_strides_validated = {"decode"}
|
||||
manager._transfer_data = Mock(return_value=0)
|
||||
manager._state_stride_mismatch = Mock(
|
||||
side_effect=AssertionError("must not re-check a validated session")
|
||||
)
|
||||
rc = manager._send_kvcache_generic(
|
||||
mooncake_session_id="decode",
|
||||
executor=None,
|
||||
src_data_ptrs=[0x1000],
|
||||
dst_data_ptrs=[0x2000],
|
||||
item_lens=[32],
|
||||
dst_item_lens=[64],
|
||||
prefill_data_indices=np.array([2], dtype=np.int32),
|
||||
dst_data_indices=np.array([5], dtype=np.int32),
|
||||
state_type=StateType.SWA,
|
||||
bootstrap_room=42,
|
||||
)
|
||||
self.assertEqual(rc, 0)
|
||||
manager._state_stride_mismatch.assert_not_called()
|
||||
manager._transfer_data.assert_called_once_with(
|
||||
"decode", [(0x1000 + 2 * 32, 0x2000 + 5 * 32, 32)]
|
||||
)
|
||||
|
||||
def test_mooncake_layout_rejected_session_fails_requests_without_writing(self):
|
||||
"""A rejected peer fails every request and stays off the probe-clearable
|
||||
transport blacklist."""
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.kv_args = SimpleNamespace(kv_data_ptrs=[], state_types=[StateType.SWA])
|
||||
manager.is_mla_backend = True
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.attn_tp_size = manager.attn_cp_size = manager.pp_size = 1
|
||||
manager.attn_tp_rank = manager.attn_cp_rank = manager.pp_rank = 0
|
||||
manager.enable_custom_mem_pool = False
|
||||
manager.max_transfer_batch_indices = 0
|
||||
manager.enable_trace = manager.enable_staging = False
|
||||
manager.bootstrap_port = 1234
|
||||
manager.session_lock = threading.Lock()
|
||||
manager.failure_lock = threading.Lock()
|
||||
manager.failed_sessions = set()
|
||||
manager.session_failures = defaultdict(int)
|
||||
manager.failure_records = {}
|
||||
manager._staging_outstanding = defaultdict(int)
|
||||
manager.request_status = {42: KVPoll.Transferring}
|
||||
manager.req_to_decode_prefix_len = {42: 0}
|
||||
reason = f"{StateType.SWA} item length mismatch for positional entry 0"
|
||||
manager.state_layout_rejections = {"decode:42": reason}
|
||||
manager.state_strides_validated = set()
|
||||
req = TransferInfo(
|
||||
room=42,
|
||||
endpoint="127.0.0.1",
|
||||
dst_port=1234,
|
||||
mooncake_session_id="decode:42",
|
||||
dst_kv_indices=np.array([], dtype=np.int32),
|
||||
dst_aux_index=5,
|
||||
dst_state_indices=[[5]],
|
||||
required_dst_info_num=1,
|
||||
is_dummy=False,
|
||||
)
|
||||
manager.transfer_infos = {42: {"decode:42": req}}
|
||||
manager.decode_kv_args_table = {}
|
||||
manager._send_multipart_locked = Mock()
|
||||
manager._transfer_data = Mock(return_value=0)
|
||||
queue = FastQueue()
|
||||
queue.put(
|
||||
TransferKVChunk(
|
||||
room=42,
|
||||
prefill_kv_indices=np.array([], dtype=np.int32),
|
||||
index_slice=slice(0, 0),
|
||||
is_last_chunk=True,
|
||||
prefill_aux_index=2,
|
||||
state_indices=[[2]],
|
||||
)
|
||||
)
|
||||
queue.put(None)
|
||||
manager.transfer_worker(queue, executor=None)
|
||||
|
||||
self.assertEqual(manager.request_status[42], KVPoll.Failed)
|
||||
self.assertEqual(manager.failure_records[42], reason)
|
||||
manager._transfer_data.assert_not_called()
|
||||
# Not a transport fault, so the probe loop must not clear it.
|
||||
self.assertFalse(manager.failed_sessions)
|
||||
self.assertFalse(manager.session_failures)
|
||||
self.assertEqual(manager.state_layout_rejections, {"decode:42": reason})
|
||||
|
||||
def test_mooncake_probe_recovery_keeps_a_layout_rejection(self):
|
||||
"""A probe clears the transport blacklist but not a layout rejection."""
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.session_lock = threading.Lock()
|
||||
manager.failed_sessions = {"decode"}
|
||||
manager.session_failures = defaultdict(int, {"decode": 1})
|
||||
reason = f"{StateType.SWA} item length mismatch for positional entry 0"
|
||||
manager.state_layout_rejections = {"decode": reason}
|
||||
manager.engine = SimpleNamespace(send_probe=Mock(return_value=0))
|
||||
|
||||
manager._run_one_probe_pass()
|
||||
|
||||
self.assertFalse(manager.failed_sessions)
|
||||
self.assertFalse(manager.session_failures)
|
||||
self.assertEqual(manager.state_layout_rejections, {"decode": reason})
|
||||
|
||||
def test_mooncake_state_stride_matches_pp_slice(self):
|
||||
cases = [
|
||||
([], [64, 128], [32, 64, 128, 256], [1, 2]),
|
||||
@@ -301,6 +553,7 @@ class TestDisaggregationWire(unittest.TestCase):
|
||||
)
|
||||
manager.is_mla_backend = True
|
||||
manager.pp_size = 2
|
||||
manager.state_strides_validated = set()
|
||||
manager.enable_custom_mem_pool = False
|
||||
manager.max_transfer_batch_indices = 0
|
||||
manager._transfer_data = Mock(return_value=0)
|
||||
|
||||
@@ -663,6 +663,8 @@ class TestAllocationIntegration(CustomTestCase):
|
||||
mgr._staging_outstanding = defaultdict(int)
|
||||
mgr.session_lock = threading.Lock()
|
||||
mgr.failed_sessions = set()
|
||||
mgr.state_layout_rejections = {}
|
||||
mgr.state_strides_validated = set()
|
||||
mgr.enable_deferred_decode_kv_release = True
|
||||
mgr.decode_kv_args_table = {
|
||||
"session": SimpleNamespace(requires_dcp_relayout=False, dst_aux_ptrs=[])
|
||||
|
||||
Reference in New Issue
Block a user