diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index a4b5c01b3b..c41215413b 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -8,7 +8,7 @@ import struct import threading import time from collections import defaultdict -from typing import List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Set, Tuple, Union import numpy as np import numpy.typing as npt @@ -216,6 +216,27 @@ class KVArgsRegisterInfo: ) +def _uses_flat_entry_layout( + *, + mla_backend: bool, + force_flat: bool, + has_layer_ids: bool, + state_type: Optional[StateType], + num_src_entries: int, + num_dst_entries: int, +) -> bool: + """One region per transfer entry, as opposed to plain MHA's K-then-V split. + + Shared by the transfer and the registration-time stride check so both + judge the same entries. Unified SWA publishes one page-envelope region + even on an MHA backend. + """ + is_single_region_swa = ( + state_type == StateType.SWA and num_src_entries == 1 and num_dst_entries == 1 + ) + return mla_backend or force_flat or has_layer_ids or is_single_region_swa + + class MooncakeKVManager(StagingManagerMixin, CommonKVManager): AUX_DATA_HEADER = b"AUX_DATA" # Implements teardown() below, so runtime PD role switching is supported. @@ -250,6 +271,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): if self.disaggregation_mode == DisaggregationMode.PREFILL: self.session_failures = defaultdict(int) self.failed_sessions = set() + # Separate from failed_sessions, which _run_one_probe_pass clears + # on a successful probe: an incompatible decode is still reachable, + # so only a re-registration may lift this. + self.state_layout_rejections: Dict[str, str] = {} + # Peers already cleared at registration; skip the per-transfer check. + self.state_strides_validated: Set[str] = set() self.session_lock = threading.Lock() self.start_prefill_thread() # Determine the number of threads to use for kv sender @@ -762,6 +789,167 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths) ) + def _state_stride_mismatch( + self, + *, + src_data_ptrs: list[int], + dst_data_ptrs: list[int], + item_lens: list[int], + dst_item_lens: list[int], + state_type: Optional[StateType] = None, + force_flat: bool = False, + src_layer_ids: Optional[List[int]] = None, + dst_layer_ids: Optional[List[int]] = None, + ) -> Optional[str]: + """First entry whose decode stride differs from prefill's, else None. + + The destination is addressed as ``dst_ptr + dst_index * src_item_len``, + so mapped entries must agree on the stride or the write lands in the + wrong slot. + """ + has_layer_ids = bool(src_layer_ids or dst_layer_ids) + if not _uses_flat_entry_layout( + mla_backend=self.is_mla_backend or self.is_hybrid_mla_backend, + force_flat=force_flat, + has_layer_ids=has_layer_ids, + state_type=state_type, + num_src_entries=len(src_data_ptrs), + num_dst_entries=len(dst_data_ptrs), + ): + # The MHA branch splits the entries into K/V and is not covered. + return None + if has_layer_ids: + pairs = build_transfer_entry_pairs( + src_layer_ids or [], + dst_layer_ids or [], + len(src_data_ptrs), + len(dst_data_ptrs), + allow_positional_fallback=self.pp_size == 1, + ) + for i, j in pairs: + if j >= len(dst_item_lens): + return ( + f"{state_type} decode published {len(dst_item_lens)} item " + f"lengths but paired entry dst[{j}] needs one" + ) + if item_lens[i] != dst_item_lens[j]: + return ( + f"{state_type} item length mismatch for paired " + f"entries src[{i}]={item_lens[i]} " + f"dst[{j}]={dst_item_lens[j]}" + ) + return None + _, _, layers_current_pp_stage = self.get_mla_kv_ptrs_with_pp( + src_data_ptrs, dst_data_ptrs, state_type + ) + # Lens are parallel to pointers, so the same slice gives each entry's + # decode stride. + _, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp( + item_lens, dst_item_lens, state_type + ) + if len(mapped_dst_lens) < layers_current_pp_stage: + return ( + f"{state_type} decode published {len(mapped_dst_lens)} item " + f"lengths for this stage's {layers_current_pp_stage} entries" + ) + for layer_id in range(layers_current_pp_stage): + if item_lens[layer_id] != mapped_dst_lens[layer_id]: + return ( + f"{state_type} item length mismatch for positional " + f"entry {layer_id}: prefill={item_lens[layer_id]} " + f"decode={mapped_dst_lens[layer_id]}" + ) + return None + + def _publish_peer_registration( + self, mooncake_session_id: str, decode_kv_args: KVArgsRegisterInfo + ) -> Optional[str]: + """Record this peer's state-layout verdict, then publish its metadata. + + Publishing last stops a transfer from reading the new layout while the + previous registration's verdict still stands. Returns the rejection + reason, if any. + """ + fully_checked, mismatch = self._validate_peer_state_layout(decode_kv_args) + with self.session_lock: + self.failed_sessions.discard(mooncake_session_id) + self.session_failures.pop(mooncake_session_id, None) + self.state_layout_rejections.pop(mooncake_session_id, None) + self.state_strides_validated.discard(mooncake_session_id) + if mismatch is not None: + self.state_layout_rejections[mooncake_session_id] = mismatch + elif fully_checked: + self.state_strides_validated.add(mooncake_session_id) + self.decode_kv_args_table[mooncake_session_id] = decode_kv_args + return mismatch + + def _validate_peer_state_layout( + self, registration_info: KVArgsRegisterInfo + ) -> Tuple[bool, Optional[str]]: + """Check every generic state component against one decode peer. + + Neither side changes its entry sizes after registering, so one verdict + covers the whole session. Returns ``(fully_checked, mismatch_reason)``: + a reason rejects the peer, ``fully_checked`` retires the per-transfer + check. + """ + state_types = getattr(self.kv_args, "state_types", []) + fully_checked = True + for i, st in enumerate(state_types): + if not self._is_generic_kvcache_state_type(st): + # Mamba, DSA tail and MiniMax send paths check their own layout. + continue + dst_data_ptrs = ( + registration_info.dst_state_data_ptrs[i] + if i < len(registration_info.dst_state_data_ptrs) + else [] + ) + dst_item_lens = ( + registration_info.dst_state_item_lens[i] + if i < len(registration_info.dst_state_item_lens) + else [] + ) + if not dst_data_ptrs or not dst_item_lens: + # Nothing published for this component; let the send path report it. + fully_checked = False + continue + try: + mismatch = self._state_stride_mismatch( + src_data_ptrs=self.kv_args.state_data_ptrs[i], + dst_data_ptrs=dst_data_ptrs, + item_lens=self.kv_args.state_item_lens[i], + dst_item_lens=dst_item_lens, + state_type=st, + force_flat=( + st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED) + or get_memory().enable_unified_memory + ), + src_layer_ids=( + self.kv_args.state_layer_ids[i] + if i < len(self.kv_args.state_layer_ids) + else [] + ), + dst_layer_ids=( + registration_info.dst_state_layer_ids[i] + if i < len(registration_info.dst_state_layer_ids) + else [] + ), + ) + except Exception as e: + # Pairing can reject a layout on its own; never let that kill + # the bootstrap thread. Fall back to the send path's check. + logger.warning( + "Could not pre-check %s layout for session %s: %s", + st, + registration_info.mooncake_session_id, + e, + ) + fully_checked = False + continue + if mismatch is not None: + return False, mismatch + return fully_checked, None + def _send_kvcache_generic( self, mooncake_session_id: str, @@ -803,22 +991,41 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): layers_params = None + # Skipped once registration has cleared this peer's layout. + if ( + dst_item_lens is not None + and mooncake_session_id not in self.state_strides_validated + ): + mismatch = self._state_stride_mismatch( + src_data_ptrs=src_data_ptrs, + dst_data_ptrs=dst_data_ptrs, + item_lens=item_lens, + dst_item_lens=dst_item_lens, + state_type=state_type, + force_flat=force_flat, + src_layer_ids=src_layer_ids, + dst_layer_ids=dst_layer_ids, + ) + if mismatch is not None: + assert bootstrap_room is not None + logger.error(mismatch) + self.conclude_failure( + bootstrap_room=bootstrap_room, + failure_reason=mismatch, + ) + return -1 + # Decode pp size should be equal to prefill pp size or 1 # Published layer IDs give exact pairing; plain-MHA peers publish none # and keep positional slicing. has_layer_ids = bool(src_layer_ids or dst_layer_ids) - # Unified SWA publishes one page-envelope region even on an MHA backend. - is_single_region_swa = ( - state_type == StateType.SWA - and len(src_data_ptrs) == 1 - and len(dst_data_ptrs) == 1 - ) - if ( - self.is_mla_backend - or self.is_hybrid_mla_backend - or force_flat - or has_layer_ids - or is_single_region_swa + if _uses_flat_entry_layout( + mla_backend=self.is_mla_backend or self.is_hybrid_mla_backend, + force_flat=force_flat, + has_layer_ids=has_layer_ids, + state_type=state_type, + num_src_entries=len(src_data_ptrs), + num_dst_entries=len(dst_data_ptrs), ): # Layer IDs map PP-local buffers to global decode entries. # Registrations without them retain the existing PP mapping. @@ -830,21 +1037,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): len(dst_data_ptrs), allow_positional_fallback=self.pp_size == 1, ) - if dst_item_lens is not None: - for i, j in pairs: - if item_lens[i] != dst_item_lens[j]: - assert bootstrap_room is not None - failure_reason = ( - f"{state_type} item length mismatch for paired " - f"entries src[{i}]={item_lens[i]} " - f"dst[{j}]={dst_item_lens[j]}" - ) - logger.error(failure_reason) - self.conclude_failure( - bootstrap_room=bootstrap_room, - failure_reason=failure_reason, - ) - return -1 layers_params = [ (src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs ] @@ -854,24 +1046,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): src_data_ptrs, dst_data_ptrs, state_type ) ) - if dst_item_lens is not None: - _, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp( - item_lens, dst_item_lens, state_type - ) - for layer_id in range(layers_current_pp_stage): - if item_lens[layer_id] != mapped_dst_lens[layer_id]: - assert bootstrap_room is not None - failure_reason = ( - f"{state_type} item length mismatch for positional " - f"entry {layer_id}: prefill={item_lens[layer_id]} " - f"decode={mapped_dst_lens[layer_id]}" - ) - logger.error(failure_reason) - self.conclude_failure( - bootstrap_room=bootstrap_room, - failure_reason=failure_reason, - ) - return -1 layers_params = [ ( src_kv_ptrs[layer_id], @@ -2199,10 +2373,17 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): if not req.is_dummy: # Early exit if the request has failed with self.session_lock: - if req.mooncake_session_id in self.failed_sessions: + layout_rejection = self.state_layout_rejections.get( + req.mooncake_session_id + ) + if ( + layout_rejection is not None + or req.mooncake_session_id in self.failed_sessions + ): self.conclude_failure( bootstrap_room=kv_chunk.room, - failure_reason=( + failure_reason=layout_rejection + or ( "Decode instance could be dead, remote " f"mooncake session {req.mooncake_session_id} " "is not alive" @@ -2594,15 +2775,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): self._init_dcp_pack_buffers_once( decode_kv_args.dst_dcp_size, include_draft=True ) - self.decode_kv_args_table[mooncake_session_id] = decode_kv_args - with self.session_lock: - if mooncake_session_id in self.failed_sessions: - self.failed_sessions.remove(mooncake_session_id) - if mooncake_session_id in self.session_failures: - del self.session_failures[mooncake_session_id] - logger.debug( - f"Register KVArgs from {mooncake_session_id} successfully" + layout_mismatch = self._publish_peer_registration( + mooncake_session_id, decode_kv_args ) + if layout_mismatch is not None: + logger.error( + "Rejecting state transfers to decode session %s: %s", + mooncake_session_id, + layout_mismatch, + ) + else: + logger.debug( + f"Register KVArgs from {mooncake_session_id} successfully" + ) continue else: required_dst_info_num = int(waiting_req_bytes[7].decode("ascii")) diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 9db56f3346..c6c6abcd51 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -156,6 +156,8 @@ class TestDisaggregationWire(unittest.TestCase): manager.failure_lock = threading.Lock() manager.failed_sessions = set() manager.session_failures = defaultdict(int) + manager.state_layout_rejections = {} + manager.state_strides_validated = set() manager.failure_records = {} manager._staging_outstanding = defaultdict(int) manager.request_status = { @@ -280,6 +282,256 @@ class TestDisaggregationWire(unittest.TestCase): self.assertFalse(manager.transfer_infos) self.assertFalse(manager.req_to_decode_prefix_len) + def test_mooncake_short_decode_item_lens_is_a_verdict_not_an_index_error(self): + """A decode publishing fewer item lengths than entries used to raise + IndexError out of the stride check and take the transfer thread down; + it must come back as a rejection reason on both pairing paths.""" + for layer_ids in ([], [3, 4]): + with self.subTest(layer_ids=layer_ids): + manager = object.__new__(MooncakeKVManager) + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.pp_size = 1 + manager.kv_args = SimpleNamespace( + mla_compression_ratios=None, prefill_start_layer=0 + ) + reason = manager._state_stride_mismatch( + src_data_ptrs=[0x1000, 0x1100], + dst_data_ptrs=[0x2000, 0x2100], + item_lens=[32, 32], + dst_item_lens=[32], + state_type=StateType.SWA, + src_layer_ids=layer_ids, + dst_layer_ids=layer_ids, + ) + self.assertIsNotNone(reason) + self.assertIn("item lengths", reason) + + def test_mooncake_validates_peer_state_layout_at_registration(self): + """The verdict is reachable at registration, not just at transfer time.""" + for layer_ids in ([], [7]): + for dst_item_len in (32, 64): + with self.subTest(layer_ids=layer_ids, dst_item_len=dst_item_len): + manager = object.__new__(MooncakeKVManager) + manager.kv_args = SimpleNamespace( + state_types=[StateType.SWA, StateType.MAMBA], + state_data_ptrs=[[0x1000], [0x3000]], + state_item_lens=[[32], [16]], + state_layer_ids=[layer_ids, layer_ids], + ) + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.pp_size = 1 + registration_info = SimpleNamespace( + mooncake_session_id="decode", + dst_state_data_ptrs=[[0x2000], [0x4000]], + # Mamba disagrees too, but ships on its own path. + dst_state_item_lens=[[dst_item_len], [999]], + dst_state_layer_ids=[layer_ids, layer_ids], + ) + with get_context().override_server_args( + enable_unified_memory=False + ): + fully_checked, reason = manager._validate_peer_state_layout( + registration_info + ) + if dst_item_len == 32: + self.assertTrue(fully_checked) + self.assertIsNone(reason) + else: + self.assertFalse(fully_checked) + self.assertEqual( + reason, + f"{StateType.SWA} item length mismatch for " + + ( + "paired entries src[0]=32 dst[0]=64" + if layer_ids + else "positional entry 0: prefill=32 decode=64" + ), + ) + + def test_mooncake_registration_replaces_the_previous_layout_verdict(self): + """Re-registering re-derives the verdict, and publishes the layout last.""" + manager = object.__new__(MooncakeKVManager) + manager.kv_args = SimpleNamespace( + state_types=[StateType.SWA], + state_data_ptrs=[[0x1000]], + state_item_lens=[[32]], + state_layer_ids=[[]], + ) + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.pp_size = 1 + manager.session_lock = threading.Lock() + manager.failed_sessions = {"decode"} + manager.session_failures = defaultdict(int, {"decode": 2}) + manager.state_layout_rejections = {} + manager.state_strides_validated = set() + manager.decode_kv_args_table = {} + + def register(dst_item_len): + info = SimpleNamespace( + mooncake_session_id="decode", + dst_state_data_ptrs=[[0x2000]], + dst_state_item_lens=[[dst_item_len]], + dst_state_layer_ids=[[]], + ) + with get_context().override_server_args(enable_unified_memory=False): + return info, manager._publish_peer_registration("decode", info) + + good, mismatch = register(32) + self.assertIsNone(mismatch) + self.assertEqual(manager.state_strides_validated, {"decode"}) + self.assertFalse(manager.state_layout_rejections) + self.assertIs(manager.decode_kv_args_table["decode"], good) + # A transport blacklist is cleared by registering, as before. + self.assertFalse(manager.failed_sessions) + self.assertFalse(manager.session_failures) + + bad, mismatch = register(64) + self.assertIn("item length mismatch", mismatch) + # The stale clearance is gone before the new layout is reachable. + self.assertFalse(manager.state_strides_validated) + self.assertEqual(manager.state_layout_rejections, {"decode": mismatch}) + self.assertIs(manager.decode_kv_args_table["decode"], bad) + + def test_mooncake_unpublished_state_metadata_keeps_the_transfer_check(self): + """An unpublished component cannot be judged; keep the transfer check.""" + manager = object.__new__(MooncakeKVManager) + manager.kv_args = SimpleNamespace( + state_types=[StateType.SWA], + state_data_ptrs=[[0x1000]], + state_item_lens=[[32]], + state_layer_ids=[[]], + ) + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.pp_size = 1 + registration_info = SimpleNamespace( + mooncake_session_id="decode", + dst_state_data_ptrs=[[0x2000]], + dst_state_item_lens=[], + dst_state_layer_ids=[[]], + ) + with get_context().override_server_args(enable_unified_memory=False): + fully_checked, reason = manager._validate_peer_state_layout( + registration_info + ) + self.assertFalse(fully_checked) + self.assertIsNone(reason) + + def test_mooncake_validated_session_skips_the_per_transfer_check(self): + """A cleared session pays nothing per request. + + Driven with sizes that would be rejected, so the transfer going through + proves the check was skipped rather than merely passing. + """ + manager = object.__new__(MooncakeKVManager) + manager.kv_args = SimpleNamespace(kv_data_ptrs=[]) + manager.is_mla_backend = True + manager.pp_size = 1 + manager.enable_custom_mem_pool = False + manager.max_transfer_batch_indices = 0 + manager.state_strides_validated = {"decode"} + manager._transfer_data = Mock(return_value=0) + manager._state_stride_mismatch = Mock( + side_effect=AssertionError("must not re-check a validated session") + ) + rc = manager._send_kvcache_generic( + mooncake_session_id="decode", + executor=None, + src_data_ptrs=[0x1000], + dst_data_ptrs=[0x2000], + item_lens=[32], + dst_item_lens=[64], + prefill_data_indices=np.array([2], dtype=np.int32), + dst_data_indices=np.array([5], dtype=np.int32), + state_type=StateType.SWA, + bootstrap_room=42, + ) + self.assertEqual(rc, 0) + manager._state_stride_mismatch.assert_not_called() + manager._transfer_data.assert_called_once_with( + "decode", [(0x1000 + 2 * 32, 0x2000 + 5 * 32, 32)] + ) + + def test_mooncake_layout_rejected_session_fails_requests_without_writing(self): + """A rejected peer fails every request and stays off the probe-clearable + transport blacklist.""" + manager = object.__new__(MooncakeKVManager) + manager.kv_args = SimpleNamespace(kv_data_ptrs=[], state_types=[StateType.SWA]) + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.attn_tp_size = manager.attn_cp_size = manager.pp_size = 1 + manager.attn_tp_rank = manager.attn_cp_rank = manager.pp_rank = 0 + manager.enable_custom_mem_pool = False + manager.max_transfer_batch_indices = 0 + manager.enable_trace = manager.enable_staging = False + manager.bootstrap_port = 1234 + manager.session_lock = threading.Lock() + manager.failure_lock = threading.Lock() + manager.failed_sessions = set() + manager.session_failures = defaultdict(int) + manager.failure_records = {} + manager._staging_outstanding = defaultdict(int) + manager.request_status = {42: KVPoll.Transferring} + manager.req_to_decode_prefix_len = {42: 0} + reason = f"{StateType.SWA} item length mismatch for positional entry 0" + manager.state_layout_rejections = {"decode:42": reason} + manager.state_strides_validated = set() + req = TransferInfo( + room=42, + endpoint="127.0.0.1", + dst_port=1234, + mooncake_session_id="decode:42", + dst_kv_indices=np.array([], dtype=np.int32), + dst_aux_index=5, + dst_state_indices=[[5]], + required_dst_info_num=1, + is_dummy=False, + ) + manager.transfer_infos = {42: {"decode:42": req}} + manager.decode_kv_args_table = {} + manager._send_multipart_locked = Mock() + manager._transfer_data = Mock(return_value=0) + queue = FastQueue() + queue.put( + TransferKVChunk( + room=42, + prefill_kv_indices=np.array([], dtype=np.int32), + index_slice=slice(0, 0), + is_last_chunk=True, + prefill_aux_index=2, + state_indices=[[2]], + ) + ) + queue.put(None) + manager.transfer_worker(queue, executor=None) + + self.assertEqual(manager.request_status[42], KVPoll.Failed) + self.assertEqual(manager.failure_records[42], reason) + manager._transfer_data.assert_not_called() + # Not a transport fault, so the probe loop must not clear it. + self.assertFalse(manager.failed_sessions) + self.assertFalse(manager.session_failures) + self.assertEqual(manager.state_layout_rejections, {"decode:42": reason}) + + def test_mooncake_probe_recovery_keeps_a_layout_rejection(self): + """A probe clears the transport blacklist but not a layout rejection.""" + manager = object.__new__(MooncakeKVManager) + manager.session_lock = threading.Lock() + manager.failed_sessions = {"decode"} + manager.session_failures = defaultdict(int, {"decode": 1}) + reason = f"{StateType.SWA} item length mismatch for positional entry 0" + manager.state_layout_rejections = {"decode": reason} + manager.engine = SimpleNamespace(send_probe=Mock(return_value=0)) + + manager._run_one_probe_pass() + + self.assertFalse(manager.failed_sessions) + self.assertFalse(manager.session_failures) + self.assertEqual(manager.state_layout_rejections, {"decode": reason}) + def test_mooncake_state_stride_matches_pp_slice(self): cases = [ ([], [64, 128], [32, 64, 128, 256], [1, 2]), @@ -301,6 +553,7 @@ class TestDisaggregationWire(unittest.TestCase): ) manager.is_mla_backend = True manager.pp_size = 2 + manager.state_strides_validated = set() manager.enable_custom_mem_pool = False manager.max_transfer_batch_indices = 0 manager._transfer_data = Mock(return_value=0) diff --git a/test/registered/unit/disaggregation/test_prefill_complete_integration.py b/test/registered/unit/disaggregation/test_prefill_complete_integration.py index b0cffe502c..68bc76e581 100644 --- a/test/registered/unit/disaggregation/test_prefill_complete_integration.py +++ b/test/registered/unit/disaggregation/test_prefill_complete_integration.py @@ -663,6 +663,8 @@ class TestAllocationIntegration(CustomTestCase): mgr._staging_outstanding = defaultdict(int) mgr.session_lock = threading.Lock() mgr.failed_sessions = set() + mgr.state_layout_rejections = {} + mgr.state_strides_validated = set() mgr.enable_deferred_decode_kv_release = True mgr.decode_kv_args_table = { "session": SimpleNamespace(requires_dcp_relayout=False, dst_aux_ptrs=[])