diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index ac34e9da54..833cb61f32 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -10,6 +10,7 @@ import uuid from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple +import msgspec import numpy as np import numpy.typing as npt import torch @@ -78,6 +79,98 @@ _PEER_RELOAD_MIN_INTERVAL_S = 1.0 GUARD = "NixlMsgGuard".encode("ascii") KV_MEM_KINDS = {"VRAM", "DRAM"} +# Whole, fixed-stride slots on the generic path; DSA_TAIL encodes ring segments. +_PREPARED_STATE_TYPES = frozenset( + ( + StateType.MAMBA, + StateType.SWA, + StateType.DSA, + StateType.QSA_PENDING, + StateType.QSA_COMPRESSED, + StateType.SWA_RING, + StateType.DSV4_REQUEST_STATE, + StateType.BLOCK_SCALE, + StateType.BLOCK_SCALE_SWA, + StateType.MINIMAX_INDEX_K, + StateType.MINIMAX_DENSE_KV, + ) +) +# Flat per-entry layout on the generic path even on an MHA backend. +_FLAT_STATE_TYPES = frozenset( + ( + StateType.MAMBA, + StateType.DSA, + StateType.QSA_PENDING, + StateType.QSA_COMPRESSED, + StateType.MINIMAX_INDEX_K, + StateType.MINIMAX_DENSE_KV, + ) +) + + +def _nth(lists, index: int, default=()): + return lists[index] if index < len(lists) else default + + +def _state_slot_capacities( + *, ptrs: List[int], data_lens: List[int], item_lens: List[int] +) -> Optional[List[int]]: + # 0 for an elided or unregistered entry; None keeps the component generic. + if not (len(ptrs) == len(data_lens) == len(item_lens)): + return None + capacities = [] + for ptr, data_len, item_len in zip(ptrs, data_lens, item_lens): + if ptr == 0 or item_len == 0: + capacities.append(0) + elif data_len < item_len: + return None + else: + capacities.append(data_len // item_len) + return capacities + + +def _prepared_state_runs(*, entries, base: int, gpu_id: int): + runs, offsets = [], [] + for ptr, stride, capacity in entries: + offsets.append(base) + runs.append((ptr, stride, gpu_id, stride, capacity)) + base += capacity + column = np.asarray(offsets, dtype=np.int32).reshape(-1, 1) + column.flags.writeable = False + return runs, column, base + + +def _prepared_state_selectors(*, indices, offsets, capacity: int): + values = np.asarray(indices) + if values.ndim != 1 or values.dtype.kind not in "iu": + raise ValueError("NIXL prepared state indices must be a 1-D integer sequence") + if values.min() < 0 or values.max() >= capacity: + raise ValueError( + f"NIXL prepared state index outside the registered capacity {capacity}" + ) + if values.dtype.kind == "u": + values = values.astype(np.int64) + # int32 in one pass: NIXL borrows such selectors without copying. + return np.add(offsets, values, dtype=np.int32).ravel() + + +class _PreparedStateComponent(msgspec.Struct, frozen=True): + src_offsets: npt.NDArray[np.int32] + dst_offsets: npt.NDArray[np.int32] + src_capacity: int + dst_capacity: int + + +class _PreparedStatePlan(msgspec.Struct, frozen=True): + src_handle: Any + dst_handle: Any + components: Dict[int, _PreparedStateComponent] + + +# A decode stride that differs from the prefill's has no correct transfer path. +class _StateLayoutMismatch(ValueError): + pass + # Once one handle of a batch reports ERR, its siblings settle only when NIXL # notices their peer is gone, which for UCX means waiting out peer keepalive. @@ -255,6 +348,7 @@ class KVArgsRegisterInfo: kv_xfer_segments: Optional[List[_KVXferPreparedSegment]] = None staging_base_ptr: int = 0 staging_total_size: int = 0 + dst_state_data_lens: List[List[int]] = dataclasses.field(default_factory=list) @classmethod def from_zmq(cls, msg: List[bytes]): @@ -331,6 +425,12 @@ class KVArgsRegisterInfo: staging_total_size=( int(msg[15].decode("ascii")) if len(msg) > 15 and msg[15] != b"" else 0 ), + # Frame 23 is absent on decodes that predate prepared state transfers. + dst_state_data_lens=( + unpack_int_lists(msg[23], "Q") + if len(msg) > 23 and len(msg[23]) > 0 + else [] + ), ) @@ -522,6 +622,8 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): self.register_buffer_to_engine() self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get() + self._prepared_state_plans: Dict[str, _PreparedStatePlan] = {} + self._state_layout_rejections: Dict[str, str] = {} self.kv_buffer_tensors = None self.prep_handles: Dict[str, Any] = {} self.prep_handle_slice_src: Optional[Tuple[Any, int, int, int]] = ( @@ -539,6 +641,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): self._num_slots_src = num_kv_slots( self.kv_args.kv_data_lens, self.kv_args.kv_item_lens ) + self._prepared_state_components = self._prepared_state_component_indices( + backend + ) + logger.info( + "NIXL prepared state transfer: components=%s", + sorted(self._prepared_state_components), + ) transfer_queue_size = envs.SGLANG_DISAGGREGATION_QUEUE_SIZE.get() self.transfer_queues: List[FastQueue] = [ FastQueue() for _ in range(transfer_queue_size) @@ -575,6 +684,252 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): f"Unsupported DisaggregationMode: {self.disaggregation_mode}" ) + def _prepared_state_component_indices(self, backend: str) -> frozenset[int]: + if get_device().device != "cuda" or backend != "UCX": + return frozenset() + if self.pp_size != 1 or self.attn_cp_size != 1 or self.dcp_size != 1: + return frozenset() + return frozenset( + index + for index, state_type in enumerate(self.kv_args.state_types or []) + if state_type in _PREPARED_STATE_TYPES + ) + + def _prepare_state_xfer(self, peer_info: KVArgsRegisterInfo) -> None: + peer_name = peer_info.agent_name + self._prepared_state_plans.pop(peer_name, None) + self._state_layout_rejections.pop(peer_name, None) + if not self._prepared_state_components: + return + if peer_info.dst_dcp_size != 1 or peer_info.requires_dcp_relayout: + reason = "dcp-peer" + elif not peer_info.dst_state_data_lens: + reason = "no-decode-pool-lengths" + else: + reason = None + if reason is not None: + logger.info( + "NIXL prepared state: peer=%s all generic reason=%s", peer_name, reason + ) + return + planned, skipped, rejected = {}, [], [] + for index in sorted(self._prepared_state_components): + try: + entries = self._plan_prepared_state_component( + index=index, peer_info=peer_info + ) + except _StateLayoutMismatch as e: + rejected.append(str(e)) + continue + if isinstance(entries, str): + skipped.append(f"{index}:{entries}") + else: + planned[index] = entries + if rejected: + # Every transfer to this peer fails instead of the bootstrap thread. + self._state_layout_rejections[peer_name] = "; ".join(rejected) + logger.error("NIXL state layout rejected: peer=%s %s", peer_name, rejected) + return + plan = self._build_prepared_state_plan(peer_info=peer_info, planned=planned) + if plan is not None: + self._prepared_state_plans[peer_name] = plan + logger.info( + "NIXL prepared state: peer=%s prepared=%s generic=%s", + peer_name, + sorted(plan.components) if plan is not None else [], + skipped, + ) + + def _plan_prepared_state_component( + self, *, index: int, peer_info: KVArgsRegisterInfo + ): + state_type = self.kv_args.state_types[index] + if not self._state_component_is_whole_slot( + state_type=state_type, decode_tp_size=peer_info.decode_tp_size + ): + return "tp-layout" + src_ptrs = _nth(self.kv_args.state_data_ptrs, index) + src_strides = _nth(self.kv_args.state_item_lens, index) + src_caps = _state_slot_capacities( + ptrs=src_ptrs, + data_lens=_nth(self.kv_args.state_data_lens, index), + item_lens=src_strides, + ) + dst_ptrs = _nth(peer_info.dst_state_data_ptrs, index) + dst_strides = _nth(peer_info.dst_state_item_lens, index) + dst_caps = _state_slot_capacities( + ptrs=dst_ptrs, + data_lens=_nth(peer_info.dst_state_data_lens, index), + item_lens=dst_strides, + ) + if src_caps is None or dst_caps is None: + return "pool-geometry" + pairs = self._prepared_state_pairs( + state_type=state_type, + src_layer_ids=_nth(self.kv_args.state_layer_ids, index), + dst_layer_ids=_nth(peer_info.dst_state_layer_ids, index), + n_src=len(src_ptrs), + n_dst=len(dst_ptrs), + ) + if pairs is None: + return "entry-pairing" + pairs = [(s, d) for s, d in pairs if src_caps[s] and dst_caps[d]] + if not pairs: + return "no-entries" + for s, d in pairs: + if src_strides[s] != dst_strides[d]: + raise _StateLayoutMismatch( + f"NIXL prepared state component {index} ({state_type}) " + f"stride mismatch for peer {peer_info.agent_name}: " + f"src[{s}]={src_strides[s]}, dst[{d}]={dst_strides[d]}" + ) + return ( + [(src_ptrs[s], src_strides[s], src_caps[s]) for s, _ in pairs], + [(dst_ptrs[d], dst_strides[d], dst_caps[d]) for _, d in pairs], + ) + + def _prepared_state_pairs( + self, *, state_type, src_layer_ids, dst_layer_ids, n_src: int, n_dst: int + ) -> Optional[List[Tuple[int, int]]]: + if not (self.is_mla_backend or state_type in _FLAT_STATE_TYPES): + # The generic MHA K/V split pairs positionally; unequal counts do not line up. + return [(i, i) for i in range(n_src)] if n_src == n_dst else None + if n_src != n_dst and self.kv_args.mla_compression_ratios: + return None + try: + return build_transfer_entry_pairs( + src_layer_ids, + dst_layer_ids, + n_src, + n_dst, + allow_positional_fallback=self.pp_size == 1, + ) + except RuntimeError: + return None + + def _state_component_is_whole_slot(self, *, state_type, decode_tp_size) -> bool: + # Mirrors the dispatch in maybe_send_extra; the other layouts slice per rank. + if state_type == StateType.MAMBA: + return self.state_tp_size == decode_tp_size + if state_type in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED): + return self.is_mla_backend or self.state_tp_size == decode_tp_size + if state_type in (StateType.MINIMAX_INDEX_K, StateType.MINIMAX_DENSE_KV): + return self.attn_tp_size == decode_tp_size + if state_type == StateType.DSA: + return True + return self.is_mla_backend or self.attn_tp_size == decode_tp_size + + def _build_prepared_state_plan( + self, *, peer_info: KVArgsRegisterInfo, planned + ) -> Optional[_PreparedStatePlan]: + if not planned or not self._use_strided_descs: + return None + peer_name = peer_info.agent_name + src_total = sum(c for src, _ in planned.values() for _, _, c in src) + dst_total = sum(c for _, dst in planned.values() for _, _, c in dst) + if max(src_total, dst_total) > 0x7FFFFFFF: + # NIXL selects prepared descriptors with int32 indices. + logger.warning( + "NIXL prepared state: peer=%s all generic reason=index-space", + peer_name, + ) + return None + src_runs, dst_runs, components = [], [], {} + src_base = dst_base = 0 + for index, (src_entries, dst_entries) in planned.items(): + runs, src_offsets, src_base = _prepared_state_runs( + entries=src_entries, base=src_base, gpu_id=self.kv_args.gpu_id + ) + src_runs.extend(runs) + runs, dst_offsets, dst_base = _prepared_state_runs( + entries=dst_entries, base=dst_base, gpu_id=peer_info.gpu_id + ) + dst_runs.extend(runs) + components[index] = _PreparedStateComponent( + src_offsets=src_offsets, + dst_offsets=dst_offsets, + src_capacity=min(c for _, _, c in src_entries), + dst_capacity=min(c for _, _, c in dst_entries), + ) + try: + # Never expand to one descriptor per slot; a state pool holds millions. + src_handle = self.agent.prep_xfer_dlist( + "", self._pack_stride_descs(src_runs), "VRAM" + ) + dst_handle = self.agent.prep_xfer_dlist( + peer_name, self._pack_stride_descs(dst_runs), "VRAM" + ) + except TypeError: + logger.warning_once( + "Installed NIXL does not support strided descriptors; " + "state transfers stay generic." + ) + self._use_strided_descs = False + return None + except Exception as e: + logger.warning( + "NIXL prepared state: peer=%s all generic, dlist creation failed: %s", + peer_name, + e, + ) + return None + if src_handle is None or dst_handle is None: + return None + return _PreparedStatePlan( + src_handle=src_handle, dst_handle=dst_handle, components=components + ) + + def _send_prepared_states( + self, *, plan: _PreparedStatePlan, prefill_state_indices, dst_state_indices + ): + src_parts, dst_parts = [], [] + for index, component in plan.components.items(): + src_indices = _nth(prefill_state_indices, index, default=None) + dst_indices = _nth(dst_state_indices, index) + if src_indices is None or len(src_indices) == 0: + continue + if len(src_indices) != len(dst_indices): + raise RuntimeError( + f"State index length mismatch at component {index}: " + f"prefill={len(src_indices)}, dst={len(dst_indices)}" + ) + if ( + self.kv_args.state_types[index] == StateType.MAMBA + and len(src_indices) != 1 + ): + raise RuntimeError("Mamba should have single state index") + src_parts.append( + _prepared_state_selectors( + indices=src_indices, + offsets=component.src_offsets, + capacity=component.src_capacity, + ) + ) + dst_parts.append( + _prepared_state_selectors( + indices=dst_indices, + offsets=component.dst_offsets, + capacity=component.dst_capacity, + ) + ) + if not src_parts: + return + src_selectors = ( + src_parts[0] if len(src_parts) == 1 else np.concatenate(src_parts) + ) + dst_selectors = ( + dst_parts[0] if len(dst_parts) == 1 else np.concatenate(dst_parts) + ) + xfer_handle = self.agent.make_prepped_xfer( + "WRITE", plan.src_handle, src_selectors, plan.dst_handle, dst_selectors, b"" + ) + if not xfer_handle: + raise RuntimeError("Failed to create prepared state transfer") + # The chunk owns the handle before posting, so a failed post settles too. + yield xfer_handle + if self.agent.transfer(xfer_handle) == "ERR": + raise RuntimeError("Failed to post prepared state transfer") + def _init_staging_prefill_ctx(self): from sglang.srt.disaggregation.common.staging_handler import ( PrefillStagingContext, @@ -1239,6 +1594,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): kv_chunk: TransferKVChunk = queue.get() room = kv_chunk.room handles: List[Any] = [] + state_notifs: List[Tuple[str, str]] = [] settle_timed_out = False room_transfer_infos = None try: @@ -1476,8 +1832,20 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_dim_per_tensor=dst_info.dst_state_dim_per_tensor, dst_state_layer_ids=dst_info.dst_state_layer_ids, ) - handles.extend( - h for h in state_xfer_handles if h is not None + for state_xfer_handle in state_xfer_handles: + handles.append(state_xfer_handle) + + if req.dst_state_indices: + if not kv_chunk.state_indices: + logger.warning_once( + "Decode expects state but prefill has none to " + "send; announcing state completion anyway" + ) + state_notifs.append( + ( + req.agent_name, + f"{req.room}_state_{self.transfer_source_rank}", + ) ) if kv_chunk.prefill_aux_index is None: @@ -1520,6 +1888,10 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): if any_failed: raise RuntimeError(f"NIXL transfer encountered ERR room={room}") + if self.request_status.get(room) not in (None, KVPoll.Failed): + for peer_name, notif in state_notifs: + self.agent.send_notif(peer_name, notif.encode("ascii")) + # Clear with the decrement: the failure path below reads this # flag, so a raise after this point must not uncount twice. kv_chunk.staging_counted = False @@ -1667,6 +2039,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): self.agent.add_remote_agent(decode_kv_args.agent_metadata) if self.disaggregation_mode == DisaggregationMode.PREFILL: self._prepare_payload_xfer(decode_kv_args) + self._prepare_state_xfer(decode_kv_args) def _reload_invalidated_peers(self, room_transfer_infos: Dict[str, Any]) -> None: # NIXL drops a peer's metadata after a remote-disconnect error and never @@ -1694,10 +2067,12 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): # peer has the same peer set, so all shard to this worker. self.prep_handles.pop(agent_name, None) self.prep_handles_slice_dst.pop(agent_name, None) + self._prepared_state_plans.pop(agent_name, None) peer_info.kv_xfer_segments = None self.agent.add_remote_agent(peer_info.agent_metadata) try: self._prepare_payload_xfer(peer_info) + self._prepare_state_xfer(peer_info) except Exception: # Leave the peer invalid so a later failure retries. self.agent.remove_remote_agent(agent_name) @@ -2708,7 +3083,12 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_dim_per_tensor: List[List[int]] | None = None, dst_state_layer_ids: List[List[int]] | None = None, ): - """Send state per hybrid component, dispatching by state_type[i].""" + """Yield each state handle to the chunk owner; completion is announced there.""" + rejection = self._state_layout_rejections.get(peer_name) + if rejection is not None: + raise RuntimeError( + f"NIXL state layout rejected for {peer_name}: {rejection}" + ) state_types = getattr(self.kv_args, "state_types", []) or [] src_state_data_ptrs = self.kv_args.state_data_ptrs or [] src_state_item_lens = self.kv_args.state_item_lens or [] @@ -2726,8 +3106,18 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_dim_per_tensor = dst_state_dim_per_tensor or [] dst_state_layer_ids = dst_state_layer_ids or [] - handles = [] + plan = self._prepared_state_plans.get(peer_name) + if plan is not None: + yield from self._send_prepared_states( + plan=plan, + prefill_state_indices=prefill_state_indices, + dst_state_indices=dst_state_indices, + ) + prepared_components = plan.components if plan is not None else {} + for i, st in enumerate(state_types): + if i in prepared_components: + continue src_indices = ( prefill_state_indices[i] if i < len(prefill_state_indices) else None ) @@ -2758,7 +3148,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_dim_per_tensor[i] if i < len(dst_state_dim_per_tensor) else [] ) dst_lids = dst_state_layer_ids[i] if i < len(dst_state_layer_ids) else [] - comp_notif = f"{notif}_{i}" if st == StateType.MAMBA: if self.state_tp_size != decode_tp_size: @@ -2778,7 +3167,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_lens, dst_dims, dst_gpu_id, - comp_notif, + "", decode_tp_size, decode_tp_rank, src_conv, @@ -2795,7 +3184,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_ptrs, dst_indices, dst_gpu_id, - comp_notif, + "", src_layer_ids=src_lids, dst_layer_ids=dst_lids, ) @@ -2809,7 +3198,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): list(src_indices), list(dst_indices), dst_gpu_id, - comp_notif, + "", ) elif st == StateType.DSA: if len(src_indices) != len(dst_indices): @@ -2825,7 +3214,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=np.array(src_indices, dtype=np.int32), dst_data_indices=np.array(dst_indices, dtype=np.int32), dst_gpu_id=dst_gpu_id, - notif=comp_notif, + notif="", state_type=st, force_flat=True, ) @@ -2843,7 +3232,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_swa_page_indices=dst_indices, dst_state_item_lens=dst_lens, dst_gpu_id=dst_gpu_id, - notif=comp_notif, + notif="", decode_tp_size=decode_tp_size, decode_tp_rank=decode_tp_rank, ) @@ -2885,7 +3274,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=np.array(src_indices, dtype=np.int32), dst_data_indices=np.array(dst_indices, dtype=np.int32), dst_gpu_id=dst_gpu_id, - notif=comp_notif, + notif="", state_type=st, force_flat=st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED), src_layer_ids=src_lids, @@ -2916,7 +3305,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=np.array(src_indices, dtype=np.int32), dst_data_indices=np.array(dst_indices, dtype=np.int32), dst_gpu_id=dst_gpu_id, - notif=comp_notif, + notif="", force_flat=True, ) else: @@ -2924,8 +3313,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): f"PD Disaggregation via NIXL does NOT support {st} hybrid models yet." ) if h is not None: - handles.append(h) - return handles + yield h def add_transfer_request( self, @@ -3550,6 +3938,9 @@ class NixlKVReceiver(CommonKVReceiver): packed_state_item_lens = pack_int_lists( self.kv_mgr.kv_args.state_item_lens or [], "I" ) + packed_state_data_lens = pack_int_lists( + self.kv_mgr.kv_args.state_data_lens or [], "Q" + ) packed_state_dim_per_tensor = pack_int_lists( getattr(self.kv_mgr.kv_args, "state_dim_per_tensor", []) or [], "I" ) @@ -3606,6 +3997,7 @@ class NixlKVReceiver(CommonKVReceiver): packed_kv_layer_ids, str(self.kv_mgr.dcp_size).encode("ascii"), str(self.kv_mgr.dcp_rank).encode("ascii"), + packed_state_data_lens, ] ) except zmq.ZMQError: diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 2b54fbe60b..e99fb69282 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -1263,16 +1263,20 @@ class TestHeteroTpSwaStateTransfer(CustomTestCase): def _send_nixl(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len): prefill.agent = _MemcpyNixlAgent(memory) - handles = prefill.maybe_send_extra( - "decode", - [np.array(self.SRC_PAGES, dtype=np.int32)], - [dst_ptrs], - [self.DST_PAGES], - 0, - "7_state_0", - decode_tp, - decode_tp_rank=dst_rank, - dst_state_item_lens=[[item_len] * len(dst_ptrs)], + prefill._prepared_state_plans = {} + prefill._state_layout_rejections = {} + handles = list( + prefill.maybe_send_extra( + "decode", + [np.array(self.SRC_PAGES, dtype=np.int32)], + [dst_ptrs], + [self.DST_PAGES], + 0, + "7_state_0", + decode_tp, + decode_tp_rank=dst_rank, + dst_state_item_lens=[[item_len] * len(dst_ptrs)], + ) ) self.assertEqual(len(handles), 1) diff --git a/test/registered/unit/disaggregation/test_nixl_backend_basic.py b/test/registered/unit/disaggregation/test_nixl_backend_basic.py index dc97fab645..e439c4495b 100644 --- a/test/registered/unit/disaggregation/test_nixl_backend_basic.py +++ b/test/registered/unit/disaggregation/test_nixl_backend_basic.py @@ -116,11 +116,13 @@ class TestNixlBackendInitialization(CustomTestCase): gpu_id=gpu_id, kv_data_ptrs=[], kv_item_lens=[], + state_types=[], ) mgr = object.__new__(NixlKVManager) mgr.kv_args = args mgr.disaggregation_mode = mode mgr.enable_deferred_decode_kv_release = False + mgr.pp_size = mgr.attn_cp_size = mgr.dcp_size = 1 api = types.ModuleType("nixl._api") api.nixl_agent = MagicMock(return_value=agent) @@ -510,6 +512,57 @@ class TestNixlEmptyStateTransfer(CustomTestCase): self.assertEqual(mgr.agent.get_xfer_descs_calls, []) self.assertEqual(mgr.agent.initialize_xfer_calls, []) + def test_prepared_mamba_rejects_stride_mismatch_before_fallback(self): + """Different SSM dtypes must not write into the wrong decode request slot.""" + for src_stride, dst_stride in ((32, 16), (16, 32)): + with self.subTest(src_stride=src_stride, dst_stride=dst_stride): + mgr = object.__new__(NixlKVManager) + mgr.agent = MagicMock() + mgr.pp_size = mgr.state_tp_size = 1 + mgr.is_mla_backend = False + mgr._use_strided_descs = True + mgr._prepared_state_components = frozenset({0}) + mgr._prepared_state_plans = {} + mgr._state_layout_rejections = {} + mgr.kv_args = SimpleNamespace( + state_types=[StateType.MAMBA], + state_data_ptrs=[[0x10000, 0x11000]], + state_data_lens=[[16 * 16, src_stride * 16]], + state_item_lens=[[16, src_stride]], + state_layer_ids=[[0, 0]], + mla_compression_ratios=None, + ) + peer = SimpleNamespace( + agent_name="decode", + decode_tp_size=1, + dst_dcp_size=1, + requires_dcp_relayout=False, + dst_state_data_ptrs=[[0x20000, 0x21000]], + dst_state_data_lens=[[16 * 16, dst_stride * 16]], + dst_state_item_lens=[[16, dst_stride]], + dst_state_layer_ids=[[0, 0]], + ) + + # Registration records the rejection; each transfer to the peer fails. + mgr._prepare_state_xfer(peer) + + self.assertIn("stride mismatch", mgr._state_layout_rejections["decode"]) + self.assertEqual(mgr._prepared_state_plans, {}) + mgr.agent.prep_xfer_dlist.assert_not_called() + with self.assertRaisesRegex(RuntimeError, "stride mismatch"): + list( + mgr.maybe_send_extra( + "decode", + [[1]], + peer.dst_state_data_ptrs, + [[2]], + 0, + "7_state_0", + 1, + ) + ) + mgr.agent.transfer.assert_not_called() + def test_paired_state_entries_reject_item_length_mismatch(self): mgr = object.__new__(NixlKVManager) mgr.agent = StagingFakeAgent() @@ -1504,6 +1557,9 @@ class TestNixlInvalidatedPeerReload(CustomTestCase): mgr._num_slots_src = 4 mgr.prep_handles = {} mgr.prep_handles_slice_dst = {} + mgr._prepared_state_plans = {} + mgr._prepared_state_components = frozenset() + mgr._state_layout_rejections = {} mgr._peer_reload_lock = threading.Lock() mgr._peer_reload_times = {} mgr.kv_args = SimpleNamespace(