From 3eebf987ed622395f8fae3ed99ec6a1ed1ffb83e Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:21:54 -0700 Subject: [PATCH] [PD] Support heterogeneous TP for hybrid SWA KV transfer (#40909) --- .../advanced_features/pd_disaggregation.mdx | 2 + python/sglang/srt/disaggregation/base/conn.py | 3 + .../sglang/srt/disaggregation/common/conn.py | 122 +++++++++ .../srt/disaggregation/mooncake/conn.py | 67 ++++- python/sglang/srt/disaggregation/nixl/conn.py | 84 +++++- python/sglang/srt/disaggregation/utils.py | 8 + .../test_disaggregation_swa_different_tp.py | 43 +++ .../test_disaggregation_wire.py | 246 ++++++++++++++++++ .../test_staging_draft_kv_slots.py | 11 + 9 files changed, 583 insertions(+), 3 deletions(-) create mode 100644 test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index 17fa62441d..154b4070b9 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -347,6 +347,8 @@ Enable the staging buffer when prefill and decode use **different TP sizes** wit > **Note:** The staging buffer is designed for non-MLA models (e.g. GQA, MHA). MLA models (e.g. DeepSeek-V2/V3) should not enable this flag. +> **Note:** You can use different attention TP sizes for hybrid sliding-window models (e.g. `openai/gpt-oss-120b`) with Mooncake or NIXL. For unequal TP, both backends transfer the sliding-window KV by token and KV-head slice. The staging buffer carries only full-attention KV. Use `--pp-size 1` and leave `--enable-unified-memory` disabled for this configuration. MXFP8 KV cache is not supported for unequal-TP SWA transfer. + ### Environment Variables diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index e9a057cf44..d31682c773 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -81,6 +81,9 @@ class KVArgs: # scatter transfer can slice each independently head-sharded sub-block; None # per tensor when the single contiguous slice already matches the layout. state_conv_shard_groups: List[List[Optional[List[int]]]] + # Physical layout of the SWA sub-pool ("nhd", "hnd" or "vectorized_5d"); + # None when the SWA state is not held in an MHA sub-pool. + swa_kv_cache_layout: Optional[str] ib_device: str gpu_id: int kv_head_num: int diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index db2b6d14ac..ee137d1e3b 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -1230,6 +1230,26 @@ class CommonKVManager(BaseKVManager): "servers (including prefill CP ranks for MLA)" ) + # Every send path rejects head-sharded MXFP8 block scales across + # attention TP sizes, so each request would fail at its first transfer. + if ( + info.attn_tp_size != self.attn_tp_size + and not self.is_mla_backend + and not self.is_hybrid_mla_backend + ): + block_scale_states = [ + st + for st in getattr(self.kv_args, "state_types", None) or [] + if st in (StateType.BLOCK_SCALE, StateType.BLOCK_SCALE_SWA) + ] + if block_scale_states: + raise RuntimeError( + "PD Disaggregation does NOT support different attention TP " + "sizes with MXFP8 KV cache: prefill attn_tp_size=" + f"{info.attn_tp_size}, decode attn_tp_size={self.attn_tp_size}, " + f"state types {[st.value for st in block_scale_states]}." + ) + if self.dcp_size > 1: if not (self.is_mla_backend or self.is_hybrid_mla_backend): raise RuntimeError( @@ -1541,6 +1561,108 @@ class CommonKVManager(BaseKVManager): layers_current_pp_stage = len(src_k_ptrs) return src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage + def _get_mha_head_slice_blocks( + self, + src_ptrs: List[int], + src_item_lens: List[int], + dst_ptrs: List[int], + dst_item_lens: List[int], + src_page_indices: npt.NDArray[np.int32], + dst_page_indices: npt.NDArray[np.int32], + dst_attn_tp_size: int, + dst_tp_rank: int, + ) -> List[Tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint64], int]]: + # SWA state has no layer ids and its pool starts every PP stage at layer + # 0, so a prefill stage's entries cannot be paired with the decode layers. + if self.pp_size > 1: + raise RuntimeError( + "PD Disaggregation does NOT support PD different TP sizes for " + "non-MLA SWA hybrid models with pipeline parallelism yet." + ) + # The addresses below assume NHD pages [page_size, heads, head_dim]; HND + # and vectorized_5d pages have the same size in another element order. + if self.kv_args.swa_kv_cache_layout != "nhd": + raise RuntimeError( + "Head-sliced PD transfer needs the NHD KV cache layout, got " + f"{self.kv_args.swa_kv_cache_layout}" + ) + from sglang.srt.disaggregation.common.staging_buffer import ( + compute_head_slice_params, + resolve_total_kv_heads, + ) + + src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, num_layers = ( + self.get_mha_kv_ptrs_with_pp(src_ptrs, dst_ptrs) + ) + _, _, dst_k_lens, dst_v_lens, _ = self.get_mha_kv_ptrs_with_pp( + src_item_lens, dst_item_lens + ) + entries = list( + zip( + src_k_ptrs + src_v_ptrs, + dst_k_ptrs + dst_v_ptrs, + src_item_lens, + dst_k_lens + dst_v_lens, + ) + ) + # A single page-envelope region (unified memory) has no per-layer K/V + # split to slice heads from. + if len(entries) != len(src_ptrs) or 2 * num_layers != len(src_ptrs): + raise RuntimeError( + "Head-sliced PD transfer needs matching per-layer K/V buffers: " + f"prefill has {len(src_ptrs)} entries, decode has {len(dst_ptrs)}" + ) + if len(src_page_indices) != len(dst_page_indices): + raise RuntimeError( + "Head-sliced PD transfer page count mismatch: " + f"prefill={len(src_page_indices)}, decode={len(dst_page_indices)}" + ) + + total_kv_heads = resolve_total_kv_heads(self.kv_args, self.attn_tp_size) + src_head_start, num_heads, dst_head_start, _ = compute_head_slice_params( + src_attn_tp_size=self.attn_tp_size, + dst_attn_tp_size=dst_attn_tp_size, + src_tp_rank=self.kv_args.engine_rank, + dst_tp_rank=dst_tp_rank, + total_kv_heads=total_kv_heads, + ) + src_heads = max(1, total_kv_heads // self.attn_tp_size) + dst_heads = max(1, total_kv_heads // dst_attn_tp_size) + page_size = self.kv_args.page_size + tokens = np.arange(page_size, dtype=np.uint64) + src_pages = np.asarray(src_page_indices, dtype=np.uint64)[:, None] + dst_pages = np.asarray(dst_page_indices, dtype=np.uint64)[:, None] + + blocks = [] + for src_ptr, dst_ptr, src_item_len, dst_item_len in entries: + head_len = src_item_len // (page_size * src_heads) + # Both peers must hold whole heads of one size, or this state is + # sharded differently from total_kv_head_num. + if ( + head_len * page_size * src_heads != src_item_len + or head_len * page_size * dst_heads != dst_item_len + ): + raise RuntimeError( + f"KV item lengths (prefill={src_item_len}, decode={dst_item_len}) " + f"do not split into {src_heads} and {dst_heads} heads of " + f"{total_kv_heads} total with page_size={page_size}" + ) + # Each page item is [page_size, heads, head_dim]: one block per token. + src_addrs = ( + np.uint64(src_ptr) + + src_pages * np.uint64(src_item_len) + + tokens * np.uint64(src_heads * head_len) + + np.uint64(src_head_start * head_len) + ) + dst_addrs = ( + np.uint64(dst_ptr) + + dst_pages * np.uint64(dst_item_len) + + tokens * np.uint64(dst_heads * head_len) + + np.uint64(dst_head_start * head_len) + ) + blocks.append((src_addrs.ravel(), dst_addrs.ravel(), num_heads * head_len)) + return blocks + def get_mla_kv_ptrs_with_pp( self, src_kv_ptrs: List[int], diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index a07cfc5139..eb97bf514d 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -1966,12 +1966,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): and self.attn_tp_size != target_rank_registration_info.dst_attn_tp_size ) - if ( + slice_kv_heads = ( has_heterogeneous_attn_tp and not self.is_mla_backend and not self.is_hybrid_mla_backend and not is_qwen4_qsa_state - ): + ) + if slice_kv_heads and st != StateType.SWA: raise RuntimeError( f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet." ) @@ -2038,6 +2039,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): src_indices = src_indices[: len(dst_indices_local)] else: dst_indices_local = dst_indices_local[: len(src_indices)] + if slice_kv_heads: + rc = ( + self._send_swa_state_slice( + req=req, + prefill_swa_page_indices=src_indices, + src_state_data_ptrs=src_data_ptrs, + src_state_item_lens=src_item_lens, + dst_state_data_ptrs=dst_data_ptrs, + dst_swa_page_indices=dst_indices_local, + dst_state_item_lens=dst_item_lens, + dst_tp_rank=target_rank_registration_info.dst_tp_rank, + dst_attn_tp_size=( + target_rank_registration_info.dst_attn_tp_size + ), + executor=executor, + ) + or rc + ) + continue rc = ( self._send_kvcache_generic( mooncake_session_id=req.mooncake_session_id, @@ -2292,6 +2312,49 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): return self._transfer_data(req.mooncake_session_id, transfer_blocks) + def _send_swa_state_slice( + self, + req: TransferInfo, + prefill_swa_page_indices: List[int], + src_state_data_ptrs: List[int], + src_state_item_lens: List[int], + dst_state_data_ptrs: List[int], + dst_swa_page_indices: List[int], + dst_state_item_lens: List[int], + dst_tp_rank: int, + dst_attn_tp_size: int, + executor: concurrent.futures.ThreadPoolExecutor, + ) -> int: + blocks = self._get_mha_head_slice_blocks( + src_ptrs=src_state_data_ptrs, + src_item_lens=src_state_item_lens, + dst_ptrs=dst_state_data_ptrs, + dst_item_lens=dst_state_item_lens, + src_page_indices=np.asarray(prefill_swa_page_indices), + dst_page_indices=np.asarray(dst_swa_page_indices), + dst_attn_tp_size=dst_attn_tp_size, + dst_tp_rank=dst_tp_rank, + ) + logger.warning_once( + "Using SWA state head-slice transfer for different attention TP sizes: " + f"prefill={self.attn_tp_size}, decode={dst_attn_tp_size}." + ) + futures = [ + executor.submit( + self._transfer_data, + req.mooncake_session_id, + list( + zip( + src_addrs.tolist(), + dst_addrs.tolist(), + [length] * src_addrs.size, + ) + ), + ) + for src_addrs, dst_addrs, length in blocks + ] + return self._await_transfer_futures(futures) + def _poison_unsettled_send(self, room: int) -> None: """Keep the drain ack from firing after a submitted transfer failed. diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 5959701ec1..4c8abb9d1f 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -621,7 +621,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): f"(ptr=0x{ptr:x}, size={size})" ) - def set_kv_buffer_tensors(self, k_buffers: list, v_buffers: list, page_size: int): + def set_kv_buffer_tensors( + self, + k_buffers: list, + v_buffers: list, + page_size: int, + slot_layer_ids: Optional[List[int]] = None, + ): # NOTE: matches mooncake behavior -- staging buffers are now # created in __init__ (per-worker), independent of the kv # tensors. This setter only stashes the tensor metadata used by @@ -630,6 +636,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): "k_buffers": k_buffers, "v_buffers": v_buffers, "page_size": page_size, + "slot_layer_ids": list(slot_layer_ids or []), } def register_staging_room_bootstrap(self, room, bootstrap_infos, receiver): @@ -2608,6 +2615,63 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): raise Exception("Failed to post Mamba state slice transfer") return xfer_handle + def _send_swa_state_slice( + self, + peer_name: str, + prefill_swa_page_indices: List[int], + src_state_data_ptrs: List[int], + src_state_item_lens: List[int], + dst_state_data_ptrs: List[int], + dst_swa_page_indices: List[int], + dst_state_item_lens: List[int], + dst_gpu_id: int, + notif: str, + decode_tp_size: int, + decode_tp_rank: int, + ): + blocks = self._get_mha_head_slice_blocks( + src_ptrs=src_state_data_ptrs, + src_item_lens=src_state_item_lens, + dst_ptrs=dst_state_data_ptrs, + dst_item_lens=dst_state_item_lens, + src_page_indices=np.asarray(prefill_swa_page_indices), + dst_page_indices=np.asarray(dst_swa_page_indices), + dst_attn_tp_size=decode_tp_size, + dst_tp_rank=decode_tp_rank, + ) + logger.warning_once( + "Using SWA state head-slice transfer for different attention TP sizes: " + f"prefill={self.attn_tp_size}, decode={decode_tp_size}." + ) + src_reqs, dst_reqs = [], [] + for src_addrs, dst_addrs, length in blocks: + lengths = np.full(src_addrs.size, length, dtype=np.uint64) + src_reqs.append( + np.column_stack( + [src_addrs, lengths, np.full_like(lengths, self.kv_args.gpu_id)] + ) + ) + dst_reqs.append( + np.column_stack([dst_addrs, lengths, np.full_like(lengths, dst_gpu_id)]) + ) + if not src_reqs: + return None + src_descs = self.agent.get_xfer_descs(np.vstack(src_reqs), "VRAM") + dst_descs = self.agent.get_xfer_descs(np.vstack(dst_reqs), "VRAM") + xfer_handle = self.agent.initialize_xfer( + "WRITE", + src_descs, + dst_descs, + peer_name, + notif.encode("ascii"), + ) + if not xfer_handle: + raise Exception("Failed to create SWA state slice transfer") + state = self.agent.transfer(xfer_handle) + if state == "ERR": + raise Exception("Failed to post SWA state slice transfer") + return xfer_handle + def maybe_send_extra( self, peer_name: str, @@ -2743,6 +2807,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): state_type=st, force_flat=True, ) + elif ( + st == StateType.SWA + and not self.is_mla_backend + and self.attn_tp_size != decode_tp_size + ): + h = self._send_swa_state_slice( + peer_name=peer_name, + prefill_swa_page_indices=src_indices, + src_state_data_ptrs=src_ptrs, + src_state_item_lens=src_lens, + dst_state_data_ptrs=dst_ptrs, + dst_swa_page_indices=dst_indices, + dst_state_item_lens=dst_lens, + dst_gpu_id=dst_gpu_id, + notif=comp_notif, + decode_tp_size=decode_tp_size, + decode_tp_rank=decode_tp_rank, + ) elif st in ( StateType.SWA, StateType.BLOCK_SCALE, diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 140f82d48f..d78966c822 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -1392,6 +1392,7 @@ def setup_state_kv_args( from sglang.srt.mem_cache.memory_pool import ( DSATokenToKVPool, HybridLinearKVPool, + MHATokenToKVPool, MHATokenToKVPoolMXFP8, MiniMaxSparseKVPool, ) @@ -1407,6 +1408,7 @@ def setup_state_kv_args( kv_args.state_layer_ids = [] kv_args.is_hybrid_mla_backend = False kv_args.state_conv_shard_groups = [] + kv_args.swa_kv_cache_layout = None # V4's KVCache is organized by compression-ratio buckets rather than by layer. kv_args.mla_compression_ratios = ( list(token_to_kv_pool.compression_ratios) @@ -1457,6 +1459,12 @@ def setup_state_kv_args( append_state_component( kv_args, StateType.SWA, data_ptrs, data_lens, item_lens ) + if isinstance(token_to_kv_pool, SWAKVPool) and isinstance( + token_to_kv_pool.swa_kv_pool, MHATokenToKVPool + ): + kv_args.swa_kv_cache_layout = ( + token_to_kv_pool.swa_kv_pool.kv_cache_layout + ) # MXFP8 KV: each sub-pool's block scales ride as their own component # so they inherit the index payload of the KV they describe. # Only the concrete SWAKVPool owns a full sub-pool; other diff --git a/test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py b/test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py new file mode 100644 index 0000000000..51378aed1e --- /dev/null +++ b/test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py @@ -0,0 +1,43 @@ +"""Prefill TP2 -> decode TP1: two prefill ranks write disjoint KV heads per SWA page.""" + +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, + configure_nixl_pd_backend, +) +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE, + try_cached_model, +) + +register_cuda_ci(est_time=300, stage="extra-b", runner_config="4-gpu-h100") + +SWA_SERVER_ARGS = ["--page-size", "64", "--attention-backend", "triton"] + + +class TestDisaggregationSWAPrefillLargerTP(PDDisaggregationServerBase, GSM8KMixin): + prefill_tp_size = 2 + decode_tp_size = 1 + decode_base_gpu_id = 2 + extra_prefill_args = SWA_SERVER_ARGS + extra_decode_args = SWA_SERVER_ARGS + # Same eval and floor as the equal-TP case in + # test_disaggregation_decode_radix_cache_swa.py. + gsm8k_score_threshold = 0.45 + gsm8k_num_examples = 500 + gsm8k_num_threads = 100 + gsm8k_num_shots = 6 + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE) + configure_nixl_pd_backend(cls) + cls.launch_all() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 35d0ca91f1..96a2690a90 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -1,3 +1,4 @@ +import concurrent.futures import struct import threading import unittest @@ -37,6 +38,7 @@ from sglang.srt.disaggregation.mooncake.conn import ( MooncakeKVSender, TransferInfo, ) +from sglang.srt.disaggregation.nixl.conn import NixlKVManager from sglang.srt.disaggregation.utils import ( MetadataBuffers, build_transfer_entry_pairs, @@ -101,6 +103,7 @@ class TestMixedSWAKVBlockScales(CustomTestCase): MHATokenToKVPoolMXFP8 if quantized else MHATokenToKVPool ) pool.get_contiguous_buf_infos = Mock(return_value=swa_infos) + pool.kv_cache_layout = "nhd" if quantized: pool.get_kv_scale_buf_infos = Mock(return_value=scale_infos) setattr(target, name, pool) @@ -1062,6 +1065,249 @@ class TestQwen4StateWire(unittest.TestCase): ) +class _MemcpyNixlAgent: + """NIXL agent stand-in whose WRITE copies bytes within one host array.""" + + def __init__(self, memory: np.ndarray): + self.memory = memory + + def get_xfer_descs(self, reqs, mem_type): + return reqs + + def initialize_xfer(self, op, src_descs, dst_descs, peer_name, notif): + return src_descs, dst_descs + + def transfer(self, handle): + for (src, length, _), (dst, dst_length, _) in zip(*handle): + assert length == dst_length + src, dst, length = int(src), int(dst), int(length) + self.memory[dst : dst + length] = self.memory[src : src + length] + return "DONE" + + +class TestHeteroTpSwaStateTransfer(CustomTestCase): + PAGE_SIZE = 2 + HEAD_BYTES = 4 + NUM_ENTRIES = 4 # K then V of two SWA layers + NUM_PAGES = 6 + # The window occupies unrelated physical pages on the two sides. + SRC_PAGES = [4, 1, 5] + DST_PAGES = [2, 5, 0] + # (prefill attn TP, decode attn TP, total KV heads); the last three + # replicate KV heads across the attention TP ranks of one side. + TOPOLOGIES = [(2, 4, 8), (4, 2, 8), (1, 4, 2), (8, 2, 4), (2, 8, 4)] + + @staticmethod + def _rank_heads(rank, tp_size, total_heads): + # Model-side KV sharding: ranks beyond the head count replicate a head. + if total_heads >= tp_size: + per_rank = total_heads // tp_size + return list(range(rank * per_rank, (rank + 1) * per_rank)) + return [rank // (tp_size // total_heads)] + + @staticmethod + def _source_ranks(prefill_tp, decode_tp, dst_rank): + decode = object.__new__(CommonKVManager) + decode.attn_tp_size = decode_tp + decode.kv_args = SimpleNamespace(engine_rank=dst_rank) + decode.is_mla_backend = decode.is_hybrid_mla_backend = False + decode.attn_cp_size, decode.attn_cp_rank = 1, 0 + decode.pp_size, decode.pp_rank = 1, 0 + info = SimpleNamespace(attn_tp_size=prefill_tp, attn_cp_size=1, pp_size=1) + decode._resolve_rank_mapping(info) + return info.target_tp_ranks + + def _send_mooncake(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len): + def copy_blocks(session_id, blocks): + for src, dst, length in blocks: + memory[dst : dst + length] = memory[src : src + length] + return 0 + + prefill.is_hybrid_mla_backend = False + prefill.enable_deferred_decode_kv_release = False + prefill._transfer_data = copy_blocks + req = SimpleNamespace( + mooncake_session_id="decode", dst_state_indices=[self.DST_PAGES] + ) + dst_info = SimpleNamespace( + dst_state_data_ptrs=[dst_ptrs], + dst_state_item_lens=[[item_len] * len(dst_ptrs)], + dst_state_dim_per_tensor=[[]], + dst_state_layer_ids=[[]], + dst_attn_tp_size=decode_tp, + dst_tp_rank=dst_rank, + ) + with concurrent.futures.ThreadPoolExecutor(1) as executor: + rc = prefill.maybe_send_extra( + req, [np.array(self.SRC_PAGES, dtype=np.int32)], executor, dst_info + ) + self.assertEqual(rc, 0) + + def _send_nixl(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len): + prefill.agent = _MemcpyNixlAgent(memory) + handles = prefill.maybe_send_extra( + "decode", + [np.array(self.SRC_PAGES, dtype=np.int32)], + [dst_ptrs], + [self.DST_PAGES], + 0, + "7_state_0", + decode_tp, + decode_tp_rank=dst_rank, + dst_state_item_lens=[[item_len] * len(dst_ptrs)], + ) + self.assertEqual(len(handles), 1) + + def _prefill_manager( + self, *, backend, src_rank, prefill_tp, total_heads, src_ptrs, item_len + ): + prefill = object.__new__(backend) + prefill.attn_tp_size = prefill_tp + prefill.is_mla_backend = False + prefill.pp_size = 1 + prefill.kv_args = SimpleNamespace( + engine_rank=src_rank, + gpu_id=0, + page_size=self.PAGE_SIZE, + total_kv_head_num=total_heads, + prefill_start_layer=0, + state_types=[StateType.SWA], + state_data_ptrs=[src_ptrs], + state_item_lens=[[item_len] * len(src_ptrs)], + state_dim_per_tensor=[[]], + state_layer_ids=[[]], + swa_kv_cache_layout="nhd", + ) + return prefill + + def _pool_shape(self, tp_size, total_heads): + # [rank, entry, page, token, local head, byte] + heads = len(self._rank_heads(rank=0, tp_size=tp_size, total_heads=total_heads)) + return ( + tp_size, + self.NUM_ENTRIES, + self.NUM_PAGES, + self.PAGE_SIZE, + heads, + self.HEAD_BYTES, + ) + + def _check_transfer(self, backend, send, prefill_tp, decode_tp, total_heads): + rng = np.random.default_rng(0) + window_shape = (self.NUM_ENTRIES, len(self.SRC_PAGES), self.PAGE_SIZE) + window = rng.integers( + 0, 256, window_shape + (total_heads, self.HEAD_BYTES), dtype=np.uint8 + ) + src_shape = self._pool_shape(tp_size=prefill_tp, total_heads=total_heads) + dst_shape = self._pool_shape(tp_size=decode_tp, total_heads=total_heads) + src_bytes = int(np.prod(src_shape)) + # Prefill pages outside the window hold bytes a wrong page stride copies. + memory = np.concatenate( + [ + rng.integers(0, 256, src_bytes, dtype=np.uint8), + np.zeros(int(np.prod(dst_shape)), dtype=np.uint8), + ] + ) + src_pools = memory[:src_bytes].reshape(src_shape) + dst_pools = memory[src_bytes:].reshape(dst_shape) + expected = np.zeros_like(dst_pools) + for rank in range(prefill_tp): + heads = self._rank_heads( + rank=rank, tp_size=prefill_tp, total_heads=total_heads + ) + src_pools[rank][:, self.SRC_PAGES] = window[..., heads, :] + for rank in range(decode_tp): + heads = self._rank_heads( + rank=rank, tp_size=decode_tp, total_heads=total_heads + ) + expected[rank][:, self.DST_PAGES] = window[..., heads, :] + + src_item_len = int(np.prod(src_shape[3:])) + dst_item_len = int(np.prod(dst_shape[3:])) + for dst_rank in range(decode_tp): + dst_ptrs = [ + src_bytes + + (dst_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * dst_item_len + for e in range(self.NUM_ENTRIES) + ] + for src_rank in self._source_ranks( + prefill_tp=prefill_tp, decode_tp=decode_tp, dst_rank=dst_rank + ): + src_ptrs = [ + (src_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * src_item_len + for e in range(self.NUM_ENTRIES) + ] + prefill = self._prefill_manager( + backend=backend, + src_rank=src_rank, + prefill_tp=prefill_tp, + total_heads=total_heads, + src_ptrs=src_ptrs, + item_len=src_item_len, + ) + send( + memory=memory, + prefill=prefill, + dst_rank=dst_rank, + decode_tp=decode_tp, + dst_ptrs=dst_ptrs, + item_len=dst_item_len, + ) + + np.testing.assert_array_equal(dst_pools, expected) + + def test_decode_ranks_receive_their_heads_of_the_window(self): + for backend, send in ( + (MooncakeKVManager, self._send_mooncake), + (NixlKVManager, self._send_nixl), + ): + for prefill_tp, decode_tp, total_heads in self.TOPOLOGIES: + with self.subTest( + backend=backend.__name__, + prefill_tp=prefill_tp, + decode_tp=decode_tp, + total_heads=total_heads, + ): + self._check_transfer( + backend=backend, + send=send, + prefill_tp=prefill_tp, + decode_tp=decode_tp, + total_heads=total_heads, + ) + + def test_unsliceable_layouts_are_rejected(self): + prefill = object.__new__(CommonKVManager) + prefill.attn_tp_size = 2 + prefill.kv_args = SimpleNamespace( + engine_rank=0, page_size=2, total_kv_head_num=8, prefill_start_layer=0 + ) + for pp_size, layout, src_item_lens, dst_item_lens, message in ( + # SWA state has no layer ids, so a later PP stage would land on the + # decode peer's first SWA layers. + (2, "nhd", [16, 16], [8, 8], "pipeline parallelism"), + (1, "nhd", [64], [32], "per-layer K/V"), + # One SWA head per rank on both sides, against 8 total KV heads. + (1, "nhd", [8, 8], [8, 8], "do not split"), + # Item lengths that are valid NHD, so only the layout can reject. + (1, "vectorized_5d", [16, 16], [8, 8], "NHD KV cache layout"), + ): + prefill.pp_size = pp_size + prefill.kv_args.swa_kv_cache_layout = layout + with self.subTest(message=message): + with self.assertRaisesRegex(RuntimeError, message): + prefill._get_mha_head_slice_blocks( + src_ptrs=[0x1000 * i for i in range(len(src_item_lens))], + src_item_lens=src_item_lens, + dst_ptrs=[0x9000 * i for i in range(len(dst_item_lens))], + dst_item_lens=dst_item_lens, + src_page_indices=np.array([1]), + dst_page_indices=np.array([2]), + dst_attn_tp_size=4, + dst_tp_rank=0, + ) + + class TestMooncakeTransferInfoIsDummy(unittest.TestCase): """Truth table for mooncake's payload-inferred is_dummy, with frames built as KVSender sends them: kv and aux are empty iff dummy, state indices are diff --git a/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py b/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py index 8350b5d54c..27c0fe6a8a 100644 --- a/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py +++ b/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py @@ -9,6 +9,8 @@ draft pool exists. import unittest +from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager +from sglang.srt.disaggregation.nixl.conn import NixlKVManager from sglang.srt.disaggregation.utils import ( build_staging_slot_metadata, build_transfer_entry_pairs, @@ -50,6 +52,15 @@ class TestStagingDraftKvSlots(CustomTestCase): self.assertEqual(slot_ids, [87, 91, 92, 87, 91, 92]) self.assertNotEqual(slot_ids, _kv_layer_ids(target, draft)) + def test_every_backend_setter_accepts_slot_layer_ids(self): + # Shared staging init passes slot_layer_ids to every backend's setter. + for backend in (MooncakeKVManager, NixlKVManager): + with self.subTest(backend=backend.__name__): + manager = object.__new__(backend) + manager.set_kv_buffer_tensors( + ["tK87"], ["tV87"], 16, slot_layer_ids=[87, 87] + ) + def test_without_draft_matches_kv_layer_ids(self): # The two orders coincide with no draft pool, so every deployment that # predates draft KV must keep its exact slot labelling.