From 3eebf987ed622395f8fae3ed99ec6a1ed1ffb83e Mon Sep 17 00:00:00 2001
From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
Date: Mon, 12 Oct 2026 23:21:54 -0700
Subject: [PATCH] [PD] Support heterogeneous TP for hybrid SWA KV transfer
(#40909)
---
.../advanced_features/pd_disaggregation.mdx | 2 +
python/sglang/srt/disaggregation/base/conn.py | 3 +
.../sglang/srt/disaggregation/common/conn.py | 122 +++++++++
.../srt/disaggregation/mooncake/conn.py | 67 ++++-
python/sglang/srt/disaggregation/nixl/conn.py | 84 +++++-
python/sglang/srt/disaggregation/utils.py | 8 +
.../test_disaggregation_swa_different_tp.py | 43 +++
.../test_disaggregation_wire.py | 246 ++++++++++++++++++
.../test_staging_draft_kv_slots.py | 11 +
9 files changed, 583 insertions(+), 3 deletions(-)
create mode 100644 test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py
diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx
index 17fa62441d..154b4070b9 100644
--- a/docs/docs/advanced_features/pd_disaggregation.mdx
+++ b/docs/docs/advanced_features/pd_disaggregation.mdx
@@ -347,6 +347,8 @@ Enable the staging buffer when prefill and decode use **different TP sizes** wit
> **Note:** The staging buffer is designed for non-MLA models (e.g. GQA, MHA). MLA models (e.g. DeepSeek-V2/V3) should not enable this flag.
+> **Note:** You can use different attention TP sizes for hybrid sliding-window models (e.g. `openai/gpt-oss-120b`) with Mooncake or NIXL. For unequal TP, both backends transfer the sliding-window KV by token and KV-head slice. The staging buffer carries only full-attention KV. Use `--pp-size 1` and leave `--enable-unified-memory` disabled for this configuration. MXFP8 KV cache is not supported for unequal-TP SWA transfer.
+
### Environment Variables
diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py
index e9a057cf44..d31682c773 100644
--- a/python/sglang/srt/disaggregation/base/conn.py
+++ b/python/sglang/srt/disaggregation/base/conn.py
@@ -81,6 +81,9 @@ class KVArgs:
# scatter transfer can slice each independently head-sharded sub-block; None
# per tensor when the single contiguous slice already matches the layout.
state_conv_shard_groups: List[List[Optional[List[int]]]]
+ # Physical layout of the SWA sub-pool ("nhd", "hnd" or "vectorized_5d");
+ # None when the SWA state is not held in an MHA sub-pool.
+ swa_kv_cache_layout: Optional[str]
ib_device: str
gpu_id: int
kv_head_num: int
diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py
index db2b6d14ac..ee137d1e3b 100644
--- a/python/sglang/srt/disaggregation/common/conn.py
+++ b/python/sglang/srt/disaggregation/common/conn.py
@@ -1230,6 +1230,26 @@ class CommonKVManager(BaseKVManager):
"servers (including prefill CP ranks for MLA)"
)
+ # Every send path rejects head-sharded MXFP8 block scales across
+ # attention TP sizes, so each request would fail at its first transfer.
+ if (
+ info.attn_tp_size != self.attn_tp_size
+ and not self.is_mla_backend
+ and not self.is_hybrid_mla_backend
+ ):
+ block_scale_states = [
+ st
+ for st in getattr(self.kv_args, "state_types", None) or []
+ if st in (StateType.BLOCK_SCALE, StateType.BLOCK_SCALE_SWA)
+ ]
+ if block_scale_states:
+ raise RuntimeError(
+ "PD Disaggregation does NOT support different attention TP "
+ "sizes with MXFP8 KV cache: prefill attn_tp_size="
+ f"{info.attn_tp_size}, decode attn_tp_size={self.attn_tp_size}, "
+ f"state types {[st.value for st in block_scale_states]}."
+ )
+
if self.dcp_size > 1:
if not (self.is_mla_backend or self.is_hybrid_mla_backend):
raise RuntimeError(
@@ -1541,6 +1561,108 @@ class CommonKVManager(BaseKVManager):
layers_current_pp_stage = len(src_k_ptrs)
return src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage
+ def _get_mha_head_slice_blocks(
+ self,
+ src_ptrs: List[int],
+ src_item_lens: List[int],
+ dst_ptrs: List[int],
+ dst_item_lens: List[int],
+ src_page_indices: npt.NDArray[np.int32],
+ dst_page_indices: npt.NDArray[np.int32],
+ dst_attn_tp_size: int,
+ dst_tp_rank: int,
+ ) -> List[Tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint64], int]]:
+ # SWA state has no layer ids and its pool starts every PP stage at layer
+ # 0, so a prefill stage's entries cannot be paired with the decode layers.
+ if self.pp_size > 1:
+ raise RuntimeError(
+ "PD Disaggregation does NOT support PD different TP sizes for "
+ "non-MLA SWA hybrid models with pipeline parallelism yet."
+ )
+ # The addresses below assume NHD pages [page_size, heads, head_dim]; HND
+ # and vectorized_5d pages have the same size in another element order.
+ if self.kv_args.swa_kv_cache_layout != "nhd":
+ raise RuntimeError(
+ "Head-sliced PD transfer needs the NHD KV cache layout, got "
+ f"{self.kv_args.swa_kv_cache_layout}"
+ )
+ from sglang.srt.disaggregation.common.staging_buffer import (
+ compute_head_slice_params,
+ resolve_total_kv_heads,
+ )
+
+ src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, num_layers = (
+ self.get_mha_kv_ptrs_with_pp(src_ptrs, dst_ptrs)
+ )
+ _, _, dst_k_lens, dst_v_lens, _ = self.get_mha_kv_ptrs_with_pp(
+ src_item_lens, dst_item_lens
+ )
+ entries = list(
+ zip(
+ src_k_ptrs + src_v_ptrs,
+ dst_k_ptrs + dst_v_ptrs,
+ src_item_lens,
+ dst_k_lens + dst_v_lens,
+ )
+ )
+ # A single page-envelope region (unified memory) has no per-layer K/V
+ # split to slice heads from.
+ if len(entries) != len(src_ptrs) or 2 * num_layers != len(src_ptrs):
+ raise RuntimeError(
+ "Head-sliced PD transfer needs matching per-layer K/V buffers: "
+ f"prefill has {len(src_ptrs)} entries, decode has {len(dst_ptrs)}"
+ )
+ if len(src_page_indices) != len(dst_page_indices):
+ raise RuntimeError(
+ "Head-sliced PD transfer page count mismatch: "
+ f"prefill={len(src_page_indices)}, decode={len(dst_page_indices)}"
+ )
+
+ total_kv_heads = resolve_total_kv_heads(self.kv_args, self.attn_tp_size)
+ src_head_start, num_heads, dst_head_start, _ = compute_head_slice_params(
+ src_attn_tp_size=self.attn_tp_size,
+ dst_attn_tp_size=dst_attn_tp_size,
+ src_tp_rank=self.kv_args.engine_rank,
+ dst_tp_rank=dst_tp_rank,
+ total_kv_heads=total_kv_heads,
+ )
+ src_heads = max(1, total_kv_heads // self.attn_tp_size)
+ dst_heads = max(1, total_kv_heads // dst_attn_tp_size)
+ page_size = self.kv_args.page_size
+ tokens = np.arange(page_size, dtype=np.uint64)
+ src_pages = np.asarray(src_page_indices, dtype=np.uint64)[:, None]
+ dst_pages = np.asarray(dst_page_indices, dtype=np.uint64)[:, None]
+
+ blocks = []
+ for src_ptr, dst_ptr, src_item_len, dst_item_len in entries:
+ head_len = src_item_len // (page_size * src_heads)
+ # Both peers must hold whole heads of one size, or this state is
+ # sharded differently from total_kv_head_num.
+ if (
+ head_len * page_size * src_heads != src_item_len
+ or head_len * page_size * dst_heads != dst_item_len
+ ):
+ raise RuntimeError(
+ f"KV item lengths (prefill={src_item_len}, decode={dst_item_len}) "
+ f"do not split into {src_heads} and {dst_heads} heads of "
+ f"{total_kv_heads} total with page_size={page_size}"
+ )
+ # Each page item is [page_size, heads, head_dim]: one block per token.
+ src_addrs = (
+ np.uint64(src_ptr)
+ + src_pages * np.uint64(src_item_len)
+ + tokens * np.uint64(src_heads * head_len)
+ + np.uint64(src_head_start * head_len)
+ )
+ dst_addrs = (
+ np.uint64(dst_ptr)
+ + dst_pages * np.uint64(dst_item_len)
+ + tokens * np.uint64(dst_heads * head_len)
+ + np.uint64(dst_head_start * head_len)
+ )
+ blocks.append((src_addrs.ravel(), dst_addrs.ravel(), num_heads * head_len))
+ return blocks
+
def get_mla_kv_ptrs_with_pp(
self,
src_kv_ptrs: List[int],
diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py
index a07cfc5139..eb97bf514d 100644
--- a/python/sglang/srt/disaggregation/mooncake/conn.py
+++ b/python/sglang/srt/disaggregation/mooncake/conn.py
@@ -1966,12 +1966,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
and self.attn_tp_size
!= target_rank_registration_info.dst_attn_tp_size
)
- if (
+ slice_kv_heads = (
has_heterogeneous_attn_tp
and not self.is_mla_backend
and not self.is_hybrid_mla_backend
and not is_qwen4_qsa_state
- ):
+ )
+ if slice_kv_heads and st != StateType.SWA:
raise RuntimeError(
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
)
@@ -2038,6 +2039,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_indices = src_indices[: len(dst_indices_local)]
else:
dst_indices_local = dst_indices_local[: len(src_indices)]
+ if slice_kv_heads:
+ rc = (
+ self._send_swa_state_slice(
+ req=req,
+ prefill_swa_page_indices=src_indices,
+ src_state_data_ptrs=src_data_ptrs,
+ src_state_item_lens=src_item_lens,
+ dst_state_data_ptrs=dst_data_ptrs,
+ dst_swa_page_indices=dst_indices_local,
+ dst_state_item_lens=dst_item_lens,
+ dst_tp_rank=target_rank_registration_info.dst_tp_rank,
+ dst_attn_tp_size=(
+ target_rank_registration_info.dst_attn_tp_size
+ ),
+ executor=executor,
+ )
+ or rc
+ )
+ continue
rc = (
self._send_kvcache_generic(
mooncake_session_id=req.mooncake_session_id,
@@ -2292,6 +2312,49 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
return self._transfer_data(req.mooncake_session_id, transfer_blocks)
+ def _send_swa_state_slice(
+ self,
+ req: TransferInfo,
+ prefill_swa_page_indices: List[int],
+ src_state_data_ptrs: List[int],
+ src_state_item_lens: List[int],
+ dst_state_data_ptrs: List[int],
+ dst_swa_page_indices: List[int],
+ dst_state_item_lens: List[int],
+ dst_tp_rank: int,
+ dst_attn_tp_size: int,
+ executor: concurrent.futures.ThreadPoolExecutor,
+ ) -> int:
+ blocks = self._get_mha_head_slice_blocks(
+ src_ptrs=src_state_data_ptrs,
+ src_item_lens=src_state_item_lens,
+ dst_ptrs=dst_state_data_ptrs,
+ dst_item_lens=dst_state_item_lens,
+ src_page_indices=np.asarray(prefill_swa_page_indices),
+ dst_page_indices=np.asarray(dst_swa_page_indices),
+ dst_attn_tp_size=dst_attn_tp_size,
+ dst_tp_rank=dst_tp_rank,
+ )
+ logger.warning_once(
+ "Using SWA state head-slice transfer for different attention TP sizes: "
+ f"prefill={self.attn_tp_size}, decode={dst_attn_tp_size}."
+ )
+ futures = [
+ executor.submit(
+ self._transfer_data,
+ req.mooncake_session_id,
+ list(
+ zip(
+ src_addrs.tolist(),
+ dst_addrs.tolist(),
+ [length] * src_addrs.size,
+ )
+ ),
+ )
+ for src_addrs, dst_addrs, length in blocks
+ ]
+ return self._await_transfer_futures(futures)
+
def _poison_unsettled_send(self, room: int) -> None:
"""Keep the drain ack from firing after a submitted transfer failed.
diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py
index 5959701ec1..4c8abb9d1f 100644
--- a/python/sglang/srt/disaggregation/nixl/conn.py
+++ b/python/sglang/srt/disaggregation/nixl/conn.py
@@ -621,7 +621,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
f"(ptr=0x{ptr:x}, size={size})"
)
- def set_kv_buffer_tensors(self, k_buffers: list, v_buffers: list, page_size: int):
+ def set_kv_buffer_tensors(
+ self,
+ k_buffers: list,
+ v_buffers: list,
+ page_size: int,
+ slot_layer_ids: Optional[List[int]] = None,
+ ):
# NOTE: matches mooncake behavior -- staging buffers are now
# created in __init__ (per-worker), independent of the kv
# tensors. This setter only stashes the tensor metadata used by
@@ -630,6 +636,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
"k_buffers": k_buffers,
"v_buffers": v_buffers,
"page_size": page_size,
+ "slot_layer_ids": list(slot_layer_ids or []),
}
def register_staging_room_bootstrap(self, room, bootstrap_infos, receiver):
@@ -2608,6 +2615,63 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
raise Exception("Failed to post Mamba state slice transfer")
return xfer_handle
+ def _send_swa_state_slice(
+ self,
+ peer_name: str,
+ prefill_swa_page_indices: List[int],
+ src_state_data_ptrs: List[int],
+ src_state_item_lens: List[int],
+ dst_state_data_ptrs: List[int],
+ dst_swa_page_indices: List[int],
+ dst_state_item_lens: List[int],
+ dst_gpu_id: int,
+ notif: str,
+ decode_tp_size: int,
+ decode_tp_rank: int,
+ ):
+ blocks = self._get_mha_head_slice_blocks(
+ src_ptrs=src_state_data_ptrs,
+ src_item_lens=src_state_item_lens,
+ dst_ptrs=dst_state_data_ptrs,
+ dst_item_lens=dst_state_item_lens,
+ src_page_indices=np.asarray(prefill_swa_page_indices),
+ dst_page_indices=np.asarray(dst_swa_page_indices),
+ dst_attn_tp_size=decode_tp_size,
+ dst_tp_rank=decode_tp_rank,
+ )
+ logger.warning_once(
+ "Using SWA state head-slice transfer for different attention TP sizes: "
+ f"prefill={self.attn_tp_size}, decode={decode_tp_size}."
+ )
+ src_reqs, dst_reqs = [], []
+ for src_addrs, dst_addrs, length in blocks:
+ lengths = np.full(src_addrs.size, length, dtype=np.uint64)
+ src_reqs.append(
+ np.column_stack(
+ [src_addrs, lengths, np.full_like(lengths, self.kv_args.gpu_id)]
+ )
+ )
+ dst_reqs.append(
+ np.column_stack([dst_addrs, lengths, np.full_like(lengths, dst_gpu_id)])
+ )
+ if not src_reqs:
+ return None
+ src_descs = self.agent.get_xfer_descs(np.vstack(src_reqs), "VRAM")
+ dst_descs = self.agent.get_xfer_descs(np.vstack(dst_reqs), "VRAM")
+ xfer_handle = self.agent.initialize_xfer(
+ "WRITE",
+ src_descs,
+ dst_descs,
+ peer_name,
+ notif.encode("ascii"),
+ )
+ if not xfer_handle:
+ raise Exception("Failed to create SWA state slice transfer")
+ state = self.agent.transfer(xfer_handle)
+ if state == "ERR":
+ raise Exception("Failed to post SWA state slice transfer")
+ return xfer_handle
+
def maybe_send_extra(
self,
peer_name: str,
@@ -2743,6 +2807,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
state_type=st,
force_flat=True,
)
+ elif (
+ st == StateType.SWA
+ and not self.is_mla_backend
+ and self.attn_tp_size != decode_tp_size
+ ):
+ h = self._send_swa_state_slice(
+ peer_name=peer_name,
+ prefill_swa_page_indices=src_indices,
+ src_state_data_ptrs=src_ptrs,
+ src_state_item_lens=src_lens,
+ dst_state_data_ptrs=dst_ptrs,
+ dst_swa_page_indices=dst_indices,
+ dst_state_item_lens=dst_lens,
+ dst_gpu_id=dst_gpu_id,
+ notif=comp_notif,
+ decode_tp_size=decode_tp_size,
+ decode_tp_rank=decode_tp_rank,
+ )
elif st in (
StateType.SWA,
StateType.BLOCK_SCALE,
diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py
index 140f82d48f..d78966c822 100644
--- a/python/sglang/srt/disaggregation/utils.py
+++ b/python/sglang/srt/disaggregation/utils.py
@@ -1392,6 +1392,7 @@ def setup_state_kv_args(
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
HybridLinearKVPool,
+ MHATokenToKVPool,
MHATokenToKVPoolMXFP8,
MiniMaxSparseKVPool,
)
@@ -1407,6 +1408,7 @@ def setup_state_kv_args(
kv_args.state_layer_ids = []
kv_args.is_hybrid_mla_backend = False
kv_args.state_conv_shard_groups = []
+ kv_args.swa_kv_cache_layout = None
# V4's KVCache is organized by compression-ratio buckets rather than by layer.
kv_args.mla_compression_ratios = (
list(token_to_kv_pool.compression_ratios)
@@ -1457,6 +1459,12 @@ def setup_state_kv_args(
append_state_component(
kv_args, StateType.SWA, data_ptrs, data_lens, item_lens
)
+ if isinstance(token_to_kv_pool, SWAKVPool) and isinstance(
+ token_to_kv_pool.swa_kv_pool, MHATokenToKVPool
+ ):
+ kv_args.swa_kv_cache_layout = (
+ token_to_kv_pool.swa_kv_pool.kv_cache_layout
+ )
# MXFP8 KV: each sub-pool's block scales ride as their own component
# so they inherit the index payload of the KV they describe.
# Only the concrete SWAKVPool owns a full sub-pool; other
diff --git a/test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py b/test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py
new file mode 100644
index 0000000000..51378aed1e
--- /dev/null
+++ b/test/registered/e2e/disaggregation/test_disaggregation_swa_different_tp.py
@@ -0,0 +1,43 @@
+"""Prefill TP2 -> decode TP1: two prefill ranks write disjoint KV heads per SWA page."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
+from sglang.test.server_fixtures.disaggregation_fixture import (
+ PDDisaggregationServerBase,
+ configure_nixl_pd_backend,
+)
+from sglang.test.test_utils import (
+ DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE,
+ try_cached_model,
+)
+
+register_cuda_ci(est_time=300, stage="extra-b", runner_config="4-gpu-h100")
+
+SWA_SERVER_ARGS = ["--page-size", "64", "--attention-backend", "triton"]
+
+
+class TestDisaggregationSWAPrefillLargerTP(PDDisaggregationServerBase, GSM8KMixin):
+ prefill_tp_size = 2
+ decode_tp_size = 1
+ decode_base_gpu_id = 2
+ extra_prefill_args = SWA_SERVER_ARGS
+ extra_decode_args = SWA_SERVER_ARGS
+ # Same eval and floor as the equal-TP case in
+ # test_disaggregation_decode_radix_cache_swa.py.
+ gsm8k_score_threshold = 0.45
+ gsm8k_num_examples = 500
+ gsm8k_num_threads = 100
+ gsm8k_num_shots = 6
+
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE)
+ configure_nixl_pd_backend(cls)
+ cls.launch_all()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py
index 35d0ca91f1..96a2690a90 100644
--- a/test/registered/unit/disaggregation/test_disaggregation_wire.py
+++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py
@@ -1,3 +1,4 @@
+import concurrent.futures
import struct
import threading
import unittest
@@ -37,6 +38,7 @@ from sglang.srt.disaggregation.mooncake.conn import (
MooncakeKVSender,
TransferInfo,
)
+from sglang.srt.disaggregation.nixl.conn import NixlKVManager
from sglang.srt.disaggregation.utils import (
MetadataBuffers,
build_transfer_entry_pairs,
@@ -101,6 +103,7 @@ class TestMixedSWAKVBlockScales(CustomTestCase):
MHATokenToKVPoolMXFP8 if quantized else MHATokenToKVPool
)
pool.get_contiguous_buf_infos = Mock(return_value=swa_infos)
+ pool.kv_cache_layout = "nhd"
if quantized:
pool.get_kv_scale_buf_infos = Mock(return_value=scale_infos)
setattr(target, name, pool)
@@ -1062,6 +1065,249 @@ class TestQwen4StateWire(unittest.TestCase):
)
+class _MemcpyNixlAgent:
+ """NIXL agent stand-in whose WRITE copies bytes within one host array."""
+
+ def __init__(self, memory: np.ndarray):
+ self.memory = memory
+
+ def get_xfer_descs(self, reqs, mem_type):
+ return reqs
+
+ def initialize_xfer(self, op, src_descs, dst_descs, peer_name, notif):
+ return src_descs, dst_descs
+
+ def transfer(self, handle):
+ for (src, length, _), (dst, dst_length, _) in zip(*handle):
+ assert length == dst_length
+ src, dst, length = int(src), int(dst), int(length)
+ self.memory[dst : dst + length] = self.memory[src : src + length]
+ return "DONE"
+
+
+class TestHeteroTpSwaStateTransfer(CustomTestCase):
+ PAGE_SIZE = 2
+ HEAD_BYTES = 4
+ NUM_ENTRIES = 4 # K then V of two SWA layers
+ NUM_PAGES = 6
+ # The window occupies unrelated physical pages on the two sides.
+ SRC_PAGES = [4, 1, 5]
+ DST_PAGES = [2, 5, 0]
+ # (prefill attn TP, decode attn TP, total KV heads); the last three
+ # replicate KV heads across the attention TP ranks of one side.
+ TOPOLOGIES = [(2, 4, 8), (4, 2, 8), (1, 4, 2), (8, 2, 4), (2, 8, 4)]
+
+ @staticmethod
+ def _rank_heads(rank, tp_size, total_heads):
+ # Model-side KV sharding: ranks beyond the head count replicate a head.
+ if total_heads >= tp_size:
+ per_rank = total_heads // tp_size
+ return list(range(rank * per_rank, (rank + 1) * per_rank))
+ return [rank // (tp_size // total_heads)]
+
+ @staticmethod
+ def _source_ranks(prefill_tp, decode_tp, dst_rank):
+ decode = object.__new__(CommonKVManager)
+ decode.attn_tp_size = decode_tp
+ decode.kv_args = SimpleNamespace(engine_rank=dst_rank)
+ decode.is_mla_backend = decode.is_hybrid_mla_backend = False
+ decode.attn_cp_size, decode.attn_cp_rank = 1, 0
+ decode.pp_size, decode.pp_rank = 1, 0
+ info = SimpleNamespace(attn_tp_size=prefill_tp, attn_cp_size=1, pp_size=1)
+ decode._resolve_rank_mapping(info)
+ return info.target_tp_ranks
+
+ def _send_mooncake(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len):
+ def copy_blocks(session_id, blocks):
+ for src, dst, length in blocks:
+ memory[dst : dst + length] = memory[src : src + length]
+ return 0
+
+ prefill.is_hybrid_mla_backend = False
+ prefill.enable_deferred_decode_kv_release = False
+ prefill._transfer_data = copy_blocks
+ req = SimpleNamespace(
+ mooncake_session_id="decode", dst_state_indices=[self.DST_PAGES]
+ )
+ dst_info = SimpleNamespace(
+ dst_state_data_ptrs=[dst_ptrs],
+ dst_state_item_lens=[[item_len] * len(dst_ptrs)],
+ dst_state_dim_per_tensor=[[]],
+ dst_state_layer_ids=[[]],
+ dst_attn_tp_size=decode_tp,
+ dst_tp_rank=dst_rank,
+ )
+ with concurrent.futures.ThreadPoolExecutor(1) as executor:
+ rc = prefill.maybe_send_extra(
+ req, [np.array(self.SRC_PAGES, dtype=np.int32)], executor, dst_info
+ )
+ self.assertEqual(rc, 0)
+
+ def _send_nixl(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len):
+ prefill.agent = _MemcpyNixlAgent(memory)
+ handles = prefill.maybe_send_extra(
+ "decode",
+ [np.array(self.SRC_PAGES, dtype=np.int32)],
+ [dst_ptrs],
+ [self.DST_PAGES],
+ 0,
+ "7_state_0",
+ decode_tp,
+ decode_tp_rank=dst_rank,
+ dst_state_item_lens=[[item_len] * len(dst_ptrs)],
+ )
+ self.assertEqual(len(handles), 1)
+
+ def _prefill_manager(
+ self, *, backend, src_rank, prefill_tp, total_heads, src_ptrs, item_len
+ ):
+ prefill = object.__new__(backend)
+ prefill.attn_tp_size = prefill_tp
+ prefill.is_mla_backend = False
+ prefill.pp_size = 1
+ prefill.kv_args = SimpleNamespace(
+ engine_rank=src_rank,
+ gpu_id=0,
+ page_size=self.PAGE_SIZE,
+ total_kv_head_num=total_heads,
+ prefill_start_layer=0,
+ state_types=[StateType.SWA],
+ state_data_ptrs=[src_ptrs],
+ state_item_lens=[[item_len] * len(src_ptrs)],
+ state_dim_per_tensor=[[]],
+ state_layer_ids=[[]],
+ swa_kv_cache_layout="nhd",
+ )
+ return prefill
+
+ def _pool_shape(self, tp_size, total_heads):
+ # [rank, entry, page, token, local head, byte]
+ heads = len(self._rank_heads(rank=0, tp_size=tp_size, total_heads=total_heads))
+ return (
+ tp_size,
+ self.NUM_ENTRIES,
+ self.NUM_PAGES,
+ self.PAGE_SIZE,
+ heads,
+ self.HEAD_BYTES,
+ )
+
+ def _check_transfer(self, backend, send, prefill_tp, decode_tp, total_heads):
+ rng = np.random.default_rng(0)
+ window_shape = (self.NUM_ENTRIES, len(self.SRC_PAGES), self.PAGE_SIZE)
+ window = rng.integers(
+ 0, 256, window_shape + (total_heads, self.HEAD_BYTES), dtype=np.uint8
+ )
+ src_shape = self._pool_shape(tp_size=prefill_tp, total_heads=total_heads)
+ dst_shape = self._pool_shape(tp_size=decode_tp, total_heads=total_heads)
+ src_bytes = int(np.prod(src_shape))
+ # Prefill pages outside the window hold bytes a wrong page stride copies.
+ memory = np.concatenate(
+ [
+ rng.integers(0, 256, src_bytes, dtype=np.uint8),
+ np.zeros(int(np.prod(dst_shape)), dtype=np.uint8),
+ ]
+ )
+ src_pools = memory[:src_bytes].reshape(src_shape)
+ dst_pools = memory[src_bytes:].reshape(dst_shape)
+ expected = np.zeros_like(dst_pools)
+ for rank in range(prefill_tp):
+ heads = self._rank_heads(
+ rank=rank, tp_size=prefill_tp, total_heads=total_heads
+ )
+ src_pools[rank][:, self.SRC_PAGES] = window[..., heads, :]
+ for rank in range(decode_tp):
+ heads = self._rank_heads(
+ rank=rank, tp_size=decode_tp, total_heads=total_heads
+ )
+ expected[rank][:, self.DST_PAGES] = window[..., heads, :]
+
+ src_item_len = int(np.prod(src_shape[3:]))
+ dst_item_len = int(np.prod(dst_shape[3:]))
+ for dst_rank in range(decode_tp):
+ dst_ptrs = [
+ src_bytes
+ + (dst_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * dst_item_len
+ for e in range(self.NUM_ENTRIES)
+ ]
+ for src_rank in self._source_ranks(
+ prefill_tp=prefill_tp, decode_tp=decode_tp, dst_rank=dst_rank
+ ):
+ src_ptrs = [
+ (src_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * src_item_len
+ for e in range(self.NUM_ENTRIES)
+ ]
+ prefill = self._prefill_manager(
+ backend=backend,
+ src_rank=src_rank,
+ prefill_tp=prefill_tp,
+ total_heads=total_heads,
+ src_ptrs=src_ptrs,
+ item_len=src_item_len,
+ )
+ send(
+ memory=memory,
+ prefill=prefill,
+ dst_rank=dst_rank,
+ decode_tp=decode_tp,
+ dst_ptrs=dst_ptrs,
+ item_len=dst_item_len,
+ )
+
+ np.testing.assert_array_equal(dst_pools, expected)
+
+ def test_decode_ranks_receive_their_heads_of_the_window(self):
+ for backend, send in (
+ (MooncakeKVManager, self._send_mooncake),
+ (NixlKVManager, self._send_nixl),
+ ):
+ for prefill_tp, decode_tp, total_heads in self.TOPOLOGIES:
+ with self.subTest(
+ backend=backend.__name__,
+ prefill_tp=prefill_tp,
+ decode_tp=decode_tp,
+ total_heads=total_heads,
+ ):
+ self._check_transfer(
+ backend=backend,
+ send=send,
+ prefill_tp=prefill_tp,
+ decode_tp=decode_tp,
+ total_heads=total_heads,
+ )
+
+ def test_unsliceable_layouts_are_rejected(self):
+ prefill = object.__new__(CommonKVManager)
+ prefill.attn_tp_size = 2
+ prefill.kv_args = SimpleNamespace(
+ engine_rank=0, page_size=2, total_kv_head_num=8, prefill_start_layer=0
+ )
+ for pp_size, layout, src_item_lens, dst_item_lens, message in (
+ # SWA state has no layer ids, so a later PP stage would land on the
+ # decode peer's first SWA layers.
+ (2, "nhd", [16, 16], [8, 8], "pipeline parallelism"),
+ (1, "nhd", [64], [32], "per-layer K/V"),
+ # One SWA head per rank on both sides, against 8 total KV heads.
+ (1, "nhd", [8, 8], [8, 8], "do not split"),
+ # Item lengths that are valid NHD, so only the layout can reject.
+ (1, "vectorized_5d", [16, 16], [8, 8], "NHD KV cache layout"),
+ ):
+ prefill.pp_size = pp_size
+ prefill.kv_args.swa_kv_cache_layout = layout
+ with self.subTest(message=message):
+ with self.assertRaisesRegex(RuntimeError, message):
+ prefill._get_mha_head_slice_blocks(
+ src_ptrs=[0x1000 * i for i in range(len(src_item_lens))],
+ src_item_lens=src_item_lens,
+ dst_ptrs=[0x9000 * i for i in range(len(dst_item_lens))],
+ dst_item_lens=dst_item_lens,
+ src_page_indices=np.array([1]),
+ dst_page_indices=np.array([2]),
+ dst_attn_tp_size=4,
+ dst_tp_rank=0,
+ )
+
+
class TestMooncakeTransferInfoIsDummy(unittest.TestCase):
"""Truth table for mooncake's payload-inferred is_dummy, with frames built
as KVSender sends them: kv and aux are empty iff dummy, state indices are
diff --git a/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py b/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py
index 8350b5d54c..27c0fe6a8a 100644
--- a/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py
+++ b/test/registered/unit/disaggregation/test_staging_draft_kv_slots.py
@@ -9,6 +9,8 @@ draft pool exists.
import unittest
+from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager
+from sglang.srt.disaggregation.nixl.conn import NixlKVManager
from sglang.srt.disaggregation.utils import (
build_staging_slot_metadata,
build_transfer_entry_pairs,
@@ -50,6 +52,15 @@ class TestStagingDraftKvSlots(CustomTestCase):
self.assertEqual(slot_ids, [87, 91, 92, 87, 91, 92])
self.assertNotEqual(slot_ids, _kv_layer_ids(target, draft))
+ def test_every_backend_setter_accepts_slot_layer_ids(self):
+ # Shared staging init passes slot_layer_ids to every backend's setter.
+ for backend in (MooncakeKVManager, NixlKVManager):
+ with self.subTest(backend=backend.__name__):
+ manager = object.__new__(backend)
+ manager.set_kv_buffer_tensors(
+ ["tK87"], ["tV87"], 16, slot_layer_ids=[87, 87]
+ )
+
def test_without_draft_matches_kv_layer_ids(self):
# The two orders coincide with no draft pool, so every deployment that
# predates draft KV must keep its exact slot labelling.