[PD] Support heterogeneous TP for hybrid SWA KV transfer (#40909)

This commit is contained in:
Jimmy Shong
2026-10-12 23:21:54 -07:00
committed by GitHub
parent 603a05e6c0
commit 3eebf987ed
9 changed files with 583 additions and 3 deletions
@@ -347,6 +347,8 @@ Enable the staging buffer when prefill and decode use **different TP sizes** wit
> **Note:** The staging buffer is designed for non-MLA models (e.g. GQA, MHA). MLA models (e.g. DeepSeek-V2/V3) should not enable this flag.
> **Note:** You can use different attention TP sizes for hybrid sliding-window models (e.g. `openai/gpt-oss-120b`) with Mooncake or NIXL. For unequal TP, both backends transfer the sliding-window KV by token and KV-head slice. The staging buffer carries only full-attention KV. Use `--pp-size 1` and leave `--enable-unified-memory` disabled for this configuration. MXFP8 KV cache is not supported for unequal-TP SWA transfer.
### Environment Variables
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
@@ -81,6 +81,9 @@ class KVArgs:
# scatter transfer can slice each independently head-sharded sub-block; None
# per tensor when the single contiguous slice already matches the layout.
state_conv_shard_groups: List[List[Optional[List[int]]]]
# Physical layout of the SWA sub-pool ("nhd", "hnd" or "vectorized_5d");
# None when the SWA state is not held in an MHA sub-pool.
swa_kv_cache_layout: Optional[str]
ib_device: str
gpu_id: int
kv_head_num: int
@@ -1230,6 +1230,26 @@ class CommonKVManager(BaseKVManager):
"servers (including prefill CP ranks for MLA)"
)
# Every send path rejects head-sharded MXFP8 block scales across
# attention TP sizes, so each request would fail at its first transfer.
if (
info.attn_tp_size != self.attn_tp_size
and not self.is_mla_backend
and not self.is_hybrid_mla_backend
):
block_scale_states = [
st
for st in getattr(self.kv_args, "state_types", None) or []
if st in (StateType.BLOCK_SCALE, StateType.BLOCK_SCALE_SWA)
]
if block_scale_states:
raise RuntimeError(
"PD Disaggregation does NOT support different attention TP "
"sizes with MXFP8 KV cache: prefill attn_tp_size="
f"{info.attn_tp_size}, decode attn_tp_size={self.attn_tp_size}, "
f"state types {[st.value for st in block_scale_states]}."
)
if self.dcp_size > 1:
if not (self.is_mla_backend or self.is_hybrid_mla_backend):
raise RuntimeError(
@@ -1541,6 +1561,108 @@ class CommonKVManager(BaseKVManager):
layers_current_pp_stage = len(src_k_ptrs)
return src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage
def _get_mha_head_slice_blocks(
self,
src_ptrs: List[int],
src_item_lens: List[int],
dst_ptrs: List[int],
dst_item_lens: List[int],
src_page_indices: npt.NDArray[np.int32],
dst_page_indices: npt.NDArray[np.int32],
dst_attn_tp_size: int,
dst_tp_rank: int,
) -> List[Tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint64], int]]:
# SWA state has no layer ids and its pool starts every PP stage at layer
# 0, so a prefill stage's entries cannot be paired with the decode layers.
if self.pp_size > 1:
raise RuntimeError(
"PD Disaggregation does NOT support PD different TP sizes for "
"non-MLA SWA hybrid models with pipeline parallelism yet."
)
# The addresses below assume NHD pages [page_size, heads, head_dim]; HND
# and vectorized_5d pages have the same size in another element order.
if self.kv_args.swa_kv_cache_layout != "nhd":
raise RuntimeError(
"Head-sliced PD transfer needs the NHD KV cache layout, got "
f"{self.kv_args.swa_kv_cache_layout}"
)
from sglang.srt.disaggregation.common.staging_buffer import (
compute_head_slice_params,
resolve_total_kv_heads,
)
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, num_layers = (
self.get_mha_kv_ptrs_with_pp(src_ptrs, dst_ptrs)
)
_, _, dst_k_lens, dst_v_lens, _ = self.get_mha_kv_ptrs_with_pp(
src_item_lens, dst_item_lens
)
entries = list(
zip(
src_k_ptrs + src_v_ptrs,
dst_k_ptrs + dst_v_ptrs,
src_item_lens,
dst_k_lens + dst_v_lens,
)
)
# A single page-envelope region (unified memory) has no per-layer K/V
# split to slice heads from.
if len(entries) != len(src_ptrs) or 2 * num_layers != len(src_ptrs):
raise RuntimeError(
"Head-sliced PD transfer needs matching per-layer K/V buffers: "
f"prefill has {len(src_ptrs)} entries, decode has {len(dst_ptrs)}"
)
if len(src_page_indices) != len(dst_page_indices):
raise RuntimeError(
"Head-sliced PD transfer page count mismatch: "
f"prefill={len(src_page_indices)}, decode={len(dst_page_indices)}"
)
total_kv_heads = resolve_total_kv_heads(self.kv_args, self.attn_tp_size)
src_head_start, num_heads, dst_head_start, _ = compute_head_slice_params(
src_attn_tp_size=self.attn_tp_size,
dst_attn_tp_size=dst_attn_tp_size,
src_tp_rank=self.kv_args.engine_rank,
dst_tp_rank=dst_tp_rank,
total_kv_heads=total_kv_heads,
)
src_heads = max(1, total_kv_heads // self.attn_tp_size)
dst_heads = max(1, total_kv_heads // dst_attn_tp_size)
page_size = self.kv_args.page_size
tokens = np.arange(page_size, dtype=np.uint64)
src_pages = np.asarray(src_page_indices, dtype=np.uint64)[:, None]
dst_pages = np.asarray(dst_page_indices, dtype=np.uint64)[:, None]
blocks = []
for src_ptr, dst_ptr, src_item_len, dst_item_len in entries:
head_len = src_item_len // (page_size * src_heads)
# Both peers must hold whole heads of one size, or this state is
# sharded differently from total_kv_head_num.
if (
head_len * page_size * src_heads != src_item_len
or head_len * page_size * dst_heads != dst_item_len
):
raise RuntimeError(
f"KV item lengths (prefill={src_item_len}, decode={dst_item_len}) "
f"do not split into {src_heads} and {dst_heads} heads of "
f"{total_kv_heads} total with page_size={page_size}"
)
# Each page item is [page_size, heads, head_dim]: one block per token.
src_addrs = (
np.uint64(src_ptr)
+ src_pages * np.uint64(src_item_len)
+ tokens * np.uint64(src_heads * head_len)
+ np.uint64(src_head_start * head_len)
)
dst_addrs = (
np.uint64(dst_ptr)
+ dst_pages * np.uint64(dst_item_len)
+ tokens * np.uint64(dst_heads * head_len)
+ np.uint64(dst_head_start * head_len)
)
blocks.append((src_addrs.ravel(), dst_addrs.ravel(), num_heads * head_len))
return blocks
def get_mla_kv_ptrs_with_pp(
self,
src_kv_ptrs: List[int],
@@ -1966,12 +1966,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
and self.attn_tp_size
!= target_rank_registration_info.dst_attn_tp_size
)
if (
slice_kv_heads = (
has_heterogeneous_attn_tp
and not self.is_mla_backend
and not self.is_hybrid_mla_backend
and not is_qwen4_qsa_state
):
)
if slice_kv_heads and st != StateType.SWA:
raise RuntimeError(
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
)
@@ -2038,6 +2039,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_indices = src_indices[: len(dst_indices_local)]
else:
dst_indices_local = dst_indices_local[: len(src_indices)]
if slice_kv_heads:
rc = (
self._send_swa_state_slice(
req=req,
prefill_swa_page_indices=src_indices,
src_state_data_ptrs=src_data_ptrs,
src_state_item_lens=src_item_lens,
dst_state_data_ptrs=dst_data_ptrs,
dst_swa_page_indices=dst_indices_local,
dst_state_item_lens=dst_item_lens,
dst_tp_rank=target_rank_registration_info.dst_tp_rank,
dst_attn_tp_size=(
target_rank_registration_info.dst_attn_tp_size
),
executor=executor,
)
or rc
)
continue
rc = (
self._send_kvcache_generic(
mooncake_session_id=req.mooncake_session_id,
@@ -2292,6 +2312,49 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
return self._transfer_data(req.mooncake_session_id, transfer_blocks)
def _send_swa_state_slice(
self,
req: TransferInfo,
prefill_swa_page_indices: List[int],
src_state_data_ptrs: List[int],
src_state_item_lens: List[int],
dst_state_data_ptrs: List[int],
dst_swa_page_indices: List[int],
dst_state_item_lens: List[int],
dst_tp_rank: int,
dst_attn_tp_size: int,
executor: concurrent.futures.ThreadPoolExecutor,
) -> int:
blocks = self._get_mha_head_slice_blocks(
src_ptrs=src_state_data_ptrs,
src_item_lens=src_state_item_lens,
dst_ptrs=dst_state_data_ptrs,
dst_item_lens=dst_state_item_lens,
src_page_indices=np.asarray(prefill_swa_page_indices),
dst_page_indices=np.asarray(dst_swa_page_indices),
dst_attn_tp_size=dst_attn_tp_size,
dst_tp_rank=dst_tp_rank,
)
logger.warning_once(
"Using SWA state head-slice transfer for different attention TP sizes: "
f"prefill={self.attn_tp_size}, decode={dst_attn_tp_size}."
)
futures = [
executor.submit(
self._transfer_data,
req.mooncake_session_id,
list(
zip(
src_addrs.tolist(),
dst_addrs.tolist(),
[length] * src_addrs.size,
)
),
)
for src_addrs, dst_addrs, length in blocks
]
return self._await_transfer_futures(futures)
def _poison_unsettled_send(self, room: int) -> None:
"""Keep the drain ack from firing after a submitted transfer failed.
+83 -1
View File
@@ -621,7 +621,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
f"(ptr=0x{ptr:x}, size={size})"
)
def set_kv_buffer_tensors(self, k_buffers: list, v_buffers: list, page_size: int):
def set_kv_buffer_tensors(
self,
k_buffers: list,
v_buffers: list,
page_size: int,
slot_layer_ids: Optional[List[int]] = None,
):
# NOTE: matches mooncake behavior -- staging buffers are now
# created in __init__ (per-worker), independent of the kv
# tensors. This setter only stashes the tensor metadata used by
@@ -630,6 +636,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
"k_buffers": k_buffers,
"v_buffers": v_buffers,
"page_size": page_size,
"slot_layer_ids": list(slot_layer_ids or []),
}
def register_staging_room_bootstrap(self, room, bootstrap_infos, receiver):
@@ -2608,6 +2615,63 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
raise Exception("Failed to post Mamba state slice transfer")
return xfer_handle
def _send_swa_state_slice(
self,
peer_name: str,
prefill_swa_page_indices: List[int],
src_state_data_ptrs: List[int],
src_state_item_lens: List[int],
dst_state_data_ptrs: List[int],
dst_swa_page_indices: List[int],
dst_state_item_lens: List[int],
dst_gpu_id: int,
notif: str,
decode_tp_size: int,
decode_tp_rank: int,
):
blocks = self._get_mha_head_slice_blocks(
src_ptrs=src_state_data_ptrs,
src_item_lens=src_state_item_lens,
dst_ptrs=dst_state_data_ptrs,
dst_item_lens=dst_state_item_lens,
src_page_indices=np.asarray(prefill_swa_page_indices),
dst_page_indices=np.asarray(dst_swa_page_indices),
dst_attn_tp_size=decode_tp_size,
dst_tp_rank=decode_tp_rank,
)
logger.warning_once(
"Using SWA state head-slice transfer for different attention TP sizes: "
f"prefill={self.attn_tp_size}, decode={decode_tp_size}."
)
src_reqs, dst_reqs = [], []
for src_addrs, dst_addrs, length in blocks:
lengths = np.full(src_addrs.size, length, dtype=np.uint64)
src_reqs.append(
np.column_stack(
[src_addrs, lengths, np.full_like(lengths, self.kv_args.gpu_id)]
)
)
dst_reqs.append(
np.column_stack([dst_addrs, lengths, np.full_like(lengths, dst_gpu_id)])
)
if not src_reqs:
return None
src_descs = self.agent.get_xfer_descs(np.vstack(src_reqs), "VRAM")
dst_descs = self.agent.get_xfer_descs(np.vstack(dst_reqs), "VRAM")
xfer_handle = self.agent.initialize_xfer(
"WRITE",
src_descs,
dst_descs,
peer_name,
notif.encode("ascii"),
)
if not xfer_handle:
raise Exception("Failed to create SWA state slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post SWA state slice transfer")
return xfer_handle
def maybe_send_extra(
self,
peer_name: str,
@@ -2743,6 +2807,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
state_type=st,
force_flat=True,
)
elif (
st == StateType.SWA
and not self.is_mla_backend
and self.attn_tp_size != decode_tp_size
):
h = self._send_swa_state_slice(
peer_name=peer_name,
prefill_swa_page_indices=src_indices,
src_state_data_ptrs=src_ptrs,
src_state_item_lens=src_lens,
dst_state_data_ptrs=dst_ptrs,
dst_swa_page_indices=dst_indices,
dst_state_item_lens=dst_lens,
dst_gpu_id=dst_gpu_id,
notif=comp_notif,
decode_tp_size=decode_tp_size,
decode_tp_rank=decode_tp_rank,
)
elif st in (
StateType.SWA,
StateType.BLOCK_SCALE,
@@ -1392,6 +1392,7 @@ def setup_state_kv_args(
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
HybridLinearKVPool,
MHATokenToKVPool,
MHATokenToKVPoolMXFP8,
MiniMaxSparseKVPool,
)
@@ -1407,6 +1408,7 @@ def setup_state_kv_args(
kv_args.state_layer_ids = []
kv_args.is_hybrid_mla_backend = False
kv_args.state_conv_shard_groups = []
kv_args.swa_kv_cache_layout = None
# V4's KVCache is organized by compression-ratio buckets rather than by layer.
kv_args.mla_compression_ratios = (
list(token_to_kv_pool.compression_ratios)
@@ -1457,6 +1459,12 @@ def setup_state_kv_args(
append_state_component(
kv_args, StateType.SWA, data_ptrs, data_lens, item_lens
)
if isinstance(token_to_kv_pool, SWAKVPool) and isinstance(
token_to_kv_pool.swa_kv_pool, MHATokenToKVPool
):
kv_args.swa_kv_cache_layout = (
token_to_kv_pool.swa_kv_pool.kv_cache_layout
)
# MXFP8 KV: each sub-pool's block scales ride as their own component
# so they inherit the index payload of the KV they describe.
# Only the concrete SWAKVPool owns a full sub-pool; other
@@ -0,0 +1,43 @@
"""Prefill TP2 -> decode TP1: two prefill ranks write disjoint KV heads per SWA page."""
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
configure_nixl_pd_backend,
)
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE,
try_cached_model,
)
register_cuda_ci(est_time=300, stage="extra-b", runner_config="4-gpu-h100")
SWA_SERVER_ARGS = ["--page-size", "64", "--attention-backend", "triton"]
class TestDisaggregationSWAPrefillLargerTP(PDDisaggregationServerBase, GSM8KMixin):
prefill_tp_size = 2
decode_tp_size = 1
decode_base_gpu_id = 2
extra_prefill_args = SWA_SERVER_ARGS
extra_decode_args = SWA_SERVER_ARGS
# Same eval and floor as the equal-TP case in
# test_disaggregation_decode_radix_cache_swa.py.
gsm8k_score_threshold = 0.45
gsm8k_num_examples = 500
gsm8k_num_threads = 100
gsm8k_num_shots = 6
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE)
configure_nixl_pd_backend(cls)
cls.launch_all()
if __name__ == "__main__":
unittest.main()
@@ -1,3 +1,4 @@
import concurrent.futures
import struct
import threading
import unittest
@@ -37,6 +38,7 @@ from sglang.srt.disaggregation.mooncake.conn import (
MooncakeKVSender,
TransferInfo,
)
from sglang.srt.disaggregation.nixl.conn import NixlKVManager
from sglang.srt.disaggregation.utils import (
MetadataBuffers,
build_transfer_entry_pairs,
@@ -101,6 +103,7 @@ class TestMixedSWAKVBlockScales(CustomTestCase):
MHATokenToKVPoolMXFP8 if quantized else MHATokenToKVPool
)
pool.get_contiguous_buf_infos = Mock(return_value=swa_infos)
pool.kv_cache_layout = "nhd"
if quantized:
pool.get_kv_scale_buf_infos = Mock(return_value=scale_infos)
setattr(target, name, pool)
@@ -1062,6 +1065,249 @@ class TestQwen4StateWire(unittest.TestCase):
)
class _MemcpyNixlAgent:
"""NIXL agent stand-in whose WRITE copies bytes within one host array."""
def __init__(self, memory: np.ndarray):
self.memory = memory
def get_xfer_descs(self, reqs, mem_type):
return reqs
def initialize_xfer(self, op, src_descs, dst_descs, peer_name, notif):
return src_descs, dst_descs
def transfer(self, handle):
for (src, length, _), (dst, dst_length, _) in zip(*handle):
assert length == dst_length
src, dst, length = int(src), int(dst), int(length)
self.memory[dst : dst + length] = self.memory[src : src + length]
return "DONE"
class TestHeteroTpSwaStateTransfer(CustomTestCase):
PAGE_SIZE = 2
HEAD_BYTES = 4
NUM_ENTRIES = 4 # K then V of two SWA layers
NUM_PAGES = 6
# The window occupies unrelated physical pages on the two sides.
SRC_PAGES = [4, 1, 5]
DST_PAGES = [2, 5, 0]
# (prefill attn TP, decode attn TP, total KV heads); the last three
# replicate KV heads across the attention TP ranks of one side.
TOPOLOGIES = [(2, 4, 8), (4, 2, 8), (1, 4, 2), (8, 2, 4), (2, 8, 4)]
@staticmethod
def _rank_heads(rank, tp_size, total_heads):
# Model-side KV sharding: ranks beyond the head count replicate a head.
if total_heads >= tp_size:
per_rank = total_heads // tp_size
return list(range(rank * per_rank, (rank + 1) * per_rank))
return [rank // (tp_size // total_heads)]
@staticmethod
def _source_ranks(prefill_tp, decode_tp, dst_rank):
decode = object.__new__(CommonKVManager)
decode.attn_tp_size = decode_tp
decode.kv_args = SimpleNamespace(engine_rank=dst_rank)
decode.is_mla_backend = decode.is_hybrid_mla_backend = False
decode.attn_cp_size, decode.attn_cp_rank = 1, 0
decode.pp_size, decode.pp_rank = 1, 0
info = SimpleNamespace(attn_tp_size=prefill_tp, attn_cp_size=1, pp_size=1)
decode._resolve_rank_mapping(info)
return info.target_tp_ranks
def _send_mooncake(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len):
def copy_blocks(session_id, blocks):
for src, dst, length in blocks:
memory[dst : dst + length] = memory[src : src + length]
return 0
prefill.is_hybrid_mla_backend = False
prefill.enable_deferred_decode_kv_release = False
prefill._transfer_data = copy_blocks
req = SimpleNamespace(
mooncake_session_id="decode", dst_state_indices=[self.DST_PAGES]
)
dst_info = SimpleNamespace(
dst_state_data_ptrs=[dst_ptrs],
dst_state_item_lens=[[item_len] * len(dst_ptrs)],
dst_state_dim_per_tensor=[[]],
dst_state_layer_ids=[[]],
dst_attn_tp_size=decode_tp,
dst_tp_rank=dst_rank,
)
with concurrent.futures.ThreadPoolExecutor(1) as executor:
rc = prefill.maybe_send_extra(
req, [np.array(self.SRC_PAGES, dtype=np.int32)], executor, dst_info
)
self.assertEqual(rc, 0)
def _send_nixl(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len):
prefill.agent = _MemcpyNixlAgent(memory)
handles = prefill.maybe_send_extra(
"decode",
[np.array(self.SRC_PAGES, dtype=np.int32)],
[dst_ptrs],
[self.DST_PAGES],
0,
"7_state_0",
decode_tp,
decode_tp_rank=dst_rank,
dst_state_item_lens=[[item_len] * len(dst_ptrs)],
)
self.assertEqual(len(handles), 1)
def _prefill_manager(
self, *, backend, src_rank, prefill_tp, total_heads, src_ptrs, item_len
):
prefill = object.__new__(backend)
prefill.attn_tp_size = prefill_tp
prefill.is_mla_backend = False
prefill.pp_size = 1
prefill.kv_args = SimpleNamespace(
engine_rank=src_rank,
gpu_id=0,
page_size=self.PAGE_SIZE,
total_kv_head_num=total_heads,
prefill_start_layer=0,
state_types=[StateType.SWA],
state_data_ptrs=[src_ptrs],
state_item_lens=[[item_len] * len(src_ptrs)],
state_dim_per_tensor=[[]],
state_layer_ids=[[]],
swa_kv_cache_layout="nhd",
)
return prefill
def _pool_shape(self, tp_size, total_heads):
# [rank, entry, page, token, local head, byte]
heads = len(self._rank_heads(rank=0, tp_size=tp_size, total_heads=total_heads))
return (
tp_size,
self.NUM_ENTRIES,
self.NUM_PAGES,
self.PAGE_SIZE,
heads,
self.HEAD_BYTES,
)
def _check_transfer(self, backend, send, prefill_tp, decode_tp, total_heads):
rng = np.random.default_rng(0)
window_shape = (self.NUM_ENTRIES, len(self.SRC_PAGES), self.PAGE_SIZE)
window = rng.integers(
0, 256, window_shape + (total_heads, self.HEAD_BYTES), dtype=np.uint8
)
src_shape = self._pool_shape(tp_size=prefill_tp, total_heads=total_heads)
dst_shape = self._pool_shape(tp_size=decode_tp, total_heads=total_heads)
src_bytes = int(np.prod(src_shape))
# Prefill pages outside the window hold bytes a wrong page stride copies.
memory = np.concatenate(
[
rng.integers(0, 256, src_bytes, dtype=np.uint8),
np.zeros(int(np.prod(dst_shape)), dtype=np.uint8),
]
)
src_pools = memory[:src_bytes].reshape(src_shape)
dst_pools = memory[src_bytes:].reshape(dst_shape)
expected = np.zeros_like(dst_pools)
for rank in range(prefill_tp):
heads = self._rank_heads(
rank=rank, tp_size=prefill_tp, total_heads=total_heads
)
src_pools[rank][:, self.SRC_PAGES] = window[..., heads, :]
for rank in range(decode_tp):
heads = self._rank_heads(
rank=rank, tp_size=decode_tp, total_heads=total_heads
)
expected[rank][:, self.DST_PAGES] = window[..., heads, :]
src_item_len = int(np.prod(src_shape[3:]))
dst_item_len = int(np.prod(dst_shape[3:]))
for dst_rank in range(decode_tp):
dst_ptrs = [
src_bytes
+ (dst_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * dst_item_len
for e in range(self.NUM_ENTRIES)
]
for src_rank in self._source_ranks(
prefill_tp=prefill_tp, decode_tp=decode_tp, dst_rank=dst_rank
):
src_ptrs = [
(src_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * src_item_len
for e in range(self.NUM_ENTRIES)
]
prefill = self._prefill_manager(
backend=backend,
src_rank=src_rank,
prefill_tp=prefill_tp,
total_heads=total_heads,
src_ptrs=src_ptrs,
item_len=src_item_len,
)
send(
memory=memory,
prefill=prefill,
dst_rank=dst_rank,
decode_tp=decode_tp,
dst_ptrs=dst_ptrs,
item_len=dst_item_len,
)
np.testing.assert_array_equal(dst_pools, expected)
def test_decode_ranks_receive_their_heads_of_the_window(self):
for backend, send in (
(MooncakeKVManager, self._send_mooncake),
(NixlKVManager, self._send_nixl),
):
for prefill_tp, decode_tp, total_heads in self.TOPOLOGIES:
with self.subTest(
backend=backend.__name__,
prefill_tp=prefill_tp,
decode_tp=decode_tp,
total_heads=total_heads,
):
self._check_transfer(
backend=backend,
send=send,
prefill_tp=prefill_tp,
decode_tp=decode_tp,
total_heads=total_heads,
)
def test_unsliceable_layouts_are_rejected(self):
prefill = object.__new__(CommonKVManager)
prefill.attn_tp_size = 2
prefill.kv_args = SimpleNamespace(
engine_rank=0, page_size=2, total_kv_head_num=8, prefill_start_layer=0
)
for pp_size, layout, src_item_lens, dst_item_lens, message in (
# SWA state has no layer ids, so a later PP stage would land on the
# decode peer's first SWA layers.
(2, "nhd", [16, 16], [8, 8], "pipeline parallelism"),
(1, "nhd", [64], [32], "per-layer K/V"),
# One SWA head per rank on both sides, against 8 total KV heads.
(1, "nhd", [8, 8], [8, 8], "do not split"),
# Item lengths that are valid NHD, so only the layout can reject.
(1, "vectorized_5d", [16, 16], [8, 8], "NHD KV cache layout"),
):
prefill.pp_size = pp_size
prefill.kv_args.swa_kv_cache_layout = layout
with self.subTest(message=message):
with self.assertRaisesRegex(RuntimeError, message):
prefill._get_mha_head_slice_blocks(
src_ptrs=[0x1000 * i for i in range(len(src_item_lens))],
src_item_lens=src_item_lens,
dst_ptrs=[0x9000 * i for i in range(len(dst_item_lens))],
dst_item_lens=dst_item_lens,
src_page_indices=np.array([1]),
dst_page_indices=np.array([2]),
dst_attn_tp_size=4,
dst_tp_rank=0,
)
class TestMooncakeTransferInfoIsDummy(unittest.TestCase):
"""Truth table for mooncake's payload-inferred is_dummy, with frames built
as KVSender sends them: kv and aux are empty iff dummy, state indices are
@@ -9,6 +9,8 @@ draft pool exists.
import unittest
from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager
from sglang.srt.disaggregation.nixl.conn import NixlKVManager
from sglang.srt.disaggregation.utils import (
build_staging_slot_metadata,
build_transfer_entry_pairs,
@@ -50,6 +52,15 @@ class TestStagingDraftKvSlots(CustomTestCase):
self.assertEqual(slot_ids, [87, 91, 92, 87, 91, 92])
self.assertNotEqual(slot_ids, _kv_layer_ids(target, draft))
def test_every_backend_setter_accepts_slot_layer_ids(self):
# Shared staging init passes slot_layer_ids to every backend's setter.
for backend in (MooncakeKVManager, NixlKVManager):
with self.subTest(backend=backend.__name__):
manager = object.__new__(backend)
manager.set_kv_buffer_tensors(
["tK87"], ["tV87"], 16, slot_layer_ids=[87, 87]
)
def test_without_draft_matches_kv_layer_ids(self):
# The two orders coincide with no draft pool, so every deployment that
# predates draft KV must keep its exact slot labelling.