[PD] Support heterogeneous TP for hybrid SWA KV transfer (#40909)

This commit is contained in:
Jimmy Shong
2026-10-12 23:21:54 -07:00
committed by GitHub
parent 603a05e6c0
commit 3eebf987ed
9 changed files with 583 additions and 3 deletions
@@ -81,6 +81,9 @@ class KVArgs:
# scatter transfer can slice each independently head-sharded sub-block; None
# per tensor when the single contiguous slice already matches the layout.
state_conv_shard_groups: List[List[Optional[List[int]]]]
# Physical layout of the SWA sub-pool ("nhd", "hnd" or "vectorized_5d");
# None when the SWA state is not held in an MHA sub-pool.
swa_kv_cache_layout: Optional[str]
ib_device: str
gpu_id: int
kv_head_num: int
@@ -1230,6 +1230,26 @@ class CommonKVManager(BaseKVManager):
"servers (including prefill CP ranks for MLA)"
)
# Every send path rejects head-sharded MXFP8 block scales across
# attention TP sizes, so each request would fail at its first transfer.
if (
info.attn_tp_size != self.attn_tp_size
and not self.is_mla_backend
and not self.is_hybrid_mla_backend
):
block_scale_states = [
st
for st in getattr(self.kv_args, "state_types", None) or []
if st in (StateType.BLOCK_SCALE, StateType.BLOCK_SCALE_SWA)
]
if block_scale_states:
raise RuntimeError(
"PD Disaggregation does NOT support different attention TP "
"sizes with MXFP8 KV cache: prefill attn_tp_size="
f"{info.attn_tp_size}, decode attn_tp_size={self.attn_tp_size}, "
f"state types {[st.value for st in block_scale_states]}."
)
if self.dcp_size > 1:
if not (self.is_mla_backend or self.is_hybrid_mla_backend):
raise RuntimeError(
@@ -1541,6 +1561,108 @@ class CommonKVManager(BaseKVManager):
layers_current_pp_stage = len(src_k_ptrs)
return src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage
def _get_mha_head_slice_blocks(
self,
src_ptrs: List[int],
src_item_lens: List[int],
dst_ptrs: List[int],
dst_item_lens: List[int],
src_page_indices: npt.NDArray[np.int32],
dst_page_indices: npt.NDArray[np.int32],
dst_attn_tp_size: int,
dst_tp_rank: int,
) -> List[Tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint64], int]]:
# SWA state has no layer ids and its pool starts every PP stage at layer
# 0, so a prefill stage's entries cannot be paired with the decode layers.
if self.pp_size > 1:
raise RuntimeError(
"PD Disaggregation does NOT support PD different TP sizes for "
"non-MLA SWA hybrid models with pipeline parallelism yet."
)
# The addresses below assume NHD pages [page_size, heads, head_dim]; HND
# and vectorized_5d pages have the same size in another element order.
if self.kv_args.swa_kv_cache_layout != "nhd":
raise RuntimeError(
"Head-sliced PD transfer needs the NHD KV cache layout, got "
f"{self.kv_args.swa_kv_cache_layout}"
)
from sglang.srt.disaggregation.common.staging_buffer import (
compute_head_slice_params,
resolve_total_kv_heads,
)
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, num_layers = (
self.get_mha_kv_ptrs_with_pp(src_ptrs, dst_ptrs)
)
_, _, dst_k_lens, dst_v_lens, _ = self.get_mha_kv_ptrs_with_pp(
src_item_lens, dst_item_lens
)
entries = list(
zip(
src_k_ptrs + src_v_ptrs,
dst_k_ptrs + dst_v_ptrs,
src_item_lens,
dst_k_lens + dst_v_lens,
)
)
# A single page-envelope region (unified memory) has no per-layer K/V
# split to slice heads from.
if len(entries) != len(src_ptrs) or 2 * num_layers != len(src_ptrs):
raise RuntimeError(
"Head-sliced PD transfer needs matching per-layer K/V buffers: "
f"prefill has {len(src_ptrs)} entries, decode has {len(dst_ptrs)}"
)
if len(src_page_indices) != len(dst_page_indices):
raise RuntimeError(
"Head-sliced PD transfer page count mismatch: "
f"prefill={len(src_page_indices)}, decode={len(dst_page_indices)}"
)
total_kv_heads = resolve_total_kv_heads(self.kv_args, self.attn_tp_size)
src_head_start, num_heads, dst_head_start, _ = compute_head_slice_params(
src_attn_tp_size=self.attn_tp_size,
dst_attn_tp_size=dst_attn_tp_size,
src_tp_rank=self.kv_args.engine_rank,
dst_tp_rank=dst_tp_rank,
total_kv_heads=total_kv_heads,
)
src_heads = max(1, total_kv_heads // self.attn_tp_size)
dst_heads = max(1, total_kv_heads // dst_attn_tp_size)
page_size = self.kv_args.page_size
tokens = np.arange(page_size, dtype=np.uint64)
src_pages = np.asarray(src_page_indices, dtype=np.uint64)[:, None]
dst_pages = np.asarray(dst_page_indices, dtype=np.uint64)[:, None]
blocks = []
for src_ptr, dst_ptr, src_item_len, dst_item_len in entries:
head_len = src_item_len // (page_size * src_heads)
# Both peers must hold whole heads of one size, or this state is
# sharded differently from total_kv_head_num.
if (
head_len * page_size * src_heads != src_item_len
or head_len * page_size * dst_heads != dst_item_len
):
raise RuntimeError(
f"KV item lengths (prefill={src_item_len}, decode={dst_item_len}) "
f"do not split into {src_heads} and {dst_heads} heads of "
f"{total_kv_heads} total with page_size={page_size}"
)
# Each page item is [page_size, heads, head_dim]: one block per token.
src_addrs = (
np.uint64(src_ptr)
+ src_pages * np.uint64(src_item_len)
+ tokens * np.uint64(src_heads * head_len)
+ np.uint64(src_head_start * head_len)
)
dst_addrs = (
np.uint64(dst_ptr)
+ dst_pages * np.uint64(dst_item_len)
+ tokens * np.uint64(dst_heads * head_len)
+ np.uint64(dst_head_start * head_len)
)
blocks.append((src_addrs.ravel(), dst_addrs.ravel(), num_heads * head_len))
return blocks
def get_mla_kv_ptrs_with_pp(
self,
src_kv_ptrs: List[int],
@@ -1966,12 +1966,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
and self.attn_tp_size
!= target_rank_registration_info.dst_attn_tp_size
)
if (
slice_kv_heads = (
has_heterogeneous_attn_tp
and not self.is_mla_backend
and not self.is_hybrid_mla_backend
and not is_qwen4_qsa_state
):
)
if slice_kv_heads and st != StateType.SWA:
raise RuntimeError(
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
)
@@ -2038,6 +2039,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_indices = src_indices[: len(dst_indices_local)]
else:
dst_indices_local = dst_indices_local[: len(src_indices)]
if slice_kv_heads:
rc = (
self._send_swa_state_slice(
req=req,
prefill_swa_page_indices=src_indices,
src_state_data_ptrs=src_data_ptrs,
src_state_item_lens=src_item_lens,
dst_state_data_ptrs=dst_data_ptrs,
dst_swa_page_indices=dst_indices_local,
dst_state_item_lens=dst_item_lens,
dst_tp_rank=target_rank_registration_info.dst_tp_rank,
dst_attn_tp_size=(
target_rank_registration_info.dst_attn_tp_size
),
executor=executor,
)
or rc
)
continue
rc = (
self._send_kvcache_generic(
mooncake_session_id=req.mooncake_session_id,
@@ -2292,6 +2312,49 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
return self._transfer_data(req.mooncake_session_id, transfer_blocks)
def _send_swa_state_slice(
self,
req: TransferInfo,
prefill_swa_page_indices: List[int],
src_state_data_ptrs: List[int],
src_state_item_lens: List[int],
dst_state_data_ptrs: List[int],
dst_swa_page_indices: List[int],
dst_state_item_lens: List[int],
dst_tp_rank: int,
dst_attn_tp_size: int,
executor: concurrent.futures.ThreadPoolExecutor,
) -> int:
blocks = self._get_mha_head_slice_blocks(
src_ptrs=src_state_data_ptrs,
src_item_lens=src_state_item_lens,
dst_ptrs=dst_state_data_ptrs,
dst_item_lens=dst_state_item_lens,
src_page_indices=np.asarray(prefill_swa_page_indices),
dst_page_indices=np.asarray(dst_swa_page_indices),
dst_attn_tp_size=dst_attn_tp_size,
dst_tp_rank=dst_tp_rank,
)
logger.warning_once(
"Using SWA state head-slice transfer for different attention TP sizes: "
f"prefill={self.attn_tp_size}, decode={dst_attn_tp_size}."
)
futures = [
executor.submit(
self._transfer_data,
req.mooncake_session_id,
list(
zip(
src_addrs.tolist(),
dst_addrs.tolist(),
[length] * src_addrs.size,
)
),
)
for src_addrs, dst_addrs, length in blocks
]
return self._await_transfer_futures(futures)
def _poison_unsettled_send(self, room: int) -> None:
"""Keep the drain ack from firing after a submitted transfer failed.
+83 -1
View File
@@ -621,7 +621,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
f"(ptr=0x{ptr:x}, size={size})"
)
def set_kv_buffer_tensors(self, k_buffers: list, v_buffers: list, page_size: int):
def set_kv_buffer_tensors(
self,
k_buffers: list,
v_buffers: list,
page_size: int,
slot_layer_ids: Optional[List[int]] = None,
):
# NOTE: matches mooncake behavior -- staging buffers are now
# created in __init__ (per-worker), independent of the kv
# tensors. This setter only stashes the tensor metadata used by
@@ -630,6 +636,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
"k_buffers": k_buffers,
"v_buffers": v_buffers,
"page_size": page_size,
"slot_layer_ids": list(slot_layer_ids or []),
}
def register_staging_room_bootstrap(self, room, bootstrap_infos, receiver):
@@ -2608,6 +2615,63 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
raise Exception("Failed to post Mamba state slice transfer")
return xfer_handle
def _send_swa_state_slice(
self,
peer_name: str,
prefill_swa_page_indices: List[int],
src_state_data_ptrs: List[int],
src_state_item_lens: List[int],
dst_state_data_ptrs: List[int],
dst_swa_page_indices: List[int],
dst_state_item_lens: List[int],
dst_gpu_id: int,
notif: str,
decode_tp_size: int,
decode_tp_rank: int,
):
blocks = self._get_mha_head_slice_blocks(
src_ptrs=src_state_data_ptrs,
src_item_lens=src_state_item_lens,
dst_ptrs=dst_state_data_ptrs,
dst_item_lens=dst_state_item_lens,
src_page_indices=np.asarray(prefill_swa_page_indices),
dst_page_indices=np.asarray(dst_swa_page_indices),
dst_attn_tp_size=decode_tp_size,
dst_tp_rank=decode_tp_rank,
)
logger.warning_once(
"Using SWA state head-slice transfer for different attention TP sizes: "
f"prefill={self.attn_tp_size}, decode={decode_tp_size}."
)
src_reqs, dst_reqs = [], []
for src_addrs, dst_addrs, length in blocks:
lengths = np.full(src_addrs.size, length, dtype=np.uint64)
src_reqs.append(
np.column_stack(
[src_addrs, lengths, np.full_like(lengths, self.kv_args.gpu_id)]
)
)
dst_reqs.append(
np.column_stack([dst_addrs, lengths, np.full_like(lengths, dst_gpu_id)])
)
if not src_reqs:
return None
src_descs = self.agent.get_xfer_descs(np.vstack(src_reqs), "VRAM")
dst_descs = self.agent.get_xfer_descs(np.vstack(dst_reqs), "VRAM")
xfer_handle = self.agent.initialize_xfer(
"WRITE",
src_descs,
dst_descs,
peer_name,
notif.encode("ascii"),
)
if not xfer_handle:
raise Exception("Failed to create SWA state slice transfer")
state = self.agent.transfer(xfer_handle)
if state == "ERR":
raise Exception("Failed to post SWA state slice transfer")
return xfer_handle
def maybe_send_extra(
self,
peer_name: str,
@@ -2743,6 +2807,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
state_type=st,
force_flat=True,
)
elif (
st == StateType.SWA
and not self.is_mla_backend
and self.attn_tp_size != decode_tp_size
):
h = self._send_swa_state_slice(
peer_name=peer_name,
prefill_swa_page_indices=src_indices,
src_state_data_ptrs=src_ptrs,
src_state_item_lens=src_lens,
dst_state_data_ptrs=dst_ptrs,
dst_swa_page_indices=dst_indices,
dst_state_item_lens=dst_lens,
dst_gpu_id=dst_gpu_id,
notif=comp_notif,
decode_tp_size=decode_tp_size,
decode_tp_rank=decode_tp_rank,
)
elif st in (
StateType.SWA,
StateType.BLOCK_SCALE,
@@ -1392,6 +1392,7 @@ def setup_state_kv_args(
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
HybridLinearKVPool,
MHATokenToKVPool,
MHATokenToKVPoolMXFP8,
MiniMaxSparseKVPool,
)
@@ -1407,6 +1408,7 @@ def setup_state_kv_args(
kv_args.state_layer_ids = []
kv_args.is_hybrid_mla_backend = False
kv_args.state_conv_shard_groups = []
kv_args.swa_kv_cache_layout = None
# V4's KVCache is organized by compression-ratio buckets rather than by layer.
kv_args.mla_compression_ratios = (
list(token_to_kv_pool.compression_ratios)
@@ -1457,6 +1459,12 @@ def setup_state_kv_args(
append_state_component(
kv_args, StateType.SWA, data_ptrs, data_lens, item_lens
)
if isinstance(token_to_kv_pool, SWAKVPool) and isinstance(
token_to_kv_pool.swa_kv_pool, MHATokenToKVPool
):
kv_args.swa_kv_cache_layout = (
token_to_kv_pool.swa_kv_pool.kv_cache_layout
)
# MXFP8 KV: each sub-pool's block scales ride as their own component
# so they inherit the index payload of the KV they describe.
# Only the concrete SWAKVPool owns a full sub-pool; other